From 809d1c29753ce8ab1f3e18b57d6c9606e36551fb Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 14 Nov 2023 10:55:47 -0500 Subject: [PATCH 1/5] add validator tests --- tdrs-backend/tdpservice/parsers/fields.py | 11 +----- tdrs-backend/tdpservice/parsers/row_schema.py | 3 +- .../tdpservice/parsers/test/test_util.py | 34 +++---------------- .../parsers/test/test_validators.py | 29 ++++++++++++++++ tdrs-backend/tdpservice/parsers/validators.py | 12 +++++++ 5 files changed, 48 insertions(+), 41 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 1487cf498..098c4ceff 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -1,19 +1,10 @@ """Datafile field representations.""" import logging +from .validators import value_is_empty logger = logging.getLogger(__name__) -def value_is_empty(value, length): - """Handle 'empty' values as field inputs.""" - empty_values = [ - ' '*length, # ' ' - '#'*length, # '#####' - '_'*length, # '_____' - ] - - return value is None or value in empty_values - class Field: """Provides a mapping between a field name and its position.""" diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index 83885042c..b7fafcdd8 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -1,6 +1,7 @@ """Row schema for datafile.""" from .models import ParserErrorCategoryChoices -from .fields import Field, value_is_empty +from .fields import Field +from .validators import value_is_empty import logging logger = logging.getLogger(__name__) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index 03997597a..3f030daea 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -1,7 +1,7 @@ """Test the methods of RowSchema to ensure parsing and validation work in all individual cases.""" import pytest -from ..fields import Field, value_is_empty +from ..fields import Field from ..row_schema import RowSchema from ..util import SchemaManager @@ -51,7 +51,7 @@ def test_run_preparsing_validators_returns_invalid_and_errors(): def test_parse_line_parses_line_from_schema_to_dict(): - """Test that parse_line parses a string into a dict given start and end indices for all fields.""" + """Test that parse_line parses a string into a dict given start and end indices for all """ line = '12345001' schema = RowSchema( model=dict, @@ -74,7 +74,7 @@ def test_parse_line_parses_line_from_schema_to_dict(): def test_parse_line_parses_line_from_schema_to_object(): - """Test that parse_line parses a string into an object given start and end indices for all fields.""" + """Test that parse_line parses a string into an object given start and end indices for all """ class TestModel: first = None second = None @@ -224,6 +224,7 @@ class TestModel: @pytest.mark.parametrize('first,second', [ + ('', ''), (' ', ' '), ('#', '##'), (None, None), @@ -308,33 +309,6 @@ def test_run_postparsing_validators_returns_invalid_and_errors(): assert errors == ['Value is not valid.'] -@pytest.mark.parametrize("value,length", [ - (None, 0), - (None, 10), - (' ', 5), - ('###', 3) -]) -def test_value_is_empty_returns_true(value, length): - """Test value_is_empty returns valid.""" - result = value_is_empty(value, length) - assert result is True - - -@pytest.mark.parametrize("value,length", [ - (0, 1), - (1, 1), - (10, 2), - ('0', 1), - ('0000', 4), - ('1 ', 5), - ('##3', 3) -]) -def test_value_is_empty_returns_false(value, length): - """Test value_is_empty returns invalid.""" - result = value_is_empty(value, length) - assert result is False - - def test_multi_record_schema_parses_and_validates(): """Test SchemaManager parse_and_validate.""" line = '12345' diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index ecca06561..26f8bfaaf 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -5,6 +5,35 @@ from tdpservice.parsers.test.factories import TanfT1Factory, TanfT2Factory, TanfT3Factory, TanfT5Factory, TanfT6Factory +@pytest.mark.parametrize("value,length", [ + (None, 0), + (None, 10), + (' ', 5), + ('###', 3), + ('', 0), + ('', 10), +]) +def test_value_is_empty_returns_true(value, length): + """Test value_is_empty returns valid.""" + result = validators.value_is_empty(value, length) + assert result is True + + +@pytest.mark.parametrize("value,length", [ + (0, 1), + (1, 1), + (10, 2), + ('0', 1), + ('0000', 4), + ('1 ', 5), + ('##3', 3), +]) +def test_value_is_empty_returns_false(value, length): + """Test value_is_empty returns invalid.""" + result = validators.value_is_empty(value, length) + assert result is False + + def test_or_validators(): """Test `or_validators` gives a valid result.""" value = "2" diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index e516a6530..96b53b4a0 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -6,6 +6,18 @@ logger = logging.getLogger(__name__) + +def value_is_empty(value, length): + """Handle 'empty' values as field inputs.""" + empty_values = [ + # '', + ' '*length, # ' ' + '#'*length, # '#####' + '_'*length, # '_____' + ] + + return value is None or value in empty_values + # higher order validator func def make_validator(validator_func, error_func): From a9e44e9725120db3f10e5a84100842043530154d Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 14 Nov 2023 11:16:01 -0500 Subject: [PATCH 2/5] add notEmpty test --- .../tdpservice/parsers/test/test_validators.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 26f8bfaaf..694c554e4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -323,6 +323,17 @@ def test_notEmpty_returns_invalid_substring(): assert is_valid is False assert error == "111 333 contains blanks between positions 3 and 5." + +def test_notEmpty_returns_nonexistent_substring(): + """Test `notEmpty` gives an invalid result for a nonexistent substring.""" + value = '111 333' + + validator = validators.notEmpty(start=10, end=12) + is_valid, error = validator(value) + + assert is_valid is False + assert error == "111 333 contains blanks between positions 10 and 12." + @pytest.mark.usefixtures('db') class TestCat3ValidatorsBase: """A base test class for tests that evaluate category three validators.""" From 017b5e336076001a61c29512104306e581f733ce Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 14 Nov 2023 11:16:33 -0500 Subject: [PATCH 3/5] add blank string to value_is_empty values --- tdrs-backend/tdpservice/parsers/validators.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 96b53b4a0..25776cdfb 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -10,7 +10,7 @@ def value_is_empty(value, length): """Handle 'empty' values as field inputs.""" empty_values = [ - # '', + '', ' '*length, # ' ' '#'*length, # '#####' '_'*length, # '_____' @@ -203,17 +203,23 @@ def isStringLargerThan(val): lambda value: f'{value} is not larger than {val}.' ) + +def _is_empty(value, start, end): + end = end if end else len(str(value)) + return value_is_empty(str(value)[start:end], end-start) + + def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( - lambda value: not str(value)[start:end if end else len(str(value))].isspace(), + lambda value: not _is_empty(value, start, end), lambda value: f'{str(value)} contains blanks between positions {start} and {end if end else len(str(value))}.' ) def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( - lambda value: value[start:end if end else len(value)].isspace(), + lambda value: _is_empty(value, start, end), lambda value: f'{value} is not blank between positions {start} and {end if end else len(value)}.' ) From cc00bce0eb03786ed71e9b5d2d46c032418b36f4 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 14 Nov 2023 11:26:07 -0500 Subject: [PATCH 4/5] replace removed text --- tdrs-backend/tdpservice/parsers/test/test_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index 3f030daea..d7b2afc0e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -51,7 +51,7 @@ def test_run_preparsing_validators_returns_invalid_and_errors(): def test_parse_line_parses_line_from_schema_to_dict(): - """Test that parse_line parses a string into a dict given start and end indices for all """ + """Test that parse_line parses a string into a dict given start and end indices for all fields.""" line = '12345001' schema = RowSchema( model=dict, @@ -74,7 +74,7 @@ def test_parse_line_parses_line_from_schema_to_dict(): def test_parse_line_parses_line_from_schema_to_object(): - """Test that parse_line parses a string into an object given start and end indices for all """ + """Test that parse_line parses a string into an object given start and end indices for all fields.""" class TestModel: first = None second = None From 21e20259a4b152ce8ff19ebb30f2797f7f81a766 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 24 Nov 2023 16:57:37 -0500 Subject: [PATCH 5/5] fix partial oob string indexing --- tdrs-backend/tdpservice/parsers/fields.py | 3 ++- tdrs-backend/tdpservice/parsers/validators.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 098c4ceff..c1b610821 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -29,8 +29,9 @@ def __repr__(self): def parse_value(self, line): """Parse the value for a field given a line, startIndex, endIndex, and field type.""" value = line[self.startIndex:self.endIndex] + value_length = self.endIndex-self.startIndex - if value_is_empty(value, self.endIndex-self.startIndex): + if len(value) < value_length or value_is_empty(value, value_length): logger.debug(f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty.") return None diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 25776cdfb..eb32c666e 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -206,7 +206,9 @@ def isStringLargerThan(val): def _is_empty(value, start, end): end = end if end else len(str(value)) - return value_is_empty(str(value)[start:end], end-start) + vlen = end - start + subv = str(value)[start:end] + return value_is_empty(subv, vlen) or len(subv) < vlen def notEmpty(start=0, end=None):