2711-catch-rpt-month-year-mismatches

tdrs-backend/Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get -y update
 RUN apt-get -y upgrade
 # Install a new package:
 RUN apt-get install -y gcc && apt-get install -y graphviz && apt-get install -y graphviz-dev
+RUN apt-get install postgresql-client -y
 # Install pipenv
 RUN pip install --upgrade pip pipenv
 RUN pipenv install --dev --system --deploy

tdrs-backend/tdpservice/parsers/fields.py

@@ -1,10 +1,19 @@
 """Datafile field representations."""
 
 import logging
-from .validators import value_is_empty
 
 logger = logging.getLogger(__name__)
 
+def value_is_empty(value, length):
+    """Handle 'empty' values as field inputs."""
+    empty_values = [
+        '',
+        ' '*length,  # '     '
+        '#'*length,  # '#####'
+        '_'*length,  # '_____'
+    ]
+
+    return value is None or value in empty_values
 
 class Field:
     """Provides a mapping between a field name and its position."""

tdrs-backend/tdpservice/parsers/parse.py

@@ -49,6 +49,18 @@ def parse_datafile(datafile):
         bulk_create_errors(unsaved_parser_errors, 1, flush=True)
         return errors
 
+    rpt_month_year_is_valid, rpt_month_year_error = validators.validate_header_rpt_month_year(
+        datafile,
+        header,
+        util.make_generate_parser_error(datafile, 1)
+    )
+    if not rpt_month_year_is_valid:
+        logger.info(f"Preparser Error -> Rpt Month Year is not valid: {rpt_month_year_error.error_message}")
+        errors['document'] = [rpt_month_year_error]
+        unsaved_parser_errors = {1: [rpt_month_year_error]}
+        bulk_create_errors(unsaved_parser_errors, 1, flush=True)
+        return errors
+
     line_errors = parse_datafile_lines(datafile, program_type, section, is_encrypted)
 
     errors = errors | line_errors

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -1,11 +1,20 @@
 """Row schema for datafile."""
 from .models import ParserErrorCategoryChoices
-from .fields import Field
-from .validators import value_is_empty
+from .fields import Field, TransformField
 import logging
 
 logger = logging.getLogger(__name__)
 
+def value_is_empty(value, length):
+    """Handle 'empty' values as field inputs."""
+    empty_values = [
+        '',
+        ' '*length,  # '     '
+        '#'*length,  # '#####'
+        '_'*length,  # '_____'
+    ]
+
+    return value is None or value in empty_values
 
 class RowSchema:
     """Maps the schema for data lines."""
@@ -174,3 +183,27 @@ def get_field_by_name(self, name):
             if field.name == name:
                 return field
         return None
+
+
+class SchemaManager:
+    """Manages one or more RowSchema's and runs all parsers and validators."""
+
+    def __init__(self, schemas):
+        self.schemas = schemas
+
+    def parse_and_validate(self, line, generate_error):
+        """Run `parse_and_validate` for each schema provided and bubble up errors."""
+        records = []
+
+        for schema in self.schemas:
+            record, is_valid, errors = schema.parse_and_validate(line, generate_error)
+            records.append((record, is_valid, errors))
+
+        return records
+
+    def update_encrypted_fields(self, is_encrypted):
+        """Update whether schema fields are encrypted or not."""
+        for schema in self.schemas:
+            for field in schema.fields:
+                if type(field) == TransformField and "is_encrypted" in field.kwargs:
+                    field.kwargs['is_encrypted'] = is_encrypted

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py

@@ -1,9 +1,8 @@
 """Schema for SSP M1 record type."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.fields import Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M1
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py

@@ -1,10 +1,9 @@
 """Schema for SSP M1 record type."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import ssp_ssn_decryption_func
 from tdpservice.parsers.fields import TransformField, Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M2
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py

@@ -1,10 +1,9 @@
 """Schema for SSP M1 record type."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import ssp_ssn_decryption_func
 from tdpservice.parsers.fields import TransformField, Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M3
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py

@@ -1,9 +1,8 @@
 """Schema for SSP M1 record type."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.fields import Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M4
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py

@@ -1,10 +1,9 @@
 """Schema for SSP M1 record type."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import ssp_ssn_decryption_func
 from tdpservice.parsers.fields import TransformField, Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M5
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py

@@ -1,10 +1,9 @@
 """Schema for HEADER row of all submission types."""
 
 
-from ...util import SchemaManager
 from ...transforms import calendar_quarter_to_rpt_month_year
 from ...fields import Field, TransformField
-from ...row_schema import RowSchema
+from ...row_schema import RowSchema, SchemaManager
 from ... import validators
 from tdpservice.search_indexes.models.ssp import SSP_M6
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py

@@ -1,8 +1,7 @@
 """Schema for TANF T7 Row."""
 
-from ...util import SchemaManager
 from ...fields import Field, TransformField
-from ...row_schema import RowSchema
+from ...row_schema import RowSchema, SchemaManager
 from ...transforms import calendar_quarter_to_rpt_month_year
 from ... import validators
 from tdpservice.search_indexes.models.ssp import SSP_M7

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py

@@ -1,8 +1,7 @@
 """Schema for t1 record types."""
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.fields import Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T1
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py

@@ -1,10 +1,9 @@
 """Schema for HEADER row of all submission types."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import tanf_ssn_decryption_func
 from tdpservice.parsers.fields import TransformField, Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T2
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py

@@ -1,10 +1,9 @@
 """Schema for HEADER row of all submission types."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import tanf_ssn_decryption_func
 from tdpservice.parsers.fields import TransformField, Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T3
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py

@@ -1,9 +1,8 @@
 """Schema for HEADER row of all submission types."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.fields import Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T4
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py

@@ -1,10 +1,9 @@
 """Schema for HEADER row of all submission types."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import tanf_ssn_decryption_func
 from tdpservice.parsers.fields import TransformField, Field
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T5
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py

@@ -1,10 +1,9 @@
 """Schema for HEADER row of all submission types."""
 
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
 from tdpservice.parsers.fields import Field, TransformField
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T6
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py

@@ -1,8 +1,7 @@
 """Schema for TANF T7 Row."""
 
-from tdpservice.parsers.util import SchemaManager
 from tdpservice.parsers.fields import Field, TransformField
-from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T7

tdrs-backend/tdpservice/parsers/test/data/small_incorrect_file_cross_validator.txt

@@ -1,3 +1,3 @@
-HEADER20204A06   TAN1 N
-T12020101111111111223003403361110213120000300000000000008730000000000000000000000000000000000000222222000000002229012                                       
+HEADER20211A06   TAN1 N
+T12021101111111111223003403361110213120000300000000000008730000000000000000000000000000000000000222222000000002229012                                       
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/tanf_section1_header_test.txt

@@ -0,0 +1,3 @@
+HEADER20204A06   TAN1 N
+T12020101111111111223003403361110213120000300000000000008730010000000000000000000000000000000000222222000000002229012                                       
+TRAILER0000001