SSP Section 2 Parsing & Validation (#2724)

* - Added m4 and m5 - Added cat 2 validators * - Quick correction to preparse validator * - Updated item numbers - Added cat3 validators * - Adding datafile * - updated datafile to allow correct parsing of m4 records - added tests * - Fix lint errors * - Update permissions test * - Fixed failing unit test * - Adding order_by to keep consistent results * - Fixed obj ordering issue - Skipping super big file * - Added missing validator * - Updated import paths * - Updated DOB fields to make a little more sense * - Pre-emptively marking required false to prepare for merge * - Fix lint * - Formatting with Black * update m4/m5 validators * isInLimits -> matches (duh) * - Deconflicting migrations * - Fixing test * - Adding missing fixture * - Resolving migration name conflicts * - Fixing dependency * - Keeping consistent dependencies * - Linearizing deps --------- Co-authored-by: Jan Timpe <jtimpe@goraft.tech> Co-authored-by: jtimpe <111305129+jtimpe@users.noreply.github.com>
raft-tech · Nov 22, 2023 · bd31881 · bd31881
1 parent f1f15c1
commit bd31881
Show file tree

Hide file tree

Showing 27 changed files with 10,063 additions and 58 deletions.
diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
@@ -240,7 +240,7 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False)
         records = schema_manager.parse_and_validate(line, generate_error)
         return records
     except AttributeError as e:
-        logging.error(e)
+        logger.error(e)
         return [(None, False, [
             generate_error(
                 schema=None,

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py
@@ -1,11 +1,15 @@
 from .m1 import m1
 from .m2 import m2
 from .m3 import m3
+from .m4 import m4
+from .m5 import m5
 from .m6 import m6
 from .m7 import m7
 
 m1 = m1
 m2 = m2
 m3 = m3
+m4 = m4
+m5 = m5
 m6 = m6
 m7 = m7
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
@@ -1,10 +1,10 @@
 """Schema for SSP M1 record type."""
 
 
-from ...util import SchemaManager
-from ...fields import Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.fields import Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M1
 
 m1 = SchemaManager(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py
@@ -1,11 +1,11 @@
 """Schema for SSP M1 record type."""
 
 
-from ...util import SchemaManager
-from ...transforms import ssp_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import ssp_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M2
 
 

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py
@@ -1,11 +1,11 @@
 """Schema for SSP M1 record type."""
 
 
-from ...util import SchemaManager
-from ...transforms import ssp_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import ssp_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M3
 
 first_part_schema = RowSchema(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py
@@ -0,0 +1,146 @@
+"""Schema for SSP M1 record type."""
+
+
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.fields import Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
+from tdpservice.search_indexes.models.ssp import SSP_M4
+
+m4 = SchemaManager(
+    schemas=[
+        RowSchema(
+            model=SSP_M4,
+            preparsing_validators=[
+                validators.hasLength(66),
+            ],
+            postparsing_validators=[],
+            fields=[
+                Field(
+                    item="0",
+                    name="RecordType",
+                    type="string",
+                    startIndex=0,
+                    endIndex=2,
+                    required=True,
+                    validators=[],
+                ),
+                Field(
+                    item="3",
+                    name="RPT_MONTH_YEAR",
+                    type="number",
+                    startIndex=2,
+                    endIndex=8,
+                    required=True,
+                    validators=[
+                        validators.dateYearIsLargerThan(1998),
+                        validators.dateMonthIsValid(),
+                    ],
+                ),
+                Field(
+                    item="5",
+                    name="CASE_NUMBER",
+                    type="string",
+                    startIndex=8,
+                    endIndex=19,
+                    required=True,
+                    validators=[validators.isAlphaNumeric()],
+                ),
+                Field(
+                    item="2",
+                    name="COUNTY_FIPS_CODE",
+                    type="string",
+                    startIndex=19,
+                    endIndex=22,
+                    required=True,
+                    validators=[validators.isInStringRange(0, 999)],
+                ),
+                Field(
+                    item="4",
+                    name="STRATUM",
+                    type="string",
+                    startIndex=22,
+                    endIndex=24,
+                    required=False,
+                    validators=[validators.isInStringRange(0, 99)],
+                ),
+                Field(
+                    item="6",
+                    name="ZIP_CODE",
+                    type="string",
+                    startIndex=24,
+                    endIndex=29,
+                    required=True,
+                    validators=[validators.isInStringRange(0, 99999)],
+                ),
+                Field(
+                    item="7",
+                    name="DISPOSITION",
+                    type="number",
+                    startIndex=29,
+                    endIndex=30,
+                    required=True,
+                    validators=[validators.matches(1)],
+                ),
+                Field(
+                    item="8",
+                    name="CLOSURE_REASON",
+                    type="string",
+                    startIndex=30,
+                    endIndex=32,
+                    required=True,
+                    validators=[
+                        validators.or_validators(
+                            validators.isInStringRange(1, 19), validators.matches("99")
+                        )
+                    ],
+                ),
+                Field(
+                    item="9",
+                    name="REC_SUB_HOUSING",
+                    type="number",
+                    startIndex=32,
+                    endIndex=33,
+                    required=True,
+                    validators=[validators.isInLimits(1, 2)],
+                ),
+                Field(
+                    item="10`",
+                    name="REC_MED_ASSIST",
+                    type="number",
+                    startIndex=33,
+                    endIndex=34,
+                    required=True,
+                    validators=[validators.isInLimits(1, 2)],
+                ),
+                Field(
+                    item="11",
+                    name="REC_FOOD_STAMPS",
+                    type="number",
+                    startIndex=34,
+                    endIndex=35,
+                    required=True,
+                    validators=[validators.isInLimits(1, 2)],
+                ),
+                Field(
+                    item="12",
+                    name="REC_SUB_CC",
+                    type="number",
+                    startIndex=35,
+                    endIndex=36,
+                    required=True,
+                    validators=[validators.isInLimits(1, 2)],
+                ),
+                Field(
+                    item="-1",
+                    name="BLANK",
+                    type="string",
+                    startIndex=36,
+                    endIndex=66,
+                    required=False,
+                    validators=[],
+                ),
+            ],
+        )
+    ]
+)