Tribal Section 1 Parsing & Validation (#2742)

* Added formating for header and autofit columns

* Formatted the headers

* added year/month to the columns

* Added contants - translation column

* added friendly names to T1 and T2

* added friendly name to m1 and m2

* added friendly name to m3

* added friendly_name to t3

* added friendly_name to t4 and t5

* added friendly_name to t7

* correct missing friendly_name

* correction on failing tests

* addedfriendly name to excel report

* linting

* linting

* linting

* delete contants.py

* added test for json field in error model

* linting

* linting

* linting

* - Added Tribal models, docs, admin refs
- Added support for tribal parsing

* - Update t1 field validators

* - t2/t3 field validators

* 2599-added friendly name to postparsing validators

* - Updated cat3 validators

* - Updated how we detect tribe code

* - Added error handling for inconsistent header info wrt tribe code
- Added new test

* - Updated test

* - Grammar fix
- Removed unused function

* refining the validator tests

* - Format with Black
- Add missing tests

* - Fix lint

* added returning fields names to validators

* added friendly_name to error field

* linting

* corrections on views/tests

* corrections for fields

* failing test corrected

* failing test corrected

* correcting test failures

* linting

* corrected the excel fiel generator

* removed excessive space in validator

* linting

* - Updated based on review feedback

* - removing debug assert

* - Fix lint

* listing

* - deconflicting migrations

* - Fixing test

* added m6

* lint

* corrected new line break

* refactored validator logic

* - resolving conflict

* - Fix lint errors

* - Resolve conflict

* linting and correction on t1

* friendly_name correction from comments

* friendly_name correction

* corrected failing test for m5

* refactor the field_json creation DRY

* - Renamed tribal files
- Updated to allow case aggregates
- Updated datafile to have a more convenient date range
- Updated tests to capture case aggregates

* - Reduced file size
- Updated test

* - Fix lint errors

* friendly_name corrections

* - Adding friendly names to tribal

* - Fixing typo

* linting and cleaning errors

* linting

* correction on friendly_names

* corrected friendly_name for test_util

* correction child care - number of months

* fixed a few more typos and some spacing. (#2767)

* fixed a few more typos and some spacing.

* fixed linting issues

* missed a spot.

---------

Co-authored-by: George Hudson <ghudson@teamraft.com>

* - knowledge center

* - Integrating Jans changes. Parametrizing values_is_empty

* - Update based on review feedback

* - Updated length checks

* - removing fips code

* Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py

Co-authored-by: Alex P.  <63075587+ADPennington@users.noreply.github.com>

* Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py

Co-authored-by: Alex P.  <63075587+ADPennington@users.noreply.github.com>

* Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py

Co-authored-by: Alex P.  <63075587+ADPennington@users.noreply.github.com>

* Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py

Co-authored-by: Alex P.  <63075587+ADPennington@users.noreply.github.com>

* - update based on review

* - Fix migration

* - Fixed name clash on PARENT_MINOR_CHILD

* Revert "- Fixed name clash on PARENT_MINOR_CHILD"

This reverts commit 5850dc8.

---------

Co-authored-by: Mo Sohani <msohani@goraft.tech>
Co-authored-by: raftmsohani <97037188+raftmsohani@users.noreply.github.com>
Co-authored-by: George Hudson <georgehudson78@gmail.com>
Co-authored-by: George Hudson <ghudson@teamraft.com>
Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com>

tdrs-backend/tdpservice/parsers/parse.py

@@ -28,11 +28,31 @@ def parse_datafile(datafile):
         bulk_create_errors({1: header_errors}, 1, flush=True)
         return errors
 
-    is_encrypted = util.contains_encrypted_indicator(header_line, schema_defs.header.get_field_by_name("encryption"))
+    field_values = schema_defs.header.get_field_values_by_names(header_line,
+                                                                {"encryption", "tribe_code", "state_fips"})
+
+    # Validate tribe code in submission across program type and fips code
+    generate_error = util.make_generate_parser_error(datafile, 1)
+    tribe_is_valid, tribe_error = validators.validate_tribe_fips_program_agree(header['program_type'],
+                                                                               field_values["tribe_code"],
+                                                                               field_values["state_fips"],
+                                                                               generate_error)
+
+    if not tribe_is_valid:
+        logger.info(f"Tribe Code ({field_values['tribe_code']}) inconsistency with Program Type " +
+                    f"({header['program_type']}) and FIPS Code ({field_values['state_fips']}).",)
+        errors['header'] = [tribe_error]
+        bulk_create_errors({1: [tribe_error]}, 1, flush=True)
+        return errors
+
+    is_encrypted = field_values["encryption"] == "E"
+    is_tribal = not validators.value_is_empty(field_values["tribe_code"], 3, extra_vals={'0'*3})
+
     logger.debug(f"Datafile has encrypted fields: {is_encrypted}.")
+    logger.debug(f"Datafile: {datafile.__repr__()}, is Tribal: {is_tribal}.")
 
     # ensure file section matches upload section
-    program_type = header['program_type']
+    program_type = f"Tribal {header['program_type']}" if is_tribal else header['program_type']
     section = header['type']
     logger.debug(f"Program type: {program_type}, Section: {section}.")
 

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -168,6 +168,14 @@ def run_postparsing_validators(self, instance, generate_error):
 
         return is_valid, errors
 
+    def get_field_values_by_names(self, line, names={}):
+        """Return dictionary of field values keyed on their name."""
+        field_values = {}
+        for field in self.fields:
+            if field.name in names:
+                field_values[field.name] = field.parse_value(line)
+        return field_values
+
     def get_field_by_name(self, name):
         """Get field by it's name."""
         for field in self.fields:

tdrs-backend/tdpservice/parsers/schema_defs/__init__.py

@@ -1,9 +1,11 @@
 from .header import header
 from .trailer import trailer
 from . import tanf
+from . import tribal_tanf
 from . import ssp
 
 header = header
 trailer = trailer
 tanf = tanf
+tribal_tanf = tribal_tanf
 ssp = ssp

tdrs-backend/tdpservice/parsers/schema_defs/header.py

@@ -66,13 +66,17 @@
             type="string",
             startIndex=12,
             endIndex=14,
-            required=True,
+            required=False,
             validators=[
-                validators.oneOf(("01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15",
-                                  "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27",
-                                  "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39",
-                                  "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53",
-                                  "54", "55", "56", "66", "72", "78"))
+                validators.or_validators(
+                    validators.isInStringRange(0, 2),
+                    validators.isInStringRange(4, 6),
+                    validators.isInStringRange(8, 13),
+                    validators.isInStringRange(15, 42),
+                    validators.isInStringRange(44, 51),
+                    validators.isInStringRange(53, 56),
+                    validators.oneOf(["66", "72", "78"]),
+                )
             ],
         ),
         Field(

tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py

@@ -0,0 +1,7 @@
+from .t1 import t1
+from .t2 import t2
+from .t3 import t3
+
+t1 = t1
+t2 = t2
+t3 = t3