diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 63f8706ac..1331490ea 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -69,7 +69,7 @@ validators.if_then_validator( condition_field="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 2), - result_field="PARENT_WITH_MINOR_CHILD", + result_field="PARENT_MINOR_CHILD", result_function=validators.isInLimits(1, 3), ), validators.if_then_validator( @@ -359,8 +359,8 @@ ), Field( item="39", - name="PARENT_WITH_MINOR_CHILD", - friendly_name="parent with minor child", + name="PARENT_MINOR_CHILD", + friendly_name="parent of minor child", type="number", startIndex=53, endIndex=54, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py index e82db47b6..7c1997236 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py @@ -4,6 +4,7 @@ from .t4 import t4 from .t5 import t5 from .t6 import t6 +from .t7 import t7 t1 = t1 t2 = t2 @@ -11,3 +12,4 @@ t4 = t4 t5 = t5 t6 = t6 +t7 = t7 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index ed1d03e8d..980e653fd 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -348,8 +348,8 @@ ), Field( item="39", - name="PARENT_WITH_MINOR_CHILD", - friendly_name="parent with minor child", + name="PARENT_MINOR_CHILD", + friendly_name="parent of minor child", type="number", startIndex=53, endIndex=54, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py new file mode 100644 index 000000000..332caed97 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -0,0 +1,109 @@ +"""Schema for Tribal TANF T7 Row.""" + +from tdpservice.parsers.util import SchemaManager +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers import validators +from tdpservice.search_indexes.models.tribal import Tribal_TANF_T7 + +schemas = [] + +validator_index = 7 +section_ind_index = 7 +stratum_index = 8 +families_index = 10 +for i in range(1, 31): + month_index = (i - 1) % 3 + sub_item_labels = ["A", "B", "C"] + families_value_item_number = f"6{sub_item_labels[month_index]}" + + schemas.append( + RowSchema( + model=Tribal_TANF_T7, + quiet_preparser_errors=i > 1, + preparsing_validators=[ + validators.hasLength(247), + validators.notEmpty(0, 7), + validators.notEmpty(validator_index, validator_index + 24), + ], + postparsing_validators=[], + fields=[ + Field( + item="0", + name="RecordType", + friendly_name="record type", + type="string", + startIndex=0, + endIndex=2, + required=True, + validators=[], + ), + Field( + item="3", + name="CALENDAR_QUARTER", + friendly_name="calendar quarter", + type="number", + startIndex=2, + endIndex=7, + required=True, + validators=[ + validators.dateYearIsLargerThan(1998), + validators.quarterIsValid(), + ], + ), + TransformField( + transform_func=calendar_quarter_to_rpt_month_year(month_index), + item="3A", + name="RPT_MONTH_YEAR", + friendly_name="reporting month year", + type="number", + startIndex=2, + endIndex=7, + required=True, + validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ], + ), + Field( + item="4", + name="TDRS_SECTION_IND", + friendly_name="tdrs section indicator", + type="string", + startIndex=section_ind_index, + endIndex=section_ind_index + 1, + required=True, + validators=[validators.oneOf(["1", "2"])], + ), + Field( + item="5", + name="STRATUM", + friendly_name="stratum", + type="string", + startIndex=stratum_index, + endIndex=stratum_index + 2, + required=True, + validators=[validators.isInStringRange(0, 99)], + ), + Field( + item=families_value_item_number, + name="FAMILIES_MONTH", + friendly_name="families month", + type="number", + startIndex=families_index, + endIndex=families_index + 7, + required=True, + validators=[validators.isInLimits(0, 9999999)], + ), + ], + ) + ) + + index_offset = 0 if i % 3 != 0 else 24 + validator_index += index_offset + section_ind_index += index_offset + stratum_index += index_offset + families_index += 7 if i % 3 != 0 else 10 + +t7 = SchemaManager(schemas=schemas) diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt new file mode 100644 index 000000000..904d0bb79 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt @@ -0,0 +1,3 @@ +HEADER20194S00142TAN1EU +T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 63a6d237d..1d5bb28af 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -170,7 +170,7 @@ class Meta: RECEIVE_SSI = 1 MARITAL_STATUS = 1 RELATIONSHIP_HOH = "01" - PARENT_WITH_MINOR_CHILD = 1 + PARENT_MINOR_CHILD = 1 NEEDS_PREGNANT_WOMAN = 1 EDUCATION_LEVEL = "01" CITIZENSHIP_STATUS = 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 71f4794f0..02cd7365b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -6,7 +6,7 @@ from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 from tdpservice.search_indexes.models.tribal import Tribal_TANF_T1, Tribal_TANF_T2, Tribal_TANF_T3, Tribal_TANF_T4 -from tdpservice.search_indexes.models.tribal import Tribal_TANF_T5, Tribal_TANF_T6 +from tdpservice.search_indexes.models.tribal import Tribal_TANF_T5, Tribal_TANF_T6, Tribal_TANF_T7 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M4, SSP_M5, SSP_M6, SSP_M7 from .factories import DataFileSummaryFactory from tdpservice.data_files.models import DataFile @@ -1096,3 +1096,29 @@ def test_parse_tribal_section_3_file(tribal_section_3_file): assert t6.NUM_APPLICATIONS == 1 assert t6.NUM_FAMILIES == 41 assert t6.NUM_CLOSED_CASES == 3 + +@pytest.fixture +def tribal_section_4_file(stt_user, stt): + """Fixture for tribal_section_4_fake.txt.""" + return util.create_test_datafile('tribal_section_4_fake.txt', stt_user, stt, "Tribal Stratum Data") + +@pytest.mark.django_db() +def test_parse_tribal_section_4_file(tribal_section_4_file): + """Test parsing Tribal TANF Section 4 submission.""" + parse.parse_datafile(tribal_section_4_file) + + assert Tribal_TANF_T7.objects.all().count() == 18 + + t7_objs = Tribal_TANF_T7.objects.all().order_by('FAMILIES_MONTH') + + first = t7_objs.first() + sixth = t7_objs[5] + + assert first.RPT_MONTH_YEAR == 202011 + assert sixth.RPT_MONTH_YEAR == 202012 + + assert first.TDRS_SECTION_IND == '2' + assert sixth.TDRS_SECTION_IND == '2' + + assert first.FAMILIES_MONTH == 274 + assert sixth.FAMILIES_MONTH == 499 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index dc65d2745..bd3eb88ce 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -587,16 +587,16 @@ def test_validate_parent_with_minor(self, record): """Test cat3 validator for parent with a minor child.""" val = validators.if_then_validator( condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), - result_field='PARENT_WITH_MINOR_CHILD', result_function=validators.isInLimits(1, 3), + result_field='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), ) result = val(record) - assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_WITH_MINOR_CHILD']) + assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) - record.PARENT_WITH_MINOR_CHILD = 0 + record.PARENT_MINOR_CHILD = 0 result = val(record) - assert result == (False, 'if FAMILY_AFFILIATION :1 validator1 passed then PARENT_WITH_MINOR_CHILD 0 is not ' + + assert result == (False, 'if FAMILY_AFFILIATION :1 validator1 passed then PARENT_MINOR_CHILD 0 is not ' + 'larger or equal to 1 and smaller or equal to 3.', - ['FAMILY_AFFILIATION', 'PARENT_WITH_MINOR_CHILD']) + ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) def test_validate_education_level(self, record): """Test cat3 validator for education level.""" diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 7cd453a8a..58df9bbdd 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -201,6 +201,12 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'T6': schema_defs.tribal_tanf.t6, } }, + 'S': { + 'section': DataFile.Section.TRIBAL_STRATUM_DATA, + 'models': { + 'T7': schema_defs.tribal_tanf.t7, + } + }, }, } diff --git a/tdrs-backend/tdpservice/search_indexes/admin/__init__.py b/tdrs-backend/tdpservice/search_indexes/admin/__init__.py index 7585fece8..91469dfa5 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/__init__.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/__init__.py @@ -16,6 +16,7 @@ admin.site.register(models.tribal.Tribal_TANF_T4, tribal.Tribal_TANF_T4Admin) admin.site.register(models.tribal.Tribal_TANF_T5, tribal.Tribal_TANF_T5Admin) admin.site.register(models.tribal.Tribal_TANF_T6, tribal.Tribal_TANF_T6Admin) +admin.site.register(models.tribal.Tribal_TANF_T7, tribal.Tribal_TANF_T7Admin) admin.site.register(models.ssp.SSP_M1, ssp.SSP_M1Admin) admin.site.register(models.ssp.SSP_M2, ssp.SSP_M2Admin) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py index e2612df44..757620a24 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py @@ -99,3 +99,22 @@ class Tribal_TANF_T6Admin(admin.ModelAdmin): CreationDateFilter, 'RPT_MONTH_YEAR' ] + +class Tribal_TANF_T7Admin(admin.ModelAdmin): + """ModelAdmin class for parsed Tribal T7 data files.""" + + list_display = [ + 'RecordType', + 'CALENDAR_QUARTER', + 'RPT_MONTH_YEAR', + 'TDRS_SECTION_IND', + 'STRATUM', + 'FAMILIES_MONTH', + 'datafile', + ] + + list_filter = [ + 'CALENDAR_QUARTER', + CreationDateFilter, + 'RPT_MONTH_YEAR', + ] diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index c613c50a2..a6b5fd6b4 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -110,7 +110,7 @@ class Django: 'RECEIVE_SSI', 'MARITAL_STATUS', 'RELATIONSHIP_HOH', - 'PARENT_WITH_MINOR_CHILD', + 'PARENT_MINOR_CHILD', 'NEEDS_PREGNANT_WOMAN', 'EDUCATION_LEVEL', 'CITIZENSHIP_STATUS', diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tribal.py b/tdrs-backend/tdpservice/search_indexes/documents/tribal.py index e81a6361f..4fb107ba8 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tribal.py @@ -3,7 +3,7 @@ from django_elasticsearch_dsl import Document from django_elasticsearch_dsl.registries import registry from ..models.tribal import Tribal_TANF_T1, Tribal_TANF_T2, Tribal_TANF_T3, Tribal_TANF_T4, Tribal_TANF_T5 -from ..models.tribal import Tribal_TANF_T6 +from ..models.tribal import Tribal_TANF_T6, Tribal_TANF_T7 from .document_base import DocumentBase @registry.register_document @@ -303,3 +303,29 @@ class Django: 'NUM_OUTWEDLOCK_BIRTHS', 'NUM_CLOSED_CASES' ] + +@registry.register_document +class Tribal_TANF_T7DataSubmissionDocument(DocumentBase, Document): + """Elastic search model mapping for a parsed Tribal TANF T7 data file.""" + + class Index: + """ElasticSearch index generation settings.""" + + name = 'tribal_tanf_t7_submissions' + settings = { + 'number_of_shards': 1, + 'number_of_replicas': 0, + } + + class Django: + """Django model reference and field mapping.""" + + model = Tribal_TANF_T7 + fields = [ + "RecordType", + "CALENDAR_QUARTER", + "RPT_MONTH_YEAR", + "TDRS_SECTION_IND", + "STRATUM", + "FAMILIES_MONTH", + ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py b/tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py new file mode 100644 index 000000000..a37b323fa --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.15 on 2023-11-29 19:49 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_files', '0012_datafile_s3_versioning_id'), + ('search_indexes', '0024_tribal_tanf_t6'), + ] + + operations = [ + migrations.CreateModel( + name='Tribal_TANF_T7', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('RecordType', models.CharField(max_length=156, null=True)), + ('CALENDAR_QUARTER', models.IntegerField(blank=True, null=True)), + ('RPT_MONTH_YEAR', models.IntegerField(null=True)), + ('TDRS_SECTION_IND', models.CharField(max_length=1, null=True)), + ('STRATUM', models.CharField(max_length=2, null=True)), + ('FAMILIES_MONTH', models.IntegerField(null=True)), + ('datafile', models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tribal_t7_parent', to='data_files.datafile')), + ], + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py b/tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py new file mode 100644 index 000000000..a5fb41692 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-09-14 17:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0025_tribal_tanf_t7'), + ] + + operations = [ + migrations.RenameField( + model_name='tanf_t2', + old_name='PARENT_WITH_MINOR_CHILD', + new_name='PARENT_MINOR_CHILD', + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index f6ba10f29..204106406 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -113,7 +113,7 @@ class TANF_T2(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) MARITAL_STATUS = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.CharField(max_length=2, null=True, blank=False) - PARENT_WITH_MINOR_CHILD = models.IntegerField(null=True, blank=False) + PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/models/tribal.py b/tdrs-backend/tdpservice/search_indexes/models/tribal.py index 6c5d46955..9cdcf6a37 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tribal.py @@ -302,3 +302,31 @@ class Tribal_TANF_T6(models.Model): NUM_BIRTHS = models.IntegerField(null=True, blank=True) NUM_OUTWEDLOCK_BIRTHS = models.IntegerField(null=True, blank=True) NUM_CLOSED_CASES = models.IntegerField(null=True, blank=True) + +class Tribal_TANF_T7(models.Model): + """ + Parsed record representing a Tribal T7 data submission. + + Mapped to an elastic search index. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + datafile = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='tribal_t7_parent' + ) + + RecordType = models.CharField(max_length=156, null=True, blank=False) + CALENDAR_QUARTER = models.IntegerField(null=True, blank=True) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + TDRS_SECTION_IND = models.CharField( + max_length=1, + null=True, + blank=False + ) + STRATUM = models.CharField(max_length=2, null=True, blank=False) + FAMILIES_MONTH = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index 8ea107a35..dd66010a9 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -112,7 +112,7 @@ def test_can_create_and_index_tanf_t2_submission(test_datafile): submission.RECEIVE_SSI = 1 submission.MARITAL_STATUS = 1 submission.RELATIONSHIP_HOH = "01" - submission.PARENT_WITH_MINOR_CHILD = 1 + submission.PARENT_MINOR_CHILD = 1 submission.NEEDS_PREGNANT_WOMAN = 1 submission.EDUCATION_LEVEL = 1 submission.CITIZENSHIP_STATUS = 1 @@ -273,11 +273,11 @@ def test_can_create_and_index_tanf_t5_submission(test_datafile): submission.REC_FEDERAL_DISABILITY = 1 submission.REC_AID_TOTALLY_DISABLED = 1 submission.REC_AID_AGED_BLIND = 1 - submission.RECEIVE_SSI = 1 + submission.REC_SSI = 1 submission.MARITAL_STATUS = 1 submission.RELATIONSHIP_HOH = "01" - submission.PARENT_WITH_MINOR_CHILD = 1 - submission.NEEDS_PREGNANT_WOMAN = 1 + submission.PARENT_MINOR_CHILD = 1 + submission.NEEDS_OF_PREGNANT_WOMAN = 1 submission.EDUCATION_LEVEL = "1" submission.CITIZENSHIP_STATUS = 1 submission.COUNTABLE_MONTH_FED_TIME = "1" @@ -1022,3 +1022,29 @@ def test_can_create_and_index_tribal_tanf_t6_submission(test_datafile): response = search.execute() assert response.hits.total.value == 1 + +@pytest.mark.django_db +def test_can_create_and_index_tribal_tanf_t7_submission(test_datafile): + """Tribal TANF T7 submissions can be created and mapped.""" + record_num = fake.uuid4() + + submission = models.tribal.Tribal_TANF_T7() + submission.datafile = test_datafile + submission.RecordType = record_num + submission.CALENDAR_YEAR = 2020 + submission.CALENDAR_QUARTER = 1 + submission.TDRS_SECTION_IND = '1' + submission.STRATUM = '01' + submission.FAMILIES_MONTH = 47655 + + submission.save() + + assert submission.id is not None + + search = documents.tribal.Tribal_TANF_T7DataSubmissionDocument.search().query( + 'match', + RecordType=record_num + ) + response = search.execute() + + assert response.hits.total.value == 1 diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 2a948afc2..b6cb1ff96 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -153,6 +153,9 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'search_indexes.add_tribal_tanf_t6', 'search_indexes.view_tribal_tanf_t6', 'search_indexes.change_tribal_tanf_t6', + 'search_indexes.add_tribal_tanf_t7', + 'search_indexes.view_tribal_tanf_t7', + 'search_indexes.change_tribal_tanf_t7', } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions