Skip to content

Commit

Permalink
correction on failing tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@raftmsohani
raftmsohani committed Oct 27, 2023
1 parent 9dfd6f8 commit 90aea2f
Show file tree
Hide file tree
Showing 7 changed files with 56 additions and 54 deletions.
7 changes: 4 additions & 3 deletions tdrs-backend/tdpservice/parsers/fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ def value_is_empty(value, length):
class Field:
"""Provides a mapping between a field name and its position."""

def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]):
def __init__(self, item, name, friendly_name, type, startIndex, endIndex, required=True, validators=[]):
self.item = item
self.name = name
self.friendly_name = friendly_name
self.type = type
self.startIndex = startIndex
self.endIndex = endIndex
Expand Down Expand Up @@ -60,8 +61,8 @@ def parse_value(self, line):
class TransformField(Field):
"""Represents a field that requires some transformation before serializing."""

def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True, validators=[], **kwargs):
super().__init__(item, name, type, startIndex, endIndex, required, validators)
def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True, validators=[], **kwargs):
super().__init__(item, name, friendly_name, type, startIndex, endIndex, required, validators)
self.transform_func = transform_func
self.kwargs = kwargs

Expand Down
20 changes: 10 additions & 10 deletions tdrs-backend/tdpservice/parsers/schema_defs/header.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,37 +17,37 @@
],
postparsing_validators=[],
fields=[
Field(item="2", name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[
Field(item="2", name='title', friendly_name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[
validators.matches('HEADER'),
]),
Field(item="4", name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[
Field(item="4", name='year', friendly_name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[
validators.isInLimits(2000, 2099)
]),
Field(item="5", name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[
Field(item="5", name='quarter', friendly_name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[
validators.oneOf(['1', '2', '3', '4'])
]),
Field(item="6", name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[
Field(item="6", name='type', friendly_name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[
validators.oneOf(['A', 'C', 'G', 'S'])
]),
Field(item="1", name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[
Field(item="1", name='state_fips', friendly_name='state fips', type='string', startIndex=12, endIndex=14, required=True, validators=[
validators.oneOf(("01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15", "16", "17", "18",
"19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33",
"34", "35", "36", "37", "38", "39", "40", "41", "42", "44", "45", "46", "47", "48", "49",
"50", "51", "53", "54", "55", "56", "66", "72", "78"))
]),
Field(item="3", name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[
Field(item="3", name='tribe_code', friendly_name='tribe code', type='string', startIndex=14, endIndex=17, required=False, validators=[
validators.isInStringRange(0, 999)
]),
Field(item="7", name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[
Field(item="7", name='program_type', friendly_name='program type', type='string', startIndex=17, endIndex=20, required=True, validators=[
validators.oneOf(['TAN', 'SSP'])
]),
Field(item="8", name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[
Field(item="8", name='edit', friendly_name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[
validators.oneOf(['1', '2'])
]),
Field(item="9", name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[
Field(item="9", name='encryption', friendly_name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[
validators.oneOf([' ', 'E'])
]),
Field(item="10", name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[
Field(item="10", name='update', friendly_name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[
validators.oneOf(['N', 'D', 'U'])
]),
],
Expand Down
4 changes: 2 additions & 2 deletions tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
required=True, validators=[]),
Field(item="39", name='COOPERATION_CHILD_SUPPORT', friendly_name='cooperation child support', type='number', startIndex=58, endIndex=59,
required=True, validators=[]),
Field(item="40", name='EMPLOYMENT_STATUS', frinedly_name='employment status', type='number', startIndex=59, endIndex=60,
Field(item="40", name='EMPLOYMENT_STATUS', friendly_name='employment status', type='number', startIndex=59, endIndex=60,
required=True, validators=[]),
Field(item="41", name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator', type='number', startIndex=60, endIndex=62,
required=True, validators=[]),
Expand Down Expand Up @@ -126,7 +126,7 @@
required=True, validators=[]),
Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', friendly_name='school attendance - holiday', type='number', startIndex=112, endIndex=114,
required=True, validators=[]),
Field(item="54A", name='PROVIDE_CC_HOP', friend_name='provide child care - hours of participation', type='number', startIndex=114, endIndex=116,
Field(item="54A", name='PROVIDE_CC_HOP', friendly_name='provide child care - hours of participation', type='number', startIndex=114, endIndex=116,
required=True, validators=[]),
Field(item="54B", name='PROVIDE_CC_EA', friendly_name='provide child care - excused absence', type='number', startIndex=116, endIndex=118,
required=True, validators=[]),
Expand Down
7 changes: 4 additions & 3 deletions tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,12 @@
validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]),
],
fields=[
Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2,
Field(item="0", name='RecordType', friendly_name='record type', type='string', startIndex=0, endIndex=2,
required=True, validators=[]),
Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7,
Field(item="3", name='CALENDAR_QUARTER', friendly_name='calendar quarter', type='number', startIndex=2, endIndex=7,
required=True, validators=[]),
TransformField(calendar_quarter_to_rpt_month_year(1), item="4", name='RPT_MONTH_YEAR', type='number',
TransformField(calendar_quarter_to_rpt_month_year(1), item="4", name='RPT_MONTH_YEAR',
friendly_name='report month year', type='number',
startIndex=2, endIndex=7, required=True, validators=[]),
Field(item="4B", name='NUM_APPLICATIONS', friendly_name='total number of applications', type='number', startIndex=15, endIndex=23,
required=True, validators=[validators.isInLimits(0, 99999999)]),
Expand Down
6 changes: 3 additions & 3 deletions tdrs-backend/tdpservice/parsers/schema_defs/trailer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@
],
postparsing_validators=[],
fields=[
Field(item="1", name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[
Field(item="1", name='title', friendly_name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[
validators.matches('TRAILER')
]),
Field(item="2", name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[
Field(item="2", name='record_count', friendly_name='record count', type='number', startIndex=7, endIndex=14, required=True, validators=[
validators.between(0, 9999999)
]),
Field(item="-1", name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[
Field(item="-1", name='blank', friendly_name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[
validators.matches(' ')
]),
],
Expand Down
60 changes: 30 additions & 30 deletions tdrs-backend/tdpservice/parsers/test/test_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,16 @@ def test_parse_line_parses_line_from_schema_to_dict():
schema = RowSchema(
model=dict,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3),
Field(item=2, name='second', type='string', startIndex=3, endIndex=4),
Field(item=3, name='third', type='string', startIndex=4, endIndex=5),
Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7),
Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8),
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3),
Field(item=2, name='second', friendly_name='second', type='string', startIndex=3, endIndex=4),
Field(item=3, name='third', friendly_name='third', type='string', startIndex=4, endIndex=5),
Field(item=4, name='fourth', friendly_name='fourth', type='number', startIndex=5, endIndex=7),
Field(item=5, name='fifth', friendly_name='fifth', type='number', startIndex=7, endIndex=8),
]
)

record = schema.parse_line(line)

assert record['first'] == '123'
assert record['second'] == '4'
assert record['third'] == '5'
Expand All @@ -86,11 +86,11 @@ class TestModel:
schema = RowSchema(
model=TestModel,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3),
Field(item=2, name='second', type='string', startIndex=3, endIndex=4),
Field(item=3, name='third', type='string', startIndex=4, endIndex=5),
Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7),
Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8),
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3),
Field(item=2, name='second', friendly_name='second', type='string', startIndex=3, endIndex=4),
Field(item=3, name='third', friendly_name='third', type='string', startIndex=4, endIndex=5),
Field(item=4, name='fourth', friendly_name='fourth', type='number', startIndex=5, endIndex=7),
Field(item=5, name='fifth', friendly_name='fifth', type='number', startIndex=7, endIndex=8),
]
)

Expand All @@ -113,13 +113,13 @@ def test_run_field_validators_returns_valid_with_dict():
schema = RowSchema(
model=dict,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3, validators=[
passing_validator()
]),
Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[
Field(item=2, name='second', friendly_name='second', type='string', startIndex=3, endIndex=4, validators=[
passing_validator()
]),
Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[
Field(item=3, name='third', friendly_name='third', type='string', startIndex=4, endIndex=5, validators=[
passing_validator()
]),
]
Expand All @@ -145,13 +145,13 @@ class TestModel:
schema = RowSchema(
model=TestModel,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3, validators=[
passing_validator()
]),
Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[
Field(item=2, name='second', friendly_name='second', type='string', startIndex=3, endIndex=4, validators=[
passing_validator()
]),
Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[
Field(item=3, name='third', friendly_name='third', type='string', startIndex=4, endIndex=5, validators=[
passing_validator()
]),
]
Expand All @@ -172,14 +172,14 @@ def test_run_field_validators_returns_invalid_with_dict():
schema = RowSchema(
model=dict,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3, validators=[
passing_validator(),
failing_validator()
]),
Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[
Field(item=2, name='second', friendly_name='second', type='string', startIndex=3, endIndex=4, validators=[
passing_validator()
]),
Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[
Field(item=3, name='third', friendly_name='third', type='string', startIndex=4, endIndex=5, validators=[
passing_validator()
]),
]
Expand All @@ -205,14 +205,14 @@ class TestModel:
schema = RowSchema(
model=TestModel,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3, validators=[
passing_validator(),
failing_validator()
]),
Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[
Field(item=2, name='second', friendly_name='second', type='string', startIndex=3, endIndex=4, validators=[
passing_validator()
]),
Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[
Field(item=3, name='third', friendly_name='third', type='string', startIndex=4, endIndex=5, validators=[
passing_validator()
]),
]
Expand All @@ -237,10 +237,10 @@ def test_field_validators_blank_and_required_returns_error(first, second):
schema = RowSchema(
model=dict,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[
passing_validator(),
]),
Field(item=2, name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[
Field(item=2, name='second', friendly_name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[
passing_validator(),
]),
]
Expand All @@ -267,7 +267,7 @@ def test_field_validators_blank_and_not_required_returns_valid(first):
schema = RowSchema(
model=dict,
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[
passing_validator(),
failing_validator()
]),
Expand Down Expand Up @@ -349,7 +349,7 @@ def test_multi_record_schema_parses_and_validates():
failing_validator()
],
fields=[
Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[
Field(item=1, name='first', friendly_name='first', type='string', startIndex=0, endIndex=3, validators=[
passing_validator()
]),
]
Expand All @@ -363,7 +363,7 @@ def test_multi_record_schema_parses_and_validates():
passing_validator()
],
fields=[
Field(item=2, name='second', type='string', startIndex=2, endIndex=4, validators=[
Field(item=2, name='second', friendly_name='first', type='string', startIndex=2, endIndex=4, validators=[
passing_validator()
]),
]
Expand All @@ -377,7 +377,7 @@ def test_multi_record_schema_parses_and_validates():
passing_validator()
],
fields=[
Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[
Field(item=3, name='third', friendly_name='first', type='string', startIndex=4, endIndex=5, validators=[
passing_validator()
]),
]
Expand All @@ -391,7 +391,7 @@ def test_multi_record_schema_parses_and_validates():
passing_validator()
],
fields=[
Field(item=4, name='fourth', type='string', startIndex=4, endIndex=5, validators=[
Field(item=4, name='fourth', friendly_name='first', type='string', startIndex=4, endIndex=5, validators=[
failing_validator()
]),
]
Expand Down
6 changes: 3 additions & 3 deletions tdrs-backend/tdpservice/parsers/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,21 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m
model=schema.model if schema else None
) if record and not isinstance(record, dict) else None,
object_id=getattr(record, 'id', None) if record and not isinstance(record, dict) else None,
fields_json=None
fields_json={"friendly_name": getattr(field, 'friendly_name', None) if hasattr(field, 'friendly_name') else None}
)


def make_generate_parser_error(datafile, line_number):
"""Configure generate_parser_error with a datafile and line number."""
def generate(schema, error_category, error_message, record=None, field=None):
def generate(schema, error_category, error_message, record=None, field=None, field_json=None):
return generate_parser_error(
datafile=datafile,
line_number=line_number,
schema=schema,
error_category=error_category,
error_message=error_message,
record=record,
field=field
field=field,
)

return generate
Expand Down

0 comments on commit 90aea2f

Please sign in to comment.