Merge branch '1116-tribal-s3' of https://github.com/raft-tech/TANF-app …

…into 1117-tribal-s4
raft-tech · Dec 4, 2023 · 700f996 · 700f996
2 parents 50192aa + 0799f62
commit 700f996
Show file tree

Hide file tree

Showing 39 changed files with 1,846 additions and 794 deletions.
diff --git a/...3, Winter - TDP 3.0 Pilot Program copy.md → ...s/2023, Winter - TDP 3.0 Pilot Program.md b/...3, Winter - TDP 3.0 Pilot Program copy.md → ...s/2023, Winter - TDP 3.0 Pilot Program.md
diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh
@@ -100,7 +100,6 @@ update_backend()
 
     if [ "$1" = "rolling" ] ; then
         set_cf_envs
-
         # Do a zero downtime deploy.  This requires enough memory for
         # two apps to exist in the org/space at one time.
         cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1

diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh
diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
@@ -40,7 +40,7 @@ cd "$TARGET_DIR" || exit 2
 
 
 if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then 
-    docker network create external-net
+  docker network create external-net
 fi
 
 # Ensure the APP_URL is reachable from the zaproxy container
@@ -112,10 +112,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \
   -config globalexcludeurl.url_list.url\(14\).enabled=true \
 
-  -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \
-  -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \
-  -config globalexcludeurl.url_list.url\(15\).enabled=true \
-
   -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \
   -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \
   -config globalexcludeurl.url_list.url\(16\).enabled=true \
@@ -140,7 +136,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \
   -config globalexcludeurl.url_list.url\(21\).enabled=true \
   -config spider.postform=true"
-
 # How long ZAP will crawl the app with the spider process
 ZAP_SPIDER_MINS=10
 

diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile
@@ -58,7 +58,7 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
 sendgrid = "==6.10.0"
 
 [requires]

diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock
diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
@@ -4,14 +4,18 @@ events { worker_connections 1024;
 # This opens a route to clamav prod
 http{ 
     server {
+        client_max_body_size 100m;
         listen {{port}};
+        client_max_body_size 100m;
         location /scan {
             proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass_request_headers      on;
         }
     }
     server {
+        client_max_body_size 100m;
         listen 9000;
+        client_max_body_size 100m;
         location /scan {
             proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass_request_headers      on;

diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
@@ -4,12 +4,13 @@
 
 logger = logging.getLogger(__name__)
 
+
 def value_is_empty(value, length):
     """Handle 'empty' values as field inputs."""
     empty_values = [
-        ' '*length,  # '     '
-        '#'*length,  # '#####'
-        '_'*length,  # '_____'
+        " " * length,  # '     '
+        "#" * length,  # '#####'
+        "_" * length,  # '_____'
     ]
 
     return value is None or value in empty_values
@@ -18,9 +19,20 @@ def value_is_empty(value, length):
 class Field:
     """Provides a mapping between a field name and its position."""
 
-    def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]):
+    def __init__(
+        self,
+        item,
+        name,
+        friendly_name,
+        type,
+        startIndex,
+        endIndex,
+        required=True,
+        validators=[],
+    ):
         self.item = item
         self.name = name
+        self.friendly_name = friendly_name
         self.type = type
         self.startIndex = startIndex
         self.endIndex = endIndex
@@ -37,32 +49,43 @@ def __repr__(self):
 
     def parse_value(self, line):
         """Parse the value for a field given a line, startIndex, endIndex, and field type."""
-        value = line[self.startIndex:self.endIndex]
+        value = line[self.startIndex: self.endIndex]
 
-        if value_is_empty(value, self.endIndex-self.startIndex):
-            logger.debug(f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty.")
+        if value_is_empty(value, self.endIndex - self.startIndex):
+            logger.debug(
+                f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty."
+            )
             return None
 
         match self.type:
-            case 'number':
+            case "number":
                 try:
                     value = int(value)
                     return value
                 except ValueError:
                     logger.error(f"Error parsing field value: {value} to integer.")
                     return None
-            case 'string':
+            case "string":
                 return value
             case _:
                 logger.warn(f"Unknown field type: {self.type}.")
                 return None
 
+
 class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
-    def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True,
+    def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
                  validators=[], **kwargs):
-        super().__init__(item, name, type, startIndex, endIndex, required, validators)
+        super().__init__(
+            item=item,
+            name=name,
+            type=type,
+            friendly_name=friendly_name,
+            startIndex=startIndex,
+            endIndex=endIndex,
+            required=required,
+            validators=validators)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py
@@ -150,16 +150,18 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error = validator(instance)
+            validator_is_valid, validator_error, field_names = validator(instance)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
+                # get field from field name
+                fields = [self.get_field_by_name(name) for name in field_names]
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.VALUE_CONSISTENCY,
                         error_message=validator_error,
                         record=instance,
-                        field=None
+                        field=fields,
                     )
                 )
 

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py
@@ -20,6 +20,7 @@
         Field(
             item="2",
             name="title",
+            friendly_name="title",
             type="string",
             startIndex=0,
             endIndex=6,
@@ -31,6 +32,7 @@
         Field(
             item="4",
             name="year",
+            friendly_name="year",
             type="number",
             startIndex=6,
             endIndex=10,
@@ -40,6 +42,7 @@
         Field(
             item="5",
             name="quarter",
+            friendly_name="quarter",
             type="string",
             startIndex=10,
             endIndex=11,
@@ -49,6 +52,7 @@
         Field(
             item="6",
             name="type",
+            friendly_name="type",
             type="string",
             startIndex=11,
             endIndex=12,
@@ -58,6 +62,7 @@
         Field(
             item="1",
             name="state_fips",
+            friendly_name="state fips",
             type="string",
             startIndex=12,
             endIndex=14,
@@ -77,6 +82,7 @@
         Field(
             item="3",
             name="tribe_code",
+            friendly_name="tribe code",
             type="string",
             startIndex=14,
             endIndex=17,
@@ -86,6 +92,7 @@
         Field(
             item="7",
             name="program_type",
+            friendly_name="program type",
             type="string",
             startIndex=17,
             endIndex=20,
@@ -95,6 +102,7 @@
         Field(
             item="8",
             name="edit",
+            friendly_name="edit",
             type="string",
             startIndex=20,
             endIndex=21,
@@ -104,6 +112,7 @@
         Field(
             item="9",
             name="encryption",
+            friendly_name="encryption",
             type="string",
             startIndex=21,
             endIndex=22,
@@ -113,6 +122,7 @@
         Field(
             item="10",
             name="update",
+            friendly_name="update",
             type="string",
             startIndex=22,
             endIndex=23,