Merge branch 'develop' into 2711-catch-rpt-month-year-mismatches

.circleci/deployment/jobs.yml

@@ -1,9 +1,9 @@
 # jobs:
   deploy-dev:
-    executor: docker-executor
     parameters:
       target_env:
         type: string
+    executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
       - deploy-cloud-dot-gov:
@@ -33,26 +33,16 @@
           cf-username: CF_USERNAME_STAGING
 
   deploy-infrastructure-dev:
-    executor: terraform/default
-    working_directory: ~/tdp-deploy
     parameters:
       target_env:
         type: string
-    steps:
-      - deploy-infrastructure:
-          cf-app: << parameters.target_env >>
-
-  deploy-infrastructure-staging:
     executor: terraform/default
     working_directory: ~/tdp-deploy
     steps:
       - deploy-infrastructure:
-          cf-password: CF_PASSWORD_STAGING
-          cf-username: CF_USERNAME_STAGING
-          cf-space: tanf-staging
-          tf-path: ./terraform/staging
+                cf-app: << parameters.target_env >>
 
-  deploy-infrastructure-develop:
+  deploy-infrastructure-staging:
     executor: terraform/default
     working_directory: ~/tdp-deploy
     steps:

.circleci/deployment/workflows.yml

@@ -1,43 +1,51 @@
 #workflows:
-  dev-deployment:
+  deployment:
     when:
-      << pipeline.parameters.run_dev_deployment >>
+      and:
+        - or:
+            - equal: [ master, << pipeline.git.branch >> ]
+            - equal: [ main, << pipeline.git.branch >> ]
+            - equal: [ develop, << pipeline.git.branch >> ]
+            - << pipeline.parameters.run_dev_deployment >>
+        - not: << pipeline.parameters.run_nightly_owasp_scan >>
+
     jobs:
+      - deploy-project-updates-site:
+          filters:
+            branches:
+              only:
+                - develop
       - deploy-infrastructure-dev:
           target_env: << pipeline.parameters.target_env >>
-      - enable-versioning:
-          requires:
-            - deploy-infrastructure-dev
           filters:
             branches:
               ignore:
                 - develop
                 - main
                 - master
-      - deploy-dev:
-          target_env: << pipeline.parameters.target_env >>
-          requires:
-            - deploy-infrastructure-dev
-
-  staging-deployment:
-    unless:
-      or:
-        - << pipeline.parameters.run_dev_deployment >>
-        - << pipeline.parameters.run_nightly_owasp_scan >>
-    jobs:
-      - deploy-project-updates-site:
+      - deploy-infrastructure-staging:
           filters:
             branches:
               only:
                 - develop
-      - deploy-infrastructure-develop:
+                - main
+      - deploy-infrastructure-production:
           filters:
             branches:
               only:
+                - master
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-dev
+          filters:
+            branches:
+              ignore:
                 - develop
+                - main
+                - master
       - enable-versioning:
           requires:
-            - deploy-infrastructure-develop
+            - deploy-infrastructure-staging
           target_env: develop
           cf-password: CF_PASSWORD_STAGING
           cf-username: CF_USERNAME_STAGING
@@ -46,88 +54,76 @@
             branches:
               only:
                 - develop
-      - deploy-develop:
+      - enable-versioning:
           requires:
-            - deploy-infrastructure-develop
+            - deploy-infrastructure-staging
+          target_env: staging
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
           filters:
             branches:
               only:
-                - develop
-      - test-deployment-e2e:
+                - main
+      - enable-versioning:
           requires:
-            - deploy-develop
+            - deploy-infrastructure-production
+          target_env: prod
+          cf-password: CF_PASSWORD_PROD
+          cf-username: CF_USERNAME_PROD
+          cf-space: tanf-prod
           filters:
             branches:
               only:
-                - develop
-      - make_erd: # from ../util folder
+                - master
+      - prod-deploy-clamav:
+          requires:
+            - deploy-infrastructure-production
           filters:
             branches:
               only:
-                - develop
-      - deploy-infrastructure-staging:
+                - master
+      - deploy-dev:
+          target_env: << pipeline.parameters.target_env >>
+          requires:
+            - deploy-infrastructure-dev
           filters:
             branches:
-              only:
+              ignore:
+                - develop
                 - main
-      - enable-versioning:
+                - master
+      - deploy-develop:
           requires:
             - deploy-infrastructure-staging
-          target_env: staging
-          cf-password: CF_PASSWORD_STAGING
-          cf-username: CF_USERNAME_STAGING
-          cf-space: tanf-staging
           filters:
             branches:
               only:
-                - main
+                - develop
       - deploy-staging:
           requires:
             - deploy-infrastructure-staging
           filters:
             branches:
               only:
                 - main
-
-  production-deployment:
-    unless:
-      or:
-        - << pipeline.parameters.run_dev_deployment >>
-        - << pipeline.parameters.run_nightly_owasp_scan >>
-    jobs:
-      - deploy-infrastructure-production:
-          filters:
-            branches:
-              only:
-                - master
-      - enable-versioning:
-          requires:
-            - deploy-infrastructure-production
-          target_env: prod
-          cf-password: CF_PASSWORD_PROD
-          cf-username: CF_USERNAME_PROD
-          cf-space: tanf-prod
-          filters:
-            branches:
-              only:
-                - master
       - deploy-production:
           requires:
             - deploy-infrastructure-production
           filters:
             branches:
               only:
                 - master
-      - prod-deploy-clamav:
+      - test-deployment-e2e:
           requires:
-            - deploy-infrastructure-production
+            - deploy-develop
           filters:
             branches:
               only:
-                - master
+                - develop
       - make_erd: # from ../util folder
           filters:
             branches:
               only:
+                - develop
                 - master
-

docs/Technical-Documentation/Architecture-Decision-Record/017-elastisearch.md

@@ -35,7 +35,11 @@ We will use an Elastisearch and Kibana stack for their modern feature set and sc
    - This is mostly mitigated through a SQL Workbench provided in Kibana where you can use regular SQL syntax to query records.
  * More infrastructure to manage.
    * This is mostly mitigated due to using a Cloud.gov managed service for ES and Terraform, this greatly simplifies scaling the cluster and abstracts away a lot of the difficult cluster management tasks we would have to do if we didn't use a managed service.
-   * Additional overhead to run a proxy application to control access to ES + Kibana
+   * Additional overhead to run a proxy application to control access to ES + Kibana+
+ * Security & Authentication
+   * Cloud.gov ES service is a wrapper around AWS OpenSearch/ES. AWS ES does not support Xpack (Elastic/Kibana native security features) because it was forked off of Elastic 7.10.2 which did not support Xpack at that time. This implies that our current Xpack configuration we are using with our local Elastic 7.17.6/Kibana 7.17.10 deployments ([implemented here](https://github.com/raft-tech/TANF-app/pull/2775)) will not be applicable to our deployed environments. To get around this issue, AWS suggests introducing a [proxy EC2 node](https://aws.amazon.com/blogs/security/how-to-control-access-to-your-amazon-elasticsearch-service-domain/) to implement the same type of features that Xpack natively provides by way of IAM policies and Signature Version 4 request signing. However, Cloud.gov does not allow access to the underlying AWS resources it is wrapping, thus making this workaround impossible.
+   * Another option to workaround AWS ES and Cloud.gov would be to deploy and manage our own ES cluster to Cloud.gov in each space. This also introduces large blocks in and of itself. To deploy/manage our own cluster would take at least one dedicated Elastic SME to ensure uptime, availability, updates, security, etc... This would also imply that we would need to purchase Elastic Stack self-managed licenses from Elastic. To acquire the minimum feature set we need to have robust security and authentication integration with TDP, we would need to procure platinum tier licenses. Elastic requires a minimum of three licenses to be purchased. We at a minimum, would need three nodes per environment (9 licenses total) to have a functioning Elastic cluster. However, the cost of these licenses and the cost of at least one person to manage the cluster(s)/licenses makes this an infeasible option.
+   * With these things considered, the best security/authentication we can provide at this time (12/22/2023) is by blocking all external incoming traffic to our Elastic and Kibana servers, and by leveraging the view based auth [implemented here](https://github.com/raft-tech/TANF-app/pull/2775), which prevents non admin and non HHS AMS authenticated users from navigating to Kibana via the frontend. We will not be able to use any Xpack features (RBAC, Realms, P2P encryption, etc...) used in that PR in our deployed environments.
 
 ## Notes
 

tdrs-backend/docker-compose.local.yml

@@ -36,6 +36,7 @@ services:
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
       - logger.discovery.level=debug
     ports:
       - 9200:9200

tdrs-backend/tdpservice/data_files/views.py

@@ -54,13 +54,15 @@ class DataFileViewSet(ModelViewSet):
 
     def create(self, request, *args, **kwargs):
         """Override create to upload in case of successful scan."""
+        logger.debug(f"{self.__class__.__name__}: {request}")
         response = super().create(request, *args, **kwargs)
 
         # only if file is passed the virus scan and created successfully will we perform side-effects:
         # * Send to parsing
         # * Upload to ACF-TITAN
         # * Send email to user
 
+        logger.debug(f"{self.__class__.__name__}: status: {response.status_code}")
         if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK:
             user = request.user
             data_file_id = response.data.get('id')
@@ -109,6 +111,7 @@ def create(self, request, *args, **kwargs):
             if len(recipients) > 0:
                 send_data_submitted_email(list(recipients), data_file, email_context, subject)
 
+        logger.debug(f"{self.__class__.__name__}: return val: {response}")
         return response
 
     def get_s3_versioning_id(self, file_name, prefix):

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py

@@ -68,7 +68,7 @@
                 validators.if_then_validator(
                     condition_field="FAMILY_AFFILIATION",
                     condition_function=validators.isInLimits(1, 2),
-                    result_field="PARENT_WITH_MINOR_CHILD",
+                    result_field="PARENT_MINOR_CHILD",
                     result_function=validators.isInLimits(1, 3),
                 ),
                 validators.if_then_validator(
@@ -358,8 +358,8 @@
                 ),
                 Field(
                     item="39",
-                    name="PARENT_WITH_MINOR_CHILD",
-                    friendly_name="parent with minor child",
+                    name="PARENT_MINOR_CHILD",
+                    friendly_name="parent of minor child",
                     type="number",
                     startIndex=53,
                     endIndex=54,

tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py

@@ -4,10 +4,12 @@
 from .t4 import t4
 from .t5 import t5
 from .t6 import t6
+from .t7 import t7
 
 t1 = t1
 t2 = t2
 t3 = t3
 t4 = t4
 t5 = t5
 t6 = t6
+t7 = t7

tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py

@@ -347,8 +347,8 @@
                 ),
                 Field(
                     item="39",
-                    name="PARENT_WITH_MINOR_CHILD",
-                    friendly_name="parent with minor child",
+                    name="PARENT_MINOR_CHILD",
+                    friendly_name="parent of minor child",
                     type="number",
                     startIndex=53,
                     endIndex=54,