From 1de71fe26330fe11140372b0b43ab42850e38f1d Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 19 Dec 2023 09:24:55 -0500 Subject: [PATCH 01/25] DEVAUTH - All-local devenv sans cloud or third-party auth workflows Remedy for git confusion on problem zip files --- .gitattributes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitattributes b/.gitattributes index faa6195ab..081bf1643 100644 --- a/.gitattributes +++ b/.gitattributes @@ -40,3 +40,8 @@ *.jpg binary *.gif binary *.pdf binary + +# Annoying product-updates zip files +product-updates/knowledge-center/FTANF_2009.zip binary eol=lf +product-updates/knowledge-center/SSPMOE_2009.zip binary eol=lf +product-updates/knowledge-center/ftanf.zip binary eol=lf From 496f60236f00e6a5be31cf33a83c0ef108bd466e Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Wed, 29 Nov 2023 11:21:53 -0500 Subject: [PATCH 02/25] Changes for fully local development - Enables direct frontend/backend communication sans Login.gov/Cloud.gov - Drives off new DEVELOPMENT env var - Pre-configures and disables frontend auth functionality - Testing based on new dev user - Install via web: ./manage.py generate_dev_user --- tdrs-backend/.env.example | 2 ++ tdrs-backend/tdpservice/data_files/views.py | 3 ++ tdrs-backend/tdpservice/settings/common.py | 1 + .../tdpservice/users/authentication.py | 15 ++++++++++ .../management/commands/generate_dev_user.py | 30 +++++++++++++++++++ tdrs-backend/tdpservice/users/permissions.py | 6 ++++ tdrs-frontend/.env | 2 ++ tdrs-frontend/docker-compose.yml | 6 +--- tdrs-frontend/src/actions/auth.js | 3 ++ tdrs-frontend/src/configureStore.js | 23 +++++++++++++- 10 files changed, 85 insertions(+), 6 deletions(-) create mode 100755 tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py diff --git a/tdrs-backend/.env.example b/tdrs-backend/.env.example index 5ffe271c1..3d325bc9c 100644 --- a/tdrs-backend/.env.example +++ b/tdrs-backend/.env.example @@ -2,6 +2,8 @@ # Copy this file to `.env` and replace variables as needed # +DEVELOPMENT=1 + # ## # Required environment variables # These must be defined or the application will encounter fatal errors diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index f01bb5be6..ebc769172 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -54,6 +54,7 @@ class DataFileViewSet(ModelViewSet): def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" + logger.debug(f"{self.__class__.__name__}: {request}") response = super().create(request, *args, **kwargs) # only if file is passed the virus scan and created successfully will we perform side-effects: @@ -61,6 +62,7 @@ def create(self, request, *args, **kwargs): # * Upload to ACF-TITAN # * Send email to user + logger.debug(f"{self.__class__.__name__}: status: {response.status_code}") if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: user = request.user data_file_id = response.data.get('id') @@ -109,6 +111,7 @@ def create(self, request, *args, **kwargs): if len(recipients) > 0: send_data_submitted_email(list(recipients), data_file, email_context, subject) + logger.debug(f"{self.__class__.__name__}: return val: {response}") return response def get_s3_versioning_id(self, file_name, prefix): diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 9abbb2c15..cd01de016 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -293,6 +293,7 @@ class Common(Configuration): "DEFAULT_RENDERER_CLASSES": DEFAULT_RENDERER_CLASSES, "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], "DEFAULT_AUTHENTICATION_CLASSES": ( + "tdpservice.users.authentication.DevAuthentication", "tdpservice.users.authentication.CustomAuthentication", "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.TokenAuthentication", diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index d238771ff..9c3c699cc 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -4,8 +4,23 @@ from rest_framework.authentication import BaseAuthentication import logging +import os logger = logging.getLogger(__name__) +class DevAuthentication(BaseAuthentication): + def authenticate(self, request): + if not os.environ.get('DEVELOPMENT'): + return None + logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}") + requser = request.data.get("user") + reqname = requser if requser and requser != "undefined" else "dev@test.com" + User = get_user_model() + authuser = User.objects.get(username=reqname) + if authuser and requser == "undefined": + request.data["user"] = authuser.id + return (User.objects.get(username=reqname), True) + + class CustomAuthentication(BaseAuthentication): """Define authentication and get user functions for custom authentication.""" diff --git a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py new file mode 100755 index 000000000..721d7d1a8 --- /dev/null +++ b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group +from django.core.management import BaseCommand + +User = get_user_model() + +email = "dev@test.com" +pswd = "pass" +first = "Jon" +last = "Tester" + +class Command(BaseCommand): + + def handle(self, *args, **options): + try: + user = User.objects.get(username=email) + print(f"Found {vars(user)}") + except User.DoesNotExist: + group = Group.objects.get(name="Developer") + user = User.objects.create(username=email, + email=email, + password=pswd, + first_name=first, + last_name=last, + account_approval_status="Approved") + user.groups.add(group) + print(f"Created {vars(user)}") + diff --git a/tdrs-backend/tdpservice/users/permissions.py b/tdrs-backend/tdpservice/users/permissions.py index 81e54fe54..31106304c 100644 --- a/tdrs-backend/tdpservice/users/permissions.py +++ b/tdrs-backend/tdpservice/users/permissions.py @@ -8,6 +8,9 @@ from collections import ChainMap from copy import deepcopy from typing import List, Optional, TYPE_CHECKING +import logging + +logger = logging.getLogger(__name__) if TYPE_CHECKING: # pragma: no cover @@ -126,6 +129,7 @@ class IsApprovedPermission(permissions.DjangoModelPermissions): def has_permission(self, request, view): """Return True if the user has been assigned a group and is approved.""" + logging.debug(f"{self.__class__.__name__}: {request} ; {view}") return (request.user.groups.first() is not None and request.user.account_approval_status == AccountApprovalStatusChoices.APPROVED) @@ -160,6 +164,8 @@ def has_permission(self, request, view): Data Analyst will only have permission to files within their STT and a Regional Manager will only have permission to files within their region. """ + logging.debug(f"{self.__class__.__name__}: {request} ; {view}") + # Checks for existence of `data_files.view_datafile` Permission has_permission = super().has_permission(request, view) diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env index 087144285..da6c85ddb 100644 --- a/tdrs-frontend/.env +++ b/tdrs-frontend/.env @@ -4,6 +4,8 @@ # WARNING: This file is checked in to source control, do NOT store any secrets in this file # +DEVELOPMENT=1 + # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://127.0.0.1:8080 diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index d75772fa5..b8b9c480e 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -7,7 +7,7 @@ services: ports: - 8090:8090 networks: - - local + - default volumes: - ./reports:/zap/wrk/:rw - ../scripts/zap-hook.py:/zap/scripts/zap-hook.py:ro @@ -21,7 +21,6 @@ services: - 3000:80 - 8080:8080 networks: - - local - default volumes: - ./:/home/node/app @@ -42,9 +41,6 @@ services: && nginx -g 'daemon off;'" networks: - local: - driver: bridge - default: external: name: external-net diff --git a/tdrs-frontend/src/actions/auth.js b/tdrs-frontend/src/actions/auth.js index 6b0147c6b..7dad3b0eb 100644 --- a/tdrs-frontend/src/actions/auth.js +++ b/tdrs-frontend/src/actions/auth.js @@ -40,6 +40,9 @@ export const SET_MOCK_LOGIN_STATE = 'SET_MOCK_LOGIN_STATE' */ export const fetchAuth = () => async (dispatch) => { + if (process.env.DEVELOPMENT) { + return 0 + } dispatch({ type: FETCH_AUTH }) try { const URL = `${process.env.REACT_APP_BACKEND_URL}/auth_check` diff --git a/tdrs-frontend/src/configureStore.js b/tdrs-frontend/src/configureStore.js index a9f340685..24d09e21b 100644 --- a/tdrs-frontend/src/configureStore.js +++ b/tdrs-frontend/src/configureStore.js @@ -4,6 +4,7 @@ import { createBrowserHistory } from 'history' import thunkMiddleware from 'redux-thunk' import loggerMiddleware from './middleware/logger' import createRootReducer from './reducers' +import { permissions } from './components/Header/developer_permissions' export const history = createBrowserHistory() @@ -13,9 +14,29 @@ export const history = createBrowserHistory() export default function configureStore(preloadedState) { const middlewares = [thunkMiddleware, loggerMiddleware] const composedEnhancers = composeWithDevTools(applyMiddleware(...middlewares)) + const devState = { + router: { location: { pathname: '/profile' } }, + auth: { + user: { + email: 'dev@test.com', + first_name: 'Jon', + last_name: 'Tester', + roles: [{ id: 1, name: 'Developer', permissions }], + access_request: true, + account_approval_status: 'Approved', + stt: { + id: 31, + type: 'state', + code: 'NJ', + name: 'New Jersey', + }, + }, + authenticated: true, + }, + } const store = createStore( createRootReducer(history), - preloadedState, + process.env.DEVELOPMENT ? devState : preloadedState, composedEnhancers ) return store From ca81e02c21609ee6fa9d13b3fc52c800eb632f89 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 5 Dec 2023 11:27:53 -0500 Subject: [PATCH 03/25] Reorganized front end logic on REACT_APP_DEVAUTH env var --- tdrs-frontend/.env | 2 -- tdrs-frontend/.env.development | 2 ++ tdrs-frontend/src/actions/auth.js | 2 +- tdrs-frontend/src/configureStore.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env index da6c85ddb..087144285 100644 --- a/tdrs-frontend/.env +++ b/tdrs-frontend/.env @@ -4,8 +4,6 @@ # WARNING: This file is checked in to source control, do NOT store any secrets in this file # -DEVELOPMENT=1 - # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://127.0.0.1:8080 diff --git a/tdrs-frontend/.env.development b/tdrs-frontend/.env.development index 3c0c68d15..b4c03e4a4 100644 --- a/tdrs-frontend/.env.development +++ b/tdrs-frontend/.env.development @@ -3,6 +3,8 @@ # This file is loaded when running `npm start` # +#REACT_APP_DEVAUTH=1 + # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://localhost:3000 diff --git a/tdrs-frontend/src/actions/auth.js b/tdrs-frontend/src/actions/auth.js index 7dad3b0eb..f98b8158f 100644 --- a/tdrs-frontend/src/actions/auth.js +++ b/tdrs-frontend/src/actions/auth.js @@ -40,7 +40,7 @@ export const SET_MOCK_LOGIN_STATE = 'SET_MOCK_LOGIN_STATE' */ export const fetchAuth = () => async (dispatch) => { - if (process.env.DEVELOPMENT) { + if (process.env.REACT_APP_DEVAUTH) { return 0 } dispatch({ type: FETCH_AUTH }) diff --git a/tdrs-frontend/src/configureStore.js b/tdrs-frontend/src/configureStore.js index 24d09e21b..b96bea6b2 100644 --- a/tdrs-frontend/src/configureStore.js +++ b/tdrs-frontend/src/configureStore.js @@ -36,7 +36,7 @@ export default function configureStore(preloadedState) { } const store = createStore( createRootReducer(history), - process.env.DEVELOPMENT ? devState : preloadedState, + process.env.REACT_APP_DEVAUTH ? devState : preloadedState, composedEnhancers ) return store From 1548342f17d6223fa415e17d97cdf1a9aefb245f Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 5 Dec 2023 11:39:38 -0500 Subject: [PATCH 04/25] Reorganized backend logic on REACT_APP_DEVAUTH env var --- tdrs-backend/.env.example | 2 +- tdrs-backend/tdpservice/users/authentication.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/.env.example b/tdrs-backend/.env.example index 3d325bc9c..de65d0e9e 100644 --- a/tdrs-backend/.env.example +++ b/tdrs-backend/.env.example @@ -2,7 +2,7 @@ # Copy this file to `.env` and replace variables as needed # -DEVELOPMENT=1 +#REACT_APP_DEVAUTH=1 # ## # Required environment variables diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index 9c3c699cc..5b53ea017 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -9,7 +9,7 @@ class DevAuthentication(BaseAuthentication): def authenticate(self, request): - if not os.environ.get('DEVELOPMENT'): + if not os.environ.get('REACT_APP_DEVAUTH'): return None logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}") requser = request.data.get("user") From b12346b46894b887bccca4edacbe3c6688d5e83b Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Fri, 8 Dec 2023 17:09:11 -0500 Subject: [PATCH 05/25] added is_superuser and is_staff attrs to dev user --- .../tdpservice/users/management/commands/generate_dev_user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py index 721d7d1a8..3faeb548a 100755 --- a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py +++ b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py @@ -22,6 +22,8 @@ def handle(self, *args, **options): user = User.objects.create(username=email, email=email, password=pswd, + is_superuser=True, + is_staff=True, first_name=first, last_name=last, account_approval_status="Approved") From d0bf9a36b3bbaa8d156c6301af51f89305bd9168 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 14 Dec 2023 00:02:33 -0500 Subject: [PATCH 06/25] DevAuth feature redesign inspired by Cypress - Initializing frontend w/POST /login/cypress: {devEmail, local-cypress-token} - Changed REACT_APP_DEVAUTH to provide the email of the desired dev user - Modified CustomAuthentication.authenticate to handle both known use cases - Added stt_id=31 to the initial dev user - Disabled ES disk threshold checking for local dev which blocked ES startup - Removed DevAuthentication and other now unnecessary code Resolved cherry-pick conflict --- tdrs-backend/docker-compose.local.yml | 1 + tdrs-backend/tdpservice/settings/common.py | 4 +- .../users/api/authorization_check.py | 2 + tdrs-backend/tdpservice/users/api/login.py | 1 + .../tdpservice/users/authentication.py | 47 +++++++++---------- .../management/commands/generate_dev_user.py | 1 + tdrs-frontend/src/actions/auth.js | 3 -- tdrs-frontend/src/configureStore.js | 22 +-------- tdrs-frontend/src/index.js | 20 ++++++++ 9 files changed, 51 insertions(+), 50 deletions(-) diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index 3c8e76317..d2cd5289c 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -36,6 +36,7 @@ services: environment: - discovery.type=single-node - xpack.security.enabled=false + - cluster.routing.allocation.disk.threshold_enabled=false - logger.discovery.level=debug ports: - 9200:9200 diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index cd01de016..bbae94770 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -293,7 +293,6 @@ class Common(Configuration): "DEFAULT_RENDERER_CLASSES": DEFAULT_RENDERER_CLASSES, "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], "DEFAULT_AUTHENTICATION_CLASSES": ( - "tdpservice.users.authentication.DevAuthentication", "tdpservice.users.authentication.CustomAuthentication", "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.TokenAuthentication", @@ -473,4 +472,5 @@ class Common(Configuration): }, } - CYPRESS_TOKEN = os.getenv('CYPRESS_TOKEN', None) + REACT_APP_DEVAUTH = os.getenv('REACT_APP_DEVAUTH', None) + CYPRESS_TOKEN = 'local-cypress-token' if REACT_APP_DEVAUTH else os.getenv('CYPRESS_TOKEN', None) diff --git a/tdrs-backend/tdpservice/users/api/authorization_check.py b/tdrs-backend/tdpservice/users/api/authorization_check.py index 57ed30527..ddcfcb1be 100644 --- a/tdrs-backend/tdpservice/users/api/authorization_check.py +++ b/tdrs-backend/tdpservice/users/api/authorization_check.py @@ -21,6 +21,8 @@ class AuthorizationCheck(APIView): def get(self, request, *args, **kwargs): """Handle get request and verify user is authorized.""" + logger.debug(f"{self.__class__.__name__}: {request} {args} {kwargs}") + user = request.user serializer = UserProfileSerializer(user) diff --git a/tdrs-backend/tdpservice/users/api/login.py b/tdrs-backend/tdpservice/users/api/login.py index 338508148..0efdf5cce 100644 --- a/tdrs-backend/tdpservice/users/api/login.py +++ b/tdrs-backend/tdpservice/users/api/login.py @@ -395,6 +395,7 @@ class CypressLoginDotGovAuthenticationOverride(TokenAuthorizationOIDC): def post(self, request): """Create a session for the specified user, if they exist.""" + logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}") username = request.data.get('username', None) token = request.data.get('token', None) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index 5b53ea017..2f4e97b30 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -3,51 +3,50 @@ from django.contrib.auth import get_user_model from rest_framework.authentication import BaseAuthentication +from rest_framework.request import Request import logging import os logger = logging.getLogger(__name__) -class DevAuthentication(BaseAuthentication): - def authenticate(self, request): - if not os.environ.get('REACT_APP_DEVAUTH'): - return None - logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}") - requser = request.data.get("user") - reqname = requser if requser and requser != "undefined" else "dev@test.com" - User = get_user_model() - authuser = User.objects.get(username=reqname) - if authuser and requser == "undefined": - request.data["user"] = authuser.id - return (User.objects.get(username=reqname), True) - - class CustomAuthentication(BaseAuthentication): """Define authentication and get user functions for custom authentication.""" @staticmethod - def authenticate(username=None, login_gov_uuid=None, hhs_id=None): - """Authenticate user with the request and username.""" + def authenticate(request=None, login_gov_uuid=None, hhs_id=None): + """ HACK + This method currently needs to support two unrelated workflows. + References: + tdpservice/users/api/login.py:TokenAuthorizationOIDC.handleUser + https://www.django-rest-framework.org/api-guide/authentication + """ + if type(request) == Request: + logging.debug(f"CustomAuthentication::authenticate: {request} {request.data} " + f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") + username = request.data.get('username') + else: + logging.debug(f"CustomAuthentication::authenticate: {username} " + f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") + username = request User = get_user_model() - logging.debug("CustomAuthentication::authenticate:hhs_id {}".format(hhs_id)) - logging.debug("CustomAuthentication::authenticate:login_gov_uuid {}".format(login_gov_uuid)) - logging.debug("CustomAuthentication::authenticate:username {}".format(username)) try: if hhs_id: try: - return User.objects.get(hhs_id=hhs_id) + user_obj = User.objects.get(hhs_id=hhs_id) except User.DoesNotExist: # If below line also fails with User.DNE, will bubble up and return None user = User.objects.filter(username=username) user.update(hhs_id=hhs_id) logging.debug("Updated user {} with hhs_id {}.".format(username, hhs_id)) - return User.objects.get(hhs_id=hhs_id) + user_obj = User.objects.get(hhs_id=hhs_id) elif login_gov_uuid: - return User.objects.get(login_gov_uuid=login_gov_uuid) + user_obj = User.objects.get(login_gov_uuid=login_gov_uuid) else: - return User.objects.get(username=username) + user_obj = User.objects.get(username=username) except User.DoesNotExist: - return None + user_obj = None + logging.debug(f"CustomAuthentication::authenticate found user: {user_obj}") + return (user_obj, None) if user_obj else None @staticmethod def get_user(user_id): diff --git a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py index 3faeb548a..e95bfbce3 100755 --- a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py +++ b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py @@ -26,6 +26,7 @@ def handle(self, *args, **options): is_staff=True, first_name=first, last_name=last, + stt_id=31, account_approval_status="Approved") user.groups.add(group) print(f"Created {vars(user)}") diff --git a/tdrs-frontend/src/actions/auth.js b/tdrs-frontend/src/actions/auth.js index f98b8158f..6b0147c6b 100644 --- a/tdrs-frontend/src/actions/auth.js +++ b/tdrs-frontend/src/actions/auth.js @@ -40,9 +40,6 @@ export const SET_MOCK_LOGIN_STATE = 'SET_MOCK_LOGIN_STATE' */ export const fetchAuth = () => async (dispatch) => { - if (process.env.REACT_APP_DEVAUTH) { - return 0 - } dispatch({ type: FETCH_AUTH }) try { const URL = `${process.env.REACT_APP_BACKEND_URL}/auth_check` diff --git a/tdrs-frontend/src/configureStore.js b/tdrs-frontend/src/configureStore.js index b96bea6b2..3b20fb58a 100644 --- a/tdrs-frontend/src/configureStore.js +++ b/tdrs-frontend/src/configureStore.js @@ -14,29 +14,9 @@ export const history = createBrowserHistory() export default function configureStore(preloadedState) { const middlewares = [thunkMiddleware, loggerMiddleware] const composedEnhancers = composeWithDevTools(applyMiddleware(...middlewares)) - const devState = { - router: { location: { pathname: '/profile' } }, - auth: { - user: { - email: 'dev@test.com', - first_name: 'Jon', - last_name: 'Tester', - roles: [{ id: 1, name: 'Developer', permissions }], - access_request: true, - account_approval_status: 'Approved', - stt: { - id: 31, - type: 'state', - code: 'NJ', - name: 'New Jersey', - }, - }, - authenticated: true, - }, - } const store = createStore( createRootReducer(history), - process.env.REACT_APP_DEVAUTH ? devState : preloadedState, + preloadedState, composedEnhancers ) return store diff --git a/tdrs-frontend/src/index.js b/tdrs-frontend/src/index.js index 3a2f2060c..394371280 100644 --- a/tdrs-frontend/src/index.js +++ b/tdrs-frontend/src/index.js @@ -24,8 +24,28 @@ axios.defaults.xsrfCookieName = 'csrftoken' axios.defaults.xsrfHeaderName = 'X-CSRFToken' axios.defaults.withCredentials = true +function devLogin(devEmail) { + const BACKEND_URL = process.env.REACT_APP_BACKEND_URL + axios + .post(`${BACKEND_URL}/login/cypress`, { + username: devEmail, + token: 'local-cypress-token', + }) + .then(function (response) { + console.log(response) + }) + .catch(function (error) { + console.log(error) + }) + store.dispatch({ type: 'SET_AUTH', payload: { devEmail } }) + console.log(`dispatched SET_AUTH(${devEmail})`) +} + // call auth_check const store = configureStore() +if (process.env.REACT_APP_DEVAUTH) { + devLogin(process.env.REACT_APP_DEVAUTH) +} store.dispatch(fetchAuth()) // if (window.location.href.match(/https:\/\/.*\.app\.cloud\.gov/)) { From 0ff4a662fb1d2344cd92a8f5b99eb4f55f81ce38 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 14 Dec 2023 00:49:46 -0500 Subject: [PATCH 07/25] Fixed CustomAuthentication.authenticate return val for login.py use case --- tdrs-backend/tdpservice/users/authentication.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index 2f4e97b30..065f37853 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -46,7 +46,9 @@ def authenticate(request=None, login_gov_uuid=None, hhs_id=None): except User.DoesNotExist: user_obj = None logging.debug(f"CustomAuthentication::authenticate found user: {user_obj}") - return (user_obj, None) if user_obj else None + if type(request) == Request: + return (user_obj, None) if user_obj else None + return user_obj @staticmethod def get_user(user_id): From fe12fb0621471714f3c28b6e52a48e1f929d6a4a Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 14 Dec 2023 00:52:36 -0500 Subject: [PATCH 08/25] Fixed CustomAuthentication.authenticate logging for login.py use case --- tdrs-backend/tdpservice/users/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index 065f37853..7b3246d89 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -20,13 +20,13 @@ def authenticate(request=None, login_gov_uuid=None, hhs_id=None): https://www.django-rest-framework.org/api-guide/authentication """ if type(request) == Request: + username = request.data.get('username') logging.debug(f"CustomAuthentication::authenticate: {request} {request.data} " f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") - username = request.data.get('username') else: + username = request logging.debug(f"CustomAuthentication::authenticate: {username} " f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") - username = request User = get_user_model() try: if hhs_id: From fdcb6e21d49c5050b0be07b6e628bf6bdb4ab862 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 14 Dec 2023 00:57:57 -0500 Subject: [PATCH 09/25] Removed unneeded permissions import --- tdrs-frontend/src/configureStore.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-frontend/src/configureStore.js b/tdrs-frontend/src/configureStore.js index 3b20fb58a..a9f340685 100644 --- a/tdrs-frontend/src/configureStore.js +++ b/tdrs-frontend/src/configureStore.js @@ -4,7 +4,6 @@ import { createBrowserHistory } from 'history' import thunkMiddleware from 'redux-thunk' import loggerMiddleware from './middleware/logger' import createRootReducer from './reducers' -import { permissions } from './components/Header/developer_permissions' export const history = createBrowserHistory() From d475645e49528cea736f430b4061cd3bc997df6b Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 14 Dec 2023 08:12:49 -0500 Subject: [PATCH 10/25] Updates to REACT_APP_DEVAUTH env var settings - Enabled with an email address value - Disabled by default --- tdrs-backend/.env.example | 2 +- tdrs-frontend/.env | 2 ++ tdrs-frontend/.env.development | 2 -- tdrs-frontend/docker-compose.yml | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/.env.example b/tdrs-backend/.env.example index de65d0e9e..55fb19db7 100644 --- a/tdrs-backend/.env.example +++ b/tdrs-backend/.env.example @@ -2,7 +2,7 @@ # Copy this file to `.env` and replace variables as needed # -#REACT_APP_DEVAUTH=1 +#REACT_APP_DEVAUTH=dev@test.com # ## # Required environment variables diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env index 087144285..a0abd8b0b 100644 --- a/tdrs-frontend/.env +++ b/tdrs-frontend/.env @@ -4,6 +4,8 @@ # WARNING: This file is checked in to source control, do NOT store any secrets in this file # +#REACT_APP_DEVAUTH=dev@test.com + # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://127.0.0.1:8080 diff --git a/tdrs-frontend/.env.development b/tdrs-frontend/.env.development index b4c03e4a4..3c0c68d15 100644 --- a/tdrs-frontend/.env.development +++ b/tdrs-frontend/.env.development @@ -3,8 +3,6 @@ # This file is loaded when running `npm start` # -#REACT_APP_DEVAUTH=1 - # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://localhost:3000 diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index b8b9c480e..23c0a0669 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -28,6 +28,7 @@ services: - NGINX_FRONTEND=tdp-frontend - BACK_END=web - LOCAL_DEV=true + - REACT_APP_DEVAUTH=${REACT_APP_DEVAUTH} command: > /bin/sh -c "echo 'starting nginx' && From 45d1d70965f02dc39c8a137ae05d9092a792c112 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Fri, 15 Dec 2023 13:49:30 -0500 Subject: [PATCH 11/25] Restored support for CustomAuthentication.authenticate username keyword --- tdrs-backend/tdpservice/users/authentication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index 7b3246d89..e26a4db4e 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -12,7 +12,7 @@ class CustomAuthentication(BaseAuthentication): """Define authentication and get user functions for custom authentication.""" @staticmethod - def authenticate(request=None, login_gov_uuid=None, hhs_id=None): + def authenticate(request=None, username=None, login_gov_uuid=None, hhs_id=None): """ HACK This method currently needs to support two unrelated workflows. References: @@ -24,7 +24,6 @@ def authenticate(request=None, login_gov_uuid=None, hhs_id=None): logging.debug(f"CustomAuthentication::authenticate: {request} {request.data} " f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") else: - username = request logging.debug(f"CustomAuthentication::authenticate: {username} " f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") User = get_user_model() From ff79717a618a9e879f6acc2957e433a9afc57bf7 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Fri, 15 Dec 2023 15:51:46 -0500 Subject: [PATCH 12/25] Modified CustomAuthentication.authenticate comment to satisfy flake8 --- tdrs-backend/tdpservice/users/authentication.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index e26a4db4e..b1cc56f0e 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -5,7 +5,6 @@ from rest_framework.authentication import BaseAuthentication from rest_framework.request import Request import logging -import os logger = logging.getLogger(__name__) class CustomAuthentication(BaseAuthentication): @@ -13,12 +12,11 @@ class CustomAuthentication(BaseAuthentication): @staticmethod def authenticate(request=None, username=None, login_gov_uuid=None, hhs_id=None): - """ HACK - This method currently needs to support two unrelated workflows. - References: - tdpservice/users/api/login.py:TokenAuthorizationOIDC.handleUser - https://www.django-rest-framework.org/api-guide/authentication - """ + """Authenticate user with the request and username.""" + # HACK: This method currently needs to support two unrelated workflows. + # References: + # tdpservice/users/api/login.py:TokenAuthorizationOIDC.handleUser + # https://www.django-rest-framework.org/api-guide/authentication if type(request) == Request: username = request.data.get('username') logging.debug(f"CustomAuthentication::authenticate: {request} {request.data} " From 23f6a693442359fc31e6057083f2428110719292 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 19 Dec 2023 10:23:24 -0500 Subject: [PATCH 13/25] Removed unnecessary CYPRESS_TOKEN conditional setting --- tdrs-backend/tdpservice/settings/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index bbae94770..adb327383 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -472,5 +472,4 @@ class Common(Configuration): }, } - REACT_APP_DEVAUTH = os.getenv('REACT_APP_DEVAUTH', None) - CYPRESS_TOKEN = 'local-cypress-token' if REACT_APP_DEVAUTH else os.getenv('CYPRESS_TOKEN', None) + CYPRESS_TOKEN = os.getenv('REACT_APP_DEVAUTH', None) From d21b155913bdde78ff7ecbfbbe66c7fe6bebebce Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 19 Dec 2023 10:25:34 -0500 Subject: [PATCH 14/25] Really removed unnecessary CYPRESS_TOKEN setting --- tdrs-backend/tdpservice/settings/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index adb327383..5e77e9936 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -472,4 +472,4 @@ class Common(Configuration): }, } - CYPRESS_TOKEN = os.getenv('REACT_APP_DEVAUTH', None) + CYPRESS_TOKEN = os.getenv(CYPRESS_TOKEN, None) From 09d9d2c92bbd70928e16b785c159aa251ab246c5 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 19 Dec 2023 10:26:34 -0500 Subject: [PATCH 15/25] Really removed unnecessary CYPRESS_TOKEN setting --- tdrs-backend/tdpservice/settings/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 5e77e9936..9abbb2c15 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -472,4 +472,4 @@ class Common(Configuration): }, } - CYPRESS_TOKEN = os.getenv(CYPRESS_TOKEN, None) + CYPRESS_TOKEN = os.getenv('CYPRESS_TOKEN', None) From d15bf6dbf4f292da66a9b8a087ee79dfc727f180 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 19 Dec 2023 10:28:36 -0500 Subject: [PATCH 16/25] Removed unnecessary REACT_APP_DEVAUTH setting from backend --- tdrs-backend/.env.example | 2 -- 1 file changed, 2 deletions(-) diff --git a/tdrs-backend/.env.example b/tdrs-backend/.env.example index 55fb19db7..5ffe271c1 100644 --- a/tdrs-backend/.env.example +++ b/tdrs-backend/.env.example @@ -2,8 +2,6 @@ # Copy this file to `.env` and replace variables as needed # -#REACT_APP_DEVAUTH=dev@test.com - # ## # Required environment variables # These must be defined or the application will encounter fatal errors From bdb42a8462f7315af716816624f2ccd22bf4a911 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 19 Dec 2023 11:13:55 -0500 Subject: [PATCH 17/25] flake8 changes for generate_dev_user.py --- .../users/management/commands/generate_dev_user.py | 5 +++-- tdrs-frontend/.env | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py index e95bfbce3..fcb4932a0 100755 --- a/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py +++ b/tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +"""generate_dev_user command.""" from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -12,8 +12,10 @@ last = "Tester" class Command(BaseCommand): + """Command class.""" def handle(self, *args, **options): + """Generate dev user if it doesn't exist.""" try: user = User.objects.get(username=email) print(f"Found {vars(user)}") @@ -30,4 +32,3 @@ def handle(self, *args, **options): account_approval_status="Approved") user.groups.add(group) print(f"Created {vars(user)}") - diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env index a0abd8b0b..55fdf303d 100644 --- a/tdrs-frontend/.env +++ b/tdrs-frontend/.env @@ -4,7 +4,7 @@ # WARNING: This file is checked in to source control, do NOT store any secrets in this file # -#REACT_APP_DEVAUTH=dev@test.com +REACT_APP_DEVAUTH=dev@test.com # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://127.0.0.1:8080 From c71c64798a540713fd0a7761ba83079efe4e0d2d Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 4 Jan 2024 13:43:12 -0500 Subject: [PATCH 18/25] Fixed Profile primaryRole bug --- tdrs-frontend/src/components/Profile/Profile.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-frontend/src/components/Profile/Profile.jsx b/tdrs-frontend/src/components/Profile/Profile.jsx index 232fc3470..5ae65cdeb 100644 --- a/tdrs-frontend/src/components/Profile/Profile.jsx +++ b/tdrs-frontend/src/components/Profile/Profile.jsx @@ -12,7 +12,7 @@ import { function Profile() { const user = useSelector((state) => state.auth.user) // Most higher-env users will only have a single role, so just grab the first one. - const primaryRole = user?.roles[0] + const primaryRole = user?.roles?.[0] const missingAccessRequest = useSelector(accountIsMissingAccessRequest) const isAccessRequestPending = useSelector(accountIsInReview) From 1ccaaff80ab68ceaa412d6853eadca131af628cf Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Thu, 4 Jan 2024 13:50:25 -0500 Subject: [PATCH 19/25] Disabled REACT_APP_DEVAUTH for general cases --- tdrs-frontend/.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env index 55fdf303d..882a4aafa 100644 --- a/tdrs-frontend/.env +++ b/tdrs-frontend/.env @@ -4,7 +4,8 @@ # WARNING: This file is checked in to source control, do NOT store any secrets in this file # -REACT_APP_DEVAUTH=dev@test.com +# Uncomment for local dev only! +#REACT_APP_DEVAUTH=dev@test.com # The hostname behind the tdrs-backend Django app REACT_APP_BACKEND_HOST=http://127.0.0.1:8080 From c6702bfa438f58c231af0f63e50be037964fc675 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Mon, 8 Jan 2024 10:48:23 -0500 Subject: [PATCH 20/25] Added pre-existing user condition to DEVAUTH check Removed login.py logging --- tdrs-backend/tdpservice/users/api/authorization_check.py | 2 +- tdrs-backend/tdpservice/users/api/login.py | 1 - tdrs-frontend/src/index.js | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/users/api/authorization_check.py b/tdrs-backend/tdpservice/users/api/authorization_check.py index ddcfcb1be..3ac867be0 100644 --- a/tdrs-backend/tdpservice/users/api/authorization_check.py +++ b/tdrs-backend/tdpservice/users/api/authorization_check.py @@ -21,7 +21,7 @@ class AuthorizationCheck(APIView): def get(self, request, *args, **kwargs): """Handle get request and verify user is authorized.""" - logger.debug(f"{self.__class__.__name__}: {request} {args} {kwargs}") + logger.debug(f"{self.__class__.__name__}: {request} {request.user} {args} {kwargs}") user = request.user serializer = UserProfileSerializer(user) diff --git a/tdrs-backend/tdpservice/users/api/login.py b/tdrs-backend/tdpservice/users/api/login.py index 0efdf5cce..338508148 100644 --- a/tdrs-backend/tdpservice/users/api/login.py +++ b/tdrs-backend/tdpservice/users/api/login.py @@ -395,7 +395,6 @@ class CypressLoginDotGovAuthenticationOverride(TokenAuthorizationOIDC): def post(self, request): """Create a session for the specified user, if they exist.""" - logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}") username = request.data.get('username', None) token = request.data.get('token', None) diff --git a/tdrs-frontend/src/index.js b/tdrs-frontend/src/index.js index 394371280..809c03c6c 100644 --- a/tdrs-frontend/src/index.js +++ b/tdrs-frontend/src/index.js @@ -43,7 +43,7 @@ function devLogin(devEmail) { // call auth_check const store = configureStore() -if (process.env.REACT_APP_DEVAUTH) { +if (process.env.REACT_APP_DEVAUTH && !store.getState().auth?.user) { devLogin(process.env.REACT_APP_DEVAUTH) } store.dispatch(fetchAuth()) From 6003e338f9a8802c695c7f7b6fbcbc154bdda168 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Mon, 8 Jan 2024 19:08:37 -0500 Subject: [PATCH 21/25] Reverted changes to CustomAuthentication.authenticate --- .../tdpservice/users/authentication.py | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index b1cc56f0e..d238771ff 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -3,7 +3,6 @@ from django.contrib.auth import get_user_model from rest_framework.authentication import BaseAuthentication -from rest_framework.request import Request import logging logger = logging.getLogger(__name__) @@ -11,41 +10,29 @@ class CustomAuthentication(BaseAuthentication): """Define authentication and get user functions for custom authentication.""" @staticmethod - def authenticate(request=None, username=None, login_gov_uuid=None, hhs_id=None): + def authenticate(username=None, login_gov_uuid=None, hhs_id=None): """Authenticate user with the request and username.""" - # HACK: This method currently needs to support two unrelated workflows. - # References: - # tdpservice/users/api/login.py:TokenAuthorizationOIDC.handleUser - # https://www.django-rest-framework.org/api-guide/authentication - if type(request) == Request: - username = request.data.get('username') - logging.debug(f"CustomAuthentication::authenticate: {request} {request.data} " - f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") - else: - logging.debug(f"CustomAuthentication::authenticate: {username} " - f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}") User = get_user_model() + logging.debug("CustomAuthentication::authenticate:hhs_id {}".format(hhs_id)) + logging.debug("CustomAuthentication::authenticate:login_gov_uuid {}".format(login_gov_uuid)) + logging.debug("CustomAuthentication::authenticate:username {}".format(username)) try: if hhs_id: try: - user_obj = User.objects.get(hhs_id=hhs_id) + return User.objects.get(hhs_id=hhs_id) except User.DoesNotExist: # If below line also fails with User.DNE, will bubble up and return None user = User.objects.filter(username=username) user.update(hhs_id=hhs_id) logging.debug("Updated user {} with hhs_id {}.".format(username, hhs_id)) - user_obj = User.objects.get(hhs_id=hhs_id) + return User.objects.get(hhs_id=hhs_id) elif login_gov_uuid: - user_obj = User.objects.get(login_gov_uuid=login_gov_uuid) + return User.objects.get(login_gov_uuid=login_gov_uuid) else: - user_obj = User.objects.get(username=username) + return User.objects.get(username=username) except User.DoesNotExist: - user_obj = None - logging.debug(f"CustomAuthentication::authenticate found user: {user_obj}") - if type(request) == Request: - return (user_obj, None) if user_obj else None - return user_obj + return None @staticmethod def get_user(user_id): From f72ec10f46a6703d55cf4b6b0f4f811121e78e91 Mon Sep 17 00:00:00 2001 From: Thomas Tignor Date: Tue, 9 Jan 2024 07:17:36 -0500 Subject: [PATCH 22/25] Added CustomAuthentication.authenticate TODO and improved logging --- tdrs-backend/tdpservice/users/authentication.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/users/authentication.py b/tdrs-backend/tdpservice/users/authentication.py index d238771ff..4002f440c 100644 --- a/tdrs-backend/tdpservice/users/authentication.py +++ b/tdrs-backend/tdpservice/users/authentication.py @@ -12,10 +12,13 @@ class CustomAuthentication(BaseAuthentication): @staticmethod def authenticate(username=None, login_gov_uuid=None, hhs_id=None): """Authenticate user with the request and username.""" + # TODO: Provide separate implementations for two unrelated workflows + # both using this method. (The latter appears to always fail.) + # References: + # tdpservice/users/api/login.py:TokenAuthorizationOIDC.handleUser + # https://www.django-rest-framework.org/api-guide/authentication User = get_user_model() - logging.debug("CustomAuthentication::authenticate:hhs_id {}".format(hhs_id)) - logging.debug("CustomAuthentication::authenticate:login_gov_uuid {}".format(login_gov_uuid)) - logging.debug("CustomAuthentication::authenticate:username {}".format(username)) + logging.debug(f"CustomAuthentication::authenticate: {username}, {login_gov_uuid}, {hhs_id}") try: if hhs_id: try: From b35795fc8788cf29cbb09de2accd513f43a2f897 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Wed, 10 Jan 2024 08:36:40 -0700 Subject: [PATCH 23/25] Tribal Section 4 Parsing & Validation (#2762) * Added formating for header and autofit columns * Formatted the headers * added year/month to the columns * Added contants - translation column * added friendly names to T1 and T2 * added friendly name to m1 and m2 * added friendly name to m3 * added friendly_name to t3 * added friendly_name to t4 and t5 * added friendly_name to t7 * correct missing friendly_name * correction on failing tests * addedfriendly name to excel report * linting * linting * linting * delete contants.py * added test for json field in error model * linting * linting * linting * - Added Tribal models, docs, admin refs - Added support for tribal parsing * - Update t1 field validators * - t2/t3 field validators * 2599-added friendly name to postparsing validators * - Updated cat3 validators * - Updated how we detect tribe code * - Added error handling for inconsistent header info wrt tribe code - Added new test * - Updated test * - Grammar fix - Removed unused function * refining the validator tests * - Format with Black - Add missing tests * - Fix lint * added returning fields names to validators * added friendly_name to error field * linting * corrections on views/tests * corrections for fields * failing test corrected * failing test corrected * correcting test failures * linting * corrected the excel fiel generator * removed excessive space in validator * linting * - Updated based on review feedback * - removing debug assert * - Fix lint * listing * - deconflicting migrations * - Fixing test * added m6 * lint * - Adding datafile * - Added support for parsing tribal t4/t5 - Added test for t4/t5 parsing * - ADded cat2/cat3 validators * corrected new line break * - Update doc strings * - Fixed failing test * - Fix lint * refactored validator logic * - resolving conflict * - Fix lint errors * - Resolve conflict * linting and correction on t1 * friendly_name correction from comments * friendly_name correction * corrected failing test for m5 * refactor the field_json creation DRY * - Renamed tribal files - Updated to allow case aggregates - Updated datafile to have a more convenient date range - Updated tests to capture case aggregates * - Reduced file size - Updated test * - Fix lint errors * - Added support for parsing and validating tribal t6 records * - Fix lint * - Added support for parsing/validating tribal t7 * - Fix lint * - Docstring fix * - Docstring fix * friendly_name corrections * - Adding friendly names to tribal * - Fixing typo * - Added friendly names * - Adding friendly names * - Adding friendly names * linting and cleaning errors * linting * correction on friendly_names * corrected friendly_name for test_util * correction child care - number of months * fixed a few more typos and some spacing. (#2767) * fixed a few more typos and some spacing. * fixed linting issues * missed a spot. --------- Co-authored-by: George Hudson * - knowledge center * - Integrating Jans changes. Parametrizing values_is_empty * - Update based on review feedback * - Updated length checks * - removing fips code * Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * - update based on review * - Fix migration * - Fixed name clash on PARENT_MINOR_CHILD * - Fixed name clash on PARENT_MINOR_CHILD * - Cherry pick commit and remove test file * Revert "- Fixed name clash on PARENT_MINOR_CHILD" This reverts commit 5850dc845e2f84d7858e591cb6c0f5c381e63470. --------- Co-authored-by: Mo Sohani Co-authored-by: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Co-authored-by: George Hudson Co-authored-by: George Hudson Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../tdpservice/parsers/schema_defs/tanf/t2.py | 6 +- .../schema_defs/tribal_tanf/__init__.py | 2 + .../parsers/schema_defs/tribal_tanf/t2.py | 4 +- .../parsers/schema_defs/tribal_tanf/t7.py | 109 ++++++++++++++++++ .../test/data/tribal_section_4_fake.txt | 3 + .../tdpservice/parsers/test/factories.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 28 ++++- .../parsers/test/test_validators.py | 10 +- tdrs-backend/tdpservice/parsers/util.py | 6 + .../search_indexes/admin/__init__.py | 1 + .../tdpservice/search_indexes/admin/tribal.py | 19 +++ .../search_indexes/documents/tanf.py | 2 +- .../search_indexes/documents/tribal.py | 28 ++++- .../migrations/0025_tribal_tanf_t7.py | 29 +++++ .../0026_parent_minor_child_rename.py | 18 +++ .../tdpservice/search_indexes/models/tanf.py | 2 +- .../search_indexes/models/tribal.py | 28 +++++ .../search_indexes/test/test_model_mapping.py | 34 +++++- .../tdpservice/users/test/test_permissions.py | 3 + 19 files changed, 315 insertions(+), 19 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 63f8706ac..1331490ea 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -69,7 +69,7 @@ validators.if_then_validator( condition_field="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 2), - result_field="PARENT_WITH_MINOR_CHILD", + result_field="PARENT_MINOR_CHILD", result_function=validators.isInLimits(1, 3), ), validators.if_then_validator( @@ -359,8 +359,8 @@ ), Field( item="39", - name="PARENT_WITH_MINOR_CHILD", - friendly_name="parent with minor child", + name="PARENT_MINOR_CHILD", + friendly_name="parent of minor child", type="number", startIndex=53, endIndex=54, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py index e82db47b6..7c1997236 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/__init__.py @@ -4,6 +4,7 @@ from .t4 import t4 from .t5 import t5 from .t6 import t6 +from .t7 import t7 t1 = t1 t2 = t2 @@ -11,3 +12,4 @@ t4 = t4 t5 = t5 t6 = t6 +t7 = t7 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index ed1d03e8d..980e653fd 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -348,8 +348,8 @@ ), Field( item="39", - name="PARENT_WITH_MINOR_CHILD", - friendly_name="parent with minor child", + name="PARENT_MINOR_CHILD", + friendly_name="parent of minor child", type="number", startIndex=53, endIndex=54, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py new file mode 100644 index 000000000..332caed97 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -0,0 +1,109 @@ +"""Schema for Tribal TANF T7 Row.""" + +from tdpservice.parsers.util import SchemaManager +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers import validators +from tdpservice.search_indexes.models.tribal import Tribal_TANF_T7 + +schemas = [] + +validator_index = 7 +section_ind_index = 7 +stratum_index = 8 +families_index = 10 +for i in range(1, 31): + month_index = (i - 1) % 3 + sub_item_labels = ["A", "B", "C"] + families_value_item_number = f"6{sub_item_labels[month_index]}" + + schemas.append( + RowSchema( + model=Tribal_TANF_T7, + quiet_preparser_errors=i > 1, + preparsing_validators=[ + validators.hasLength(247), + validators.notEmpty(0, 7), + validators.notEmpty(validator_index, validator_index + 24), + ], + postparsing_validators=[], + fields=[ + Field( + item="0", + name="RecordType", + friendly_name="record type", + type="string", + startIndex=0, + endIndex=2, + required=True, + validators=[], + ), + Field( + item="3", + name="CALENDAR_QUARTER", + friendly_name="calendar quarter", + type="number", + startIndex=2, + endIndex=7, + required=True, + validators=[ + validators.dateYearIsLargerThan(1998), + validators.quarterIsValid(), + ], + ), + TransformField( + transform_func=calendar_quarter_to_rpt_month_year(month_index), + item="3A", + name="RPT_MONTH_YEAR", + friendly_name="reporting month year", + type="number", + startIndex=2, + endIndex=7, + required=True, + validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ], + ), + Field( + item="4", + name="TDRS_SECTION_IND", + friendly_name="tdrs section indicator", + type="string", + startIndex=section_ind_index, + endIndex=section_ind_index + 1, + required=True, + validators=[validators.oneOf(["1", "2"])], + ), + Field( + item="5", + name="STRATUM", + friendly_name="stratum", + type="string", + startIndex=stratum_index, + endIndex=stratum_index + 2, + required=True, + validators=[validators.isInStringRange(0, 99)], + ), + Field( + item=families_value_item_number, + name="FAMILIES_MONTH", + friendly_name="families month", + type="number", + startIndex=families_index, + endIndex=families_index + 7, + required=True, + validators=[validators.isInLimits(0, 9999999)], + ), + ], + ) + ) + + index_offset = 0 if i % 3 != 0 else 24 + validator_index += index_offset + section_ind_index += index_offset + stratum_index += index_offset + families_index += 7 if i % 3 != 0 else 10 + +t7 = SchemaManager(schemas=schemas) diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt new file mode 100644 index 000000000..904d0bb79 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt @@ -0,0 +1,3 @@ +HEADER20194S00142TAN1EU +T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 63a6d237d..1d5bb28af 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -170,7 +170,7 @@ class Meta: RECEIVE_SSI = 1 MARITAL_STATUS = 1 RELATIONSHIP_HOH = "01" - PARENT_WITH_MINOR_CHILD = 1 + PARENT_MINOR_CHILD = 1 NEEDS_PREGNANT_WOMAN = 1 EDUCATION_LEVEL = "01" CITIZENSHIP_STATUS = 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 71f4794f0..02cd7365b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -6,7 +6,7 @@ from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 from tdpservice.search_indexes.models.tribal import Tribal_TANF_T1, Tribal_TANF_T2, Tribal_TANF_T3, Tribal_TANF_T4 -from tdpservice.search_indexes.models.tribal import Tribal_TANF_T5, Tribal_TANF_T6 +from tdpservice.search_indexes.models.tribal import Tribal_TANF_T5, Tribal_TANF_T6, Tribal_TANF_T7 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M4, SSP_M5, SSP_M6, SSP_M7 from .factories import DataFileSummaryFactory from tdpservice.data_files.models import DataFile @@ -1096,3 +1096,29 @@ def test_parse_tribal_section_3_file(tribal_section_3_file): assert t6.NUM_APPLICATIONS == 1 assert t6.NUM_FAMILIES == 41 assert t6.NUM_CLOSED_CASES == 3 + +@pytest.fixture +def tribal_section_4_file(stt_user, stt): + """Fixture for tribal_section_4_fake.txt.""" + return util.create_test_datafile('tribal_section_4_fake.txt', stt_user, stt, "Tribal Stratum Data") + +@pytest.mark.django_db() +def test_parse_tribal_section_4_file(tribal_section_4_file): + """Test parsing Tribal TANF Section 4 submission.""" + parse.parse_datafile(tribal_section_4_file) + + assert Tribal_TANF_T7.objects.all().count() == 18 + + t7_objs = Tribal_TANF_T7.objects.all().order_by('FAMILIES_MONTH') + + first = t7_objs.first() + sixth = t7_objs[5] + + assert first.RPT_MONTH_YEAR == 202011 + assert sixth.RPT_MONTH_YEAR == 202012 + + assert first.TDRS_SECTION_IND == '2' + assert sixth.TDRS_SECTION_IND == '2' + + assert first.FAMILIES_MONTH == 274 + assert sixth.FAMILIES_MONTH == 499 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index dc65d2745..bd3eb88ce 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -587,16 +587,16 @@ def test_validate_parent_with_minor(self, record): """Test cat3 validator for parent with a minor child.""" val = validators.if_then_validator( condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), - result_field='PARENT_WITH_MINOR_CHILD', result_function=validators.isInLimits(1, 3), + result_field='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), ) result = val(record) - assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_WITH_MINOR_CHILD']) + assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) - record.PARENT_WITH_MINOR_CHILD = 0 + record.PARENT_MINOR_CHILD = 0 result = val(record) - assert result == (False, 'if FAMILY_AFFILIATION :1 validator1 passed then PARENT_WITH_MINOR_CHILD 0 is not ' + + assert result == (False, 'if FAMILY_AFFILIATION :1 validator1 passed then PARENT_MINOR_CHILD 0 is not ' + 'larger or equal to 1 and smaller or equal to 3.', - ['FAMILY_AFFILIATION', 'PARENT_WITH_MINOR_CHILD']) + ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) def test_validate_education_level(self, record): """Test cat3 validator for education level.""" diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 7cd453a8a..58df9bbdd 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -201,6 +201,12 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'T6': schema_defs.tribal_tanf.t6, } }, + 'S': { + 'section': DataFile.Section.TRIBAL_STRATUM_DATA, + 'models': { + 'T7': schema_defs.tribal_tanf.t7, + } + }, }, } diff --git a/tdrs-backend/tdpservice/search_indexes/admin/__init__.py b/tdrs-backend/tdpservice/search_indexes/admin/__init__.py index 7585fece8..91469dfa5 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/__init__.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/__init__.py @@ -16,6 +16,7 @@ admin.site.register(models.tribal.Tribal_TANF_T4, tribal.Tribal_TANF_T4Admin) admin.site.register(models.tribal.Tribal_TANF_T5, tribal.Tribal_TANF_T5Admin) admin.site.register(models.tribal.Tribal_TANF_T6, tribal.Tribal_TANF_T6Admin) +admin.site.register(models.tribal.Tribal_TANF_T7, tribal.Tribal_TANF_T7Admin) admin.site.register(models.ssp.SSP_M1, ssp.SSP_M1Admin) admin.site.register(models.ssp.SSP_M2, ssp.SSP_M2Admin) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py index e2612df44..757620a24 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py @@ -99,3 +99,22 @@ class Tribal_TANF_T6Admin(admin.ModelAdmin): CreationDateFilter, 'RPT_MONTH_YEAR' ] + +class Tribal_TANF_T7Admin(admin.ModelAdmin): + """ModelAdmin class for parsed Tribal T7 data files.""" + + list_display = [ + 'RecordType', + 'CALENDAR_QUARTER', + 'RPT_MONTH_YEAR', + 'TDRS_SECTION_IND', + 'STRATUM', + 'FAMILIES_MONTH', + 'datafile', + ] + + list_filter = [ + 'CALENDAR_QUARTER', + CreationDateFilter, + 'RPT_MONTH_YEAR', + ] diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index c613c50a2..a6b5fd6b4 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -110,7 +110,7 @@ class Django: 'RECEIVE_SSI', 'MARITAL_STATUS', 'RELATIONSHIP_HOH', - 'PARENT_WITH_MINOR_CHILD', + 'PARENT_MINOR_CHILD', 'NEEDS_PREGNANT_WOMAN', 'EDUCATION_LEVEL', 'CITIZENSHIP_STATUS', diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tribal.py b/tdrs-backend/tdpservice/search_indexes/documents/tribal.py index e81a6361f..4fb107ba8 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tribal.py @@ -3,7 +3,7 @@ from django_elasticsearch_dsl import Document from django_elasticsearch_dsl.registries import registry from ..models.tribal import Tribal_TANF_T1, Tribal_TANF_T2, Tribal_TANF_T3, Tribal_TANF_T4, Tribal_TANF_T5 -from ..models.tribal import Tribal_TANF_T6 +from ..models.tribal import Tribal_TANF_T6, Tribal_TANF_T7 from .document_base import DocumentBase @registry.register_document @@ -303,3 +303,29 @@ class Django: 'NUM_OUTWEDLOCK_BIRTHS', 'NUM_CLOSED_CASES' ] + +@registry.register_document +class Tribal_TANF_T7DataSubmissionDocument(DocumentBase, Document): + """Elastic search model mapping for a parsed Tribal TANF T7 data file.""" + + class Index: + """ElasticSearch index generation settings.""" + + name = 'tribal_tanf_t7_submissions' + settings = { + 'number_of_shards': 1, + 'number_of_replicas': 0, + } + + class Django: + """Django model reference and field mapping.""" + + model = Tribal_TANF_T7 + fields = [ + "RecordType", + "CALENDAR_QUARTER", + "RPT_MONTH_YEAR", + "TDRS_SECTION_IND", + "STRATUM", + "FAMILIES_MONTH", + ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py b/tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py new file mode 100644 index 000000000..a37b323fa --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0025_tribal_tanf_t7.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.15 on 2023-11-29 19:49 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_files', '0012_datafile_s3_versioning_id'), + ('search_indexes', '0024_tribal_tanf_t6'), + ] + + operations = [ + migrations.CreateModel( + name='Tribal_TANF_T7', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('RecordType', models.CharField(max_length=156, null=True)), + ('CALENDAR_QUARTER', models.IntegerField(blank=True, null=True)), + ('RPT_MONTH_YEAR', models.IntegerField(null=True)), + ('TDRS_SECTION_IND', models.CharField(max_length=1, null=True)), + ('STRATUM', models.CharField(max_length=2, null=True)), + ('FAMILIES_MONTH', models.IntegerField(null=True)), + ('datafile', models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tribal_t7_parent', to='data_files.datafile')), + ], + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py b/tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py new file mode 100644 index 000000000..a5fb41692 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0026_parent_minor_child_rename.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-09-14 17:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0025_tribal_tanf_t7'), + ] + + operations = [ + migrations.RenameField( + model_name='tanf_t2', + old_name='PARENT_WITH_MINOR_CHILD', + new_name='PARENT_MINOR_CHILD', + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index f6ba10f29..204106406 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -113,7 +113,7 @@ class TANF_T2(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) MARITAL_STATUS = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.CharField(max_length=2, null=True, blank=False) - PARENT_WITH_MINOR_CHILD = models.IntegerField(null=True, blank=False) + PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/models/tribal.py b/tdrs-backend/tdpservice/search_indexes/models/tribal.py index 6c5d46955..9cdcf6a37 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tribal.py @@ -302,3 +302,31 @@ class Tribal_TANF_T6(models.Model): NUM_BIRTHS = models.IntegerField(null=True, blank=True) NUM_OUTWEDLOCK_BIRTHS = models.IntegerField(null=True, blank=True) NUM_CLOSED_CASES = models.IntegerField(null=True, blank=True) + +class Tribal_TANF_T7(models.Model): + """ + Parsed record representing a Tribal T7 data submission. + + Mapped to an elastic search index. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + datafile = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='tribal_t7_parent' + ) + + RecordType = models.CharField(max_length=156, null=True, blank=False) + CALENDAR_QUARTER = models.IntegerField(null=True, blank=True) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + TDRS_SECTION_IND = models.CharField( + max_length=1, + null=True, + blank=False + ) + STRATUM = models.CharField(max_length=2, null=True, blank=False) + FAMILIES_MONTH = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index 8ea107a35..dd66010a9 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -112,7 +112,7 @@ def test_can_create_and_index_tanf_t2_submission(test_datafile): submission.RECEIVE_SSI = 1 submission.MARITAL_STATUS = 1 submission.RELATIONSHIP_HOH = "01" - submission.PARENT_WITH_MINOR_CHILD = 1 + submission.PARENT_MINOR_CHILD = 1 submission.NEEDS_PREGNANT_WOMAN = 1 submission.EDUCATION_LEVEL = 1 submission.CITIZENSHIP_STATUS = 1 @@ -273,11 +273,11 @@ def test_can_create_and_index_tanf_t5_submission(test_datafile): submission.REC_FEDERAL_DISABILITY = 1 submission.REC_AID_TOTALLY_DISABLED = 1 submission.REC_AID_AGED_BLIND = 1 - submission.RECEIVE_SSI = 1 + submission.REC_SSI = 1 submission.MARITAL_STATUS = 1 submission.RELATIONSHIP_HOH = "01" - submission.PARENT_WITH_MINOR_CHILD = 1 - submission.NEEDS_PREGNANT_WOMAN = 1 + submission.PARENT_MINOR_CHILD = 1 + submission.NEEDS_OF_PREGNANT_WOMAN = 1 submission.EDUCATION_LEVEL = "1" submission.CITIZENSHIP_STATUS = 1 submission.COUNTABLE_MONTH_FED_TIME = "1" @@ -1022,3 +1022,29 @@ def test_can_create_and_index_tribal_tanf_t6_submission(test_datafile): response = search.execute() assert response.hits.total.value == 1 + +@pytest.mark.django_db +def test_can_create_and_index_tribal_tanf_t7_submission(test_datafile): + """Tribal TANF T7 submissions can be created and mapped.""" + record_num = fake.uuid4() + + submission = models.tribal.Tribal_TANF_T7() + submission.datafile = test_datafile + submission.RecordType = record_num + submission.CALENDAR_YEAR = 2020 + submission.CALENDAR_QUARTER = 1 + submission.TDRS_SECTION_IND = '1' + submission.STRATUM = '01' + submission.FAMILIES_MONTH = 47655 + + submission.save() + + assert submission.id is not None + + search = documents.tribal.Tribal_TANF_T7DataSubmissionDocument.search().query( + 'match', + RecordType=record_num + ) + response = search.execute() + + assert response.hits.total.value == 1 diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 2a948afc2..b6cb1ff96 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -153,6 +153,9 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'search_indexes.add_tribal_tanf_t6', 'search_indexes.view_tribal_tanf_t6', 'search_indexes.change_tribal_tanf_t6', + 'search_indexes.add_tribal_tanf_t7', + 'search_indexes.view_tribal_tanf_t7', + 'search_indexes.change_tribal_tanf_t7', } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions From d646eeff013b18069d4ab24c7d3b751cb5f4eeff Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 10 Jan 2024 08:59:13 -0700 Subject: [PATCH 24/25] 2722/circleci (#2739) * reduced workflows from 3 deploys to 1 * fix spacing of nested 'steps:' * repaired logic for workflow * trying to get right format for conditional logic in job steps * fixed incorrect variable * adding default target_env instead of emptry string * reverting * fixed spacing * removed old requirement and updated for current jobs. * use this branch to check develop deploys * use this branch to check develop deploys * use this branch to check develop deploys * use this branch to check develop deploys * use this branch to check develop deploys * use this branch to check develop deploys * revert * revert * revert * removed check in jobs * updates site only for develop branch * removed deploy develop from github actions * fixed spacing * remove this branch from CI tracking * removed 2722/circleci branch references now that testing is done --------- Co-authored-by: George Hudson Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/deployment/jobs.yml | 16 +-- .circleci/deployment/workflows.yml | 120 +++++++++--------- .github/workflows/deploy-develop-on-merge.yml | 52 -------- 3 files changed, 61 insertions(+), 127 deletions(-) delete mode 100644 .github/workflows/deploy-develop-on-merge.yml diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 9aa40dfa7..d6faf27b8 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -1,9 +1,9 @@ # jobs: deploy-dev: - executor: docker-executor parameters: target_env: type: string + executor: docker-executor working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: @@ -33,26 +33,16 @@ cf-username: CF_USERNAME_STAGING deploy-infrastructure-dev: - executor: terraform/default - working_directory: ~/tdp-deploy parameters: target_env: type: string - steps: - - deploy-infrastructure: - cf-app: << parameters.target_env >> - - deploy-infrastructure-staging: executor: terraform/default working_directory: ~/tdp-deploy steps: - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging + cf-app: << parameters.target_env >> - deploy-infrastructure-develop: + deploy-infrastructure-staging: executor: terraform/default working_directory: ~/tdp-deploy steps: diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 5689b83bc..8a4269c04 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -1,43 +1,51 @@ #workflows: - dev-deployment: + deployment: when: - << pipeline.parameters.run_dev_deployment >> + and: + - or: + - equal: [ master, << pipeline.git.branch >> ] + - equal: [ main, << pipeline.git.branch >> ] + - equal: [ develop, << pipeline.git.branch >> ] + - << pipeline.parameters.run_dev_deployment >> + - not: << pipeline.parameters.run_nightly_owasp_scan >> + jobs: + - deploy-project-updates-site: + filters: + branches: + only: + - develop - deploy-infrastructure-dev: target_env: << pipeline.parameters.target_env >> - - enable-versioning: - requires: - - deploy-infrastructure-dev filters: branches: ignore: - develop - main - master - - deploy-dev: - target_env: << pipeline.parameters.target_env >> - requires: - - deploy-infrastructure-dev - - staging-deployment: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.run_nightly_owasp_scan >> - jobs: - - deploy-project-updates-site: + - deploy-infrastructure-staging: filters: branches: only: - develop - - deploy-infrastructure-develop: + - main + - deploy-infrastructure-production: filters: branches: only: + - master + - enable-versioning: + requires: + - deploy-infrastructure-dev + filters: + branches: + ignore: - develop + - main + - master - enable-versioning: requires: - - deploy-infrastructure-develop + - deploy-infrastructure-staging target_env: develop cf-password: CF_PASSWORD_STAGING cf-username: CF_USERNAME_STAGING @@ -46,41 +54,52 @@ branches: only: - develop - - deploy-develop: + - enable-versioning: requires: - - deploy-infrastructure-develop + - deploy-infrastructure-staging + target_env: staging + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging filters: branches: only: - - develop - - test-deployment-e2e: + - main + - enable-versioning: requires: - - deploy-develop + - deploy-infrastructure-production + target_env: prod + cf-password: CF_PASSWORD_PROD + cf-username: CF_USERNAME_PROD + cf-space: tanf-prod filters: branches: only: - - develop - - make_erd: # from ../util folder + - master + - prod-deploy-clamav: + requires: + - deploy-infrastructure-production filters: branches: only: - - develop - - deploy-infrastructure-staging: + - master + - deploy-dev: + target_env: << pipeline.parameters.target_env >> + requires: + - deploy-infrastructure-dev filters: branches: - only: + ignore: + - develop - main - - enable-versioning: + - master + - deploy-develop: requires: - deploy-infrastructure-staging - target_env: staging - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging filters: branches: only: - - main + - develop - deploy-staging: requires: - deploy-infrastructure-staging @@ -88,29 +107,6 @@ branches: only: - main - - production-deployment: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.run_nightly_owasp_scan >> - jobs: - - deploy-infrastructure-production: - filters: - branches: - only: - - master - - enable-versioning: - requires: - - deploy-infrastructure-production - target_env: prod - cf-password: CF_PASSWORD_PROD - cf-username: CF_USERNAME_PROD - cf-space: tanf-prod - filters: - branches: - only: - - master - deploy-production: requires: - deploy-infrastructure-production @@ -118,16 +114,16 @@ branches: only: - master - - prod-deploy-clamav: + - test-deployment-e2e: requires: - - deploy-infrastructure-production + - deploy-develop filters: branches: only: - - master + - develop - make_erd: # from ../util folder filters: branches: only: + - develop - master - diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml deleted file mode 100644 index ee4eee057..000000000 --- a/.github/workflows/deploy-develop-on-merge.yml +++ /dev/null @@ -1,52 +0,0 @@ -########################################################################### -# GitHub Action Workflow -# On push to the develop branch (which should only be done through PR) -# in GitHub this action will trigger a deploy job within CircleCI for the -# deployment and e2e testing of the develop environment. -# -# Step 0: Checkout latest commit on develop -# -# Step 1: Makes a request to the V2 CircleCI API to initiate the project, -# which will filter based upon the branch to initiate the -# workflow/jobs listed here: -# staging-deployment:[ -# deploy-project-updates-site, -# deploy-infrastructure-develop, -# deploy-develop, -# test-deployment-e2e -# ] -# -# Leverages the open source GitHub Action: -# https://github.com/promiseofcake/circleci-trigger-action -########################################################################### -name: Deploy Develop on PR Merge -on: - push: - branches: - - develop - paths-ignore: - - 'docs/**' - - '**.md' - - '**.txt' - - '.gitattributes' - - '.gitignore' - - 'LICENSE' -jobs: - merge_deployment: - if: github.ref == 'refs/heads/develop' - runs-on: ubuntu-latest - name: Initiate deploy job in CircleCI - steps: - - uses: actions/checkout@v2 - - name: Circle CI Deployment Trigger - id: curl-circle-ci - uses: promiseofcake/circleci-trigger-action@v1 - with: - user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} - project-slug: ${{ github.repository }} - branch: ${{ github.ref_name }} - payload: '{ - "develop_branch_deploy": true, - "target_env": "develop", - "triggered": true - }' From ea121aed3fd556cdca84b3d1999d0a63c1f29f3f Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:08:48 -0700 Subject: [PATCH 25/25] Elastic ADR Update (#2791) * - Added authentication risks * - Remove pricing info --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../Architecture-Decision-Record/017-elastisearch.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/017-elastisearch.md b/docs/Technical-Documentation/Architecture-Decision-Record/017-elastisearch.md index eaca91d52..67c8bcc06 100644 --- a/docs/Technical-Documentation/Architecture-Decision-Record/017-elastisearch.md +++ b/docs/Technical-Documentation/Architecture-Decision-Record/017-elastisearch.md @@ -35,7 +35,11 @@ We will use an Elastisearch and Kibana stack for their modern feature set and sc - This is mostly mitigated through a SQL Workbench provided in Kibana where you can use regular SQL syntax to query records. * More infrastructure to manage. * This is mostly mitigated due to using a Cloud.gov managed service for ES and Terraform, this greatly simplifies scaling the cluster and abstracts away a lot of the difficult cluster management tasks we would have to do if we didn't use a managed service. - * Additional overhead to run a proxy application to control access to ES + Kibana + * Additional overhead to run a proxy application to control access to ES + Kibana+ + * Security & Authentication + * Cloud.gov ES service is a wrapper around AWS OpenSearch/ES. AWS ES does not support Xpack (Elastic/Kibana native security features) because it was forked off of Elastic 7.10.2 which did not support Xpack at that time. This implies that our current Xpack configuration we are using with our local Elastic 7.17.6/Kibana 7.17.10 deployments ([implemented here](https://github.com/raft-tech/TANF-app/pull/2775)) will not be applicable to our deployed environments. To get around this issue, AWS suggests introducing a [proxy EC2 node](https://aws.amazon.com/blogs/security/how-to-control-access-to-your-amazon-elasticsearch-service-domain/) to implement the same type of features that Xpack natively provides by way of IAM policies and Signature Version 4 request signing. However, Cloud.gov does not allow access to the underlying AWS resources it is wrapping, thus making this workaround impossible. + * Another option to workaround AWS ES and Cloud.gov would be to deploy and manage our own ES cluster to Cloud.gov in each space. This also introduces large blocks in and of itself. To deploy/manage our own cluster would take at least one dedicated Elastic SME to ensure uptime, availability, updates, security, etc... This would also imply that we would need to purchase Elastic Stack self-managed licenses from Elastic. To acquire the minimum feature set we need to have robust security and authentication integration with TDP, we would need to procure platinum tier licenses. Elastic requires a minimum of three licenses to be purchased. We at a minimum, would need three nodes per environment (9 licenses total) to have a functioning Elastic cluster. However, the cost of these licenses and the cost of at least one person to manage the cluster(s)/licenses makes this an infeasible option. + * With these things considered, the best security/authentication we can provide at this time (12/22/2023) is by blocking all external incoming traffic to our Elastic and Kibana servers, and by leveraging the view based auth [implemented here](https://github.com/raft-tech/TANF-app/pull/2775), which prevents non admin and non HHS AMS authenticated users from navigating to Kibana via the frontend. We will not be able to use any Xpack features (RBAC, Realms, P2P encryption, etc...) used in that PR in our deployed environments. ## Notes