Merge branch 'elastic-bulk-doc-creation' of https://github.com/raft-t…

…ech/TANF-app into 1858-kibana-ofa-access

tdrs-backend/.env.example

@@ -2,7 +2,7 @@
 # Copy this file to `.env` and replace variables as needed
 # 
 
-#REACT_APP_DEVAUTH=1
+#REACT_APP_DEVAUTH=dev@test.com
 
 # ##
 # Required environment variables

tdrs-backend/docker-compose.local.yml

@@ -36,6 +36,7 @@ services:
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
       - logger.discovery.level=debug
     ports:
       - 9200:9200

tdrs-backend/tdpservice/parsers/parse.py

@@ -60,22 +60,24 @@ def bulk_create_records(unsaved_records, line_number, header_count, batch_size=1
     if (line_number % batch_size == 0 and header_count > 0) or flush:
         logger.debug("Bulk creating records.")
         try:
-            num_records_created = 0
-            num_expected_records = 0
-            num_documents_created = 0
+            num_db_records_created = 0
+            num_expected_db_records = 0
+            num_elastic_records_created = 0
             for document, records in unsaved_records.items():
-                num_expected_records += len(records)
+                num_expected_db_records += len(records)
                 created_objs = document.Django.model.objects.bulk_create(records)
-                num_documents_created += document.update(created_objs)[0]
-                num_records_created += len(created_objs)
-            if num_records_created != num_expected_records:
-                logger.error(f"Bulk Django record creation only created {num_records_created}/{num_expected_records}!")
-            elif num_documents_created != num_expected_records:
-                logger.error(f"Bulk Elastic document creation only created {num_documents_created}/" +
-                             f"{num_expected_records}!")
+                num_elastic_records_created += document.update(created_objs)[0]
+                num_db_records_created += len(created_objs)
+            if num_db_records_created != num_expected_db_records:
+                logger.error(f"Bulk Django record creation only created {num_db_records_created}/" +
+                             f"{num_expected_db_records}!")
+            elif num_elastic_records_created != num_expected_db_records:
+                logger.error(f"Bulk Elastic document creation only created {num_elastic_records_created}/" +
+                             f"{num_expected_db_records}!")
             else:
-                logger.info(f"Created {num_records_created}/{num_expected_records} records.")
-            return num_records_created == num_expected_records and num_documents_created == num_expected_records, {}
+                logger.info(f"Created {num_db_records_created}/{num_expected_db_records} records.")
+            return num_db_records_created == num_expected_db_records and \
+                num_elastic_records_created == num_expected_db_records, {}
         except DatabaseError as e:
             logger.error(f"Encountered error while creating datafile records: {e}")
             return False, unsaved_records

tdrs-backend/tdpservice/settings/common.py

@@ -293,7 +293,6 @@ class Common(Configuration):
         "DEFAULT_RENDERER_CLASSES": DEFAULT_RENDERER_CLASSES,
         "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"],
         "DEFAULT_AUTHENTICATION_CLASSES": (
-            "tdpservice.users.authentication.DevAuthentication",
             "tdpservice.users.authentication.CustomAuthentication",
             "rest_framework.authentication.SessionAuthentication",
             "rest_framework.authentication.TokenAuthentication",
@@ -476,4 +475,5 @@ class Common(Configuration):
     KIBANA_BASE_URL = os.getenv('KIBANA_BASE_URL', 'http://localhost:5601')
     BYPASS_KIBANA_AUTH = strtobool(os.getenv("BYPASS_KIBANA_AUTH", "no"))
 
-    CYPRESS_TOKEN = os.getenv('CYPRESS_TOKEN', None)
+    REACT_APP_DEVAUTH = os.getenv('REACT_APP_DEVAUTH', None)
+    CYPRESS_TOKEN = 'local-cypress-token' if REACT_APP_DEVAUTH else os.getenv('CYPRESS_TOKEN', None)

tdrs-backend/tdpservice/users/api/authorization_check.py

@@ -23,6 +23,8 @@ class AuthorizationCheck(APIView):
 
     def get(self, request, *args, **kwargs):
         """Handle get request and verify user is authorized."""
+        logger.debug(f"{self.__class__.__name__}: {request} {args} {kwargs}")
+
         user = request.user
         serializer = UserProfileSerializer(user)
 

tdrs-backend/tdpservice/users/api/login.py

@@ -395,6 +395,7 @@ class CypressLoginDotGovAuthenticationOverride(TokenAuthorizationOIDC):
 
     def post(self, request):
         """Create a session for the specified user, if they exist."""
+        logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}")
         username = request.data.get('username', None)
         token = request.data.get('token', None)
 

tdrs-backend/tdpservice/users/authentication.py

@@ -3,54 +3,49 @@
 from django.contrib.auth import get_user_model
 
 from rest_framework.authentication import BaseAuthentication
+from rest_framework.request import Request
 import logging
-import os
 logger = logging.getLogger(__name__)
 
-class DevAuthentication(BaseAuthentication):
-    """Define authentication and get user functions for local/developer authentication."""
-
-    def authenticate(self, request):
-        """Authenticate user."""
-        if not os.environ.get('REACT_APP_DEVAUTH'):
-            return None
-        logging.debug(f"{self.__class__.__name__}: {request} ; {request.data}")
-        requser = request.data.get("user")
-        reqname = requser if requser and requser != "undefined" else "dev@test.com"
-        User = get_user_model()
-        authuser = User.objects.get(username=reqname)
-        if authuser and requser == "undefined":
-            request.data["user"] = authuser.id
-        return (User.objects.get(username=reqname), True)
-
-
 class CustomAuthentication(BaseAuthentication):
     """Define authentication and get user functions for custom authentication."""
 
     @staticmethod
-    def authenticate(username=None, login_gov_uuid=None, hhs_id=None):
+    def authenticate(request=None, username=None, login_gov_uuid=None, hhs_id=None):
         """Authenticate user with the request and username."""
+        # HACK: This method currently needs to support two unrelated workflows.
+        # References:
+        #   tdpservice/users/api/login.py:TokenAuthorizationOIDC.handleUser
+        #   https://www.django-rest-framework.org/api-guide/authentication
+        if type(request) == Request:
+            username = request.data.get('username')
+            logging.debug(f"CustomAuthentication::authenticate: {request} {request.data} "
+                          f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}")
+        else:
+            logging.debug(f"CustomAuthentication::authenticate: {username} "
+                          f"login_gov_id={login_gov_uuid} hhs_id={hhs_id}")
         User = get_user_model()
-        logging.debug("CustomAuthentication::authenticate:hhs_id {}".format(hhs_id))
-        logging.debug("CustomAuthentication::authenticate:login_gov_uuid {}".format(login_gov_uuid))
-        logging.debug("CustomAuthentication::authenticate:username {}".format(username))
         try:
             if hhs_id:
                 try:
-                    return User.objects.get(hhs_id=hhs_id)
+                    user_obj = User.objects.get(hhs_id=hhs_id)
                 except User.DoesNotExist:
                     # If below line also fails with User.DNE, will bubble up and return None
                     user = User.objects.filter(username=username)
                     user.update(hhs_id=hhs_id)
                     logging.debug("Updated user {} with hhs_id {}.".format(username, hhs_id))
-                return User.objects.get(hhs_id=hhs_id)
+                    user_obj = User.objects.get(hhs_id=hhs_id)
 
             elif login_gov_uuid:
-                return User.objects.get(login_gov_uuid=login_gov_uuid)
+                user_obj = User.objects.get(login_gov_uuid=login_gov_uuid)
             else:
-                return User.objects.get(username=username)
+                user_obj = User.objects.get(username=username)
         except User.DoesNotExist:
-            return None
+            user_obj = None
+        logging.debug(f"CustomAuthentication::authenticate found user: {user_obj}")
+        if type(request) == Request:
+            return (user_obj, None) if user_obj else None
+        return user_obj
 
     @staticmethod
     def get_user(user_id):

tdrs-backend/tdpservice/users/management/commands/generate_dev_user.py

@@ -28,6 +28,7 @@ def handle(self, *args, **options):
                                        is_staff=True,
                                        first_name=first,
                                        last_name=last,
+                                       stt_id=31,
                                        account_approval_status="Approved")
             user.groups.add(group)
             print(f"Created {vars(user)}")

tdrs-frontend/.env

@@ -4,6 +4,8 @@
 # WARNING: This file is checked in to source control, do NOT store any secrets in this file
 #
 
+#REACT_APP_DEVAUTH=dev@test.com
+
 # The hostname behind the tdrs-backend Django app
 REACT_APP_BACKEND_HOST=http://127.0.0.1:8080
 

tdrs-frontend/.env.development

@@ -3,8 +3,6 @@
 # This file is loaded when running `npm start`
 #
 
-#REACT_APP_DEVAUTH=1
-
 # The hostname behind the tdrs-backend Django app
 REACT_APP_BACKEND_HOST=http://localhost:3000
 

tdrs-frontend/docker-compose.yml

@@ -28,6 +28,7 @@ services:
       - NGINX_FRONTEND=tdp-frontend
       - BACK_END=web
       - LOCAL_DEV=true
+      - REACT_APP_DEVAUTH=${REACT_APP_DEVAUTH}
     command: >
       /bin/sh -c
       "echo 'starting nginx' &&

tdrs-frontend/src/actions/auth.js

@@ -40,9 +40,6 @@ export const SET_MOCK_LOGIN_STATE = 'SET_MOCK_LOGIN_STATE'
  */
 
 export const fetchAuth = () => async (dispatch) => {
-  if (process.env.REACT_APP_DEVAUTH) {
-    return 0
-  }
   dispatch({ type: FETCH_AUTH })
   try {
     const URL = `${process.env.REACT_APP_BACKEND_URL}/auth_check`

tdrs-frontend/src/configureStore.js

@@ -4,7 +4,6 @@ import { createBrowserHistory } from 'history'
 import thunkMiddleware from 'redux-thunk'
 import loggerMiddleware from './middleware/logger'
 import createRootReducer from './reducers'
-import { permissions } from './components/Header/developer_permissions'
 
 export const history = createBrowserHistory()
 
@@ -14,29 +13,9 @@ export const history = createBrowserHistory()
 export default function configureStore(preloadedState) {
   const middlewares = [thunkMiddleware, loggerMiddleware]
   const composedEnhancers = composeWithDevTools(applyMiddleware(...middlewares))
-  const devState = {
-    router: { location: { pathname: '/profile' } },
-    auth: {
-      user: {
-        email: 'dev@test.com',
-        first_name: 'Jon',
-        last_name: 'Tester',
-        roles: [{ id: 1, name: 'Developer', permissions }],
-        access_request: true,
-        account_approval_status: 'Approved',
-        stt: {
-          id: 31,
-          type: 'state',
-          code: 'NJ',
-          name: 'New Jersey',
-        },
-      },
-      authenticated: true,
-    },
-  }
   const store = createStore(
     createRootReducer(history),
-    process.env.REACT_APP_DEVAUTH ? devState : preloadedState,
+    preloadedState,
     composedEnhancers
   )
   return store

tdrs-frontend/src/index.js

@@ -24,8 +24,28 @@ axios.defaults.xsrfCookieName = 'csrftoken'
 axios.defaults.xsrfHeaderName = 'X-CSRFToken'
 axios.defaults.withCredentials = true
 
+function devLogin(devEmail) {
+  const BACKEND_URL = process.env.REACT_APP_BACKEND_URL
+  axios
+    .post(`${BACKEND_URL}/login/cypress`, {
+      username: devEmail,
+      token: 'local-cypress-token',
+    })
+    .then(function (response) {
+      console.log(response)
+    })
+    .catch(function (error) {
+      console.log(error)
+    })
+  store.dispatch({ type: 'SET_AUTH', payload: { devEmail } })
+  console.log(`dispatched SET_AUTH(${devEmail})`)
+}
+
 // call auth_check
 const store = configureStore()
+if (process.env.REACT_APP_DEVAUTH) {
+  devLogin(process.env.REACT_APP_DEVAUTH)
+}
 store.dispatch(fetchAuth())
 
 // if (window.location.href.match(/https:\/\/.*\.app\.cloud\.gov/)) {