diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index 43adb60e3..f97354e99 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -2,12 +2,7 @@ deploy-cloud-dot-gov: parameters: environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend + default: raft type: string cf-password: default: CF_PASSWORD_DEV @@ -21,9 +16,6 @@ cf-username: default: CF_USERNAME_DEV type: env_var_name - frontend-appname: - default: tdp-frontend - type: string steps: - checkout - sudo-check @@ -34,14 +26,11 @@ cf-space: <> cf-username: <> - deploy-backend: - backend-appname: <> - frontend-appname: <> cf-space: <> - - deploy-frontend: environment: <> - backend-appname: <> - frontend-appname: <> + - deploy-frontend: cf-space: <> + environment: <> clamav-cloud-dot-gov: parameters: @@ -73,26 +62,19 @@ deploy-backend: parameters: - backend-appname: - default: tdp-backend - type: string - frontend-appname: - default: tdp-frontend - type: string cf-space: default: tanf-dev type: string + environment: + default: raft + type: string steps: - - get-app-deploy-strategy: - appname: <> - run: name: Deploy backend application command: | bash ./scripts/deploy-backend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> \ - <> + <> \ + <> deploy-clamav: steps: @@ -104,58 +86,25 @@ deploy-frontend: parameters: - environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend - type: string - frontend-appname: - default: tdp-frontend - type: string -# So the frontend knows what space its in for the banner. -# I am unclear if the domain is a reliable metric to make this function -# It seems like it might not be working cf-space: + type: string default: dev + environment: type: string + default: raft steps: - install-nodejs: node-version: "16.13" - disable-npm-audit - install-nodejs-packages: app-dir: tdrs-frontend - - get-app-deploy-strategy: - appname: <> - run: name: Deploy frontend application command: | bash ./scripts/deploy-frontend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> \ <> \ <> - get-app-deploy-strategy: - parameters: - appname: - type: string - steps: - - run: - name: Determine deploy strategy - command: | - # NOTE: The || true is a no-op included to suppress exit codes which - # would cause the step to exit early due to use of pipefail - APP_GUID=$(cf app <> --guid || true) - if [ "$APP_GUID" == "FAILED" ]; then - echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV - else - echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV - fi - deploy-infrastructure: parameters: tf-path: diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index d6faf27b8..ecf3de872 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -7,16 +7,14 @@ working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - backend-appname: tdp-backend-<< parameters.target_env >> - frontend-appname: tdp-frontend-<< parameters.target_env >> + environment: << parameters.target_env >> deploy-staging: executor: docker-executor working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - backend-appname: tdp-backend-staging - frontend-appname: tdp-frontend-staging + environment: staging cf-password: CF_PASSWORD_STAGING cf-space: tanf-staging cf-username: CF_USERNAME_STAGING @@ -26,8 +24,7 @@ working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - backend-appname: tdp-backend-develop - frontend-appname: tdp-frontend-develop + environment: develop cf-password: CF_PASSWORD_STAGING cf-space: tanf-staging cf-username: CF_USERNAME_STAGING @@ -135,9 +132,7 @@ working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - environment: production - backend-appname: tdp-backend-prod - frontend-appname: tdp-frontend-prod + environment: prod cf-password: CF_PASSWORD_PROD cf-space: tanf-prod cf-username: CF_USERNAME_PROD diff --git a/docs/Technical-Documentation/redis-and-celery.md b/docs/Technical-Documentation/redis-and-celery.md new file mode 100644 index 000000000..5c09a65ca --- /dev/null +++ b/docs/Technical-Documentation/redis-and-celery.md @@ -0,0 +1,11 @@ +# Redis Service and Celery Instance + +We use a CloudFoundry Redis service and a separate instance to run celery to run background processes like parsing data-file documents that have been submitted and have passed the [ClamAV scan](./clamav.md). + +## Redis Deployment + +As part of the move towards each environment being self-contained, one redis service is created per environment, deployed through the [CircleCI pipeline](./circle-ci.md), defined using [terraform](../../terraform/README.md). + +## Celery Deployment + +Celery is deployed at the same time as the backend through the [CircleCI pipeline](./circle-ci.md), with the details configured in the [celery manifest](../../tdrs-backend/manifest.celery.yml) and the [deploy-backend script](../../scripts/deploy-backend.sh) \ No newline at end of file diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 6e46fe93a..cb2270930 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -4,71 +4,71 @@ # Global Variable Decls ############################## -# The deployment strategy you wish to employ ( rolling update or setting up a new environment) -DEPLOY_STRATEGY=${1} +CF_SPACE=${1} +ENV=${2} -#The application name defined via the manifest yml for the frontend -CGAPPNAME_FRONTEND=${2} -CGAPPNAME_BACKEND=${3} -CF_SPACE=${4} +DEPLOY_STRATEGY=${3-tbd} +CELERY_DEPLOY_STRATEGY=${4-tbd} + +CGAPPNAME_FRONTEND="tdp-frontend-${ENV}" +CGAPPNAME_BACKEND="tdp-backend-${ENV}" +CGAPPNAME_CELERY="tdp-celery-${ENV}" strip() { - # Usage: strip "string" "pattern" - printf '%s\n' "${1##$2}" + # Usage: strip "string" "pattern" + printf '%s\n' "${1##$2}" } -# The cloud.gov space defined via environment variable (e.g., "tanf-dev", "tanf-staging") -env=$(strip $CF_SPACE "tanf-") -backend_app_name=$(echo $CGAPPNAME_BACKEND | cut -d"-" -f3) +# The cloud.gov space defined via CF_SPACE environment variable (e.g., "tanf-dev", "tanf-staging") +space=$(strip $CF_SPACE "tanf-") echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY" echo BACKEND_HOST: "$CGAPPNAME_BACKEND" +echo CELERY_HOST: "$CGAPPNAME_CELERY" echo CF_SPACE: "$CF_SPACE" -echo env: "$env" -echo backend_app_name: "$backend_app_name" +echo space: "$space" +echo environment: "$ENV" ############################## # Function Decls ############################## -set_cf_envs() -{ +set_cf_envs() { var_list=( - "ACFTITAN_HOST" - "ACFTITAN_KEY" - "ACFTITAN_USERNAME" - "AMS_CLIENT_ID" - "AMS_CLIENT_SECRET" - "AMS_CONFIGURATION_ENDPOINT" - "BASE_URL" - "CLAMAV_NEEDED" - "CYPRESS_TOKEN" - "DJANGO_CONFIGURATION" - "DJANGO_DEBUG" - "DJANGO_SECRET_KEY" - "DJANGO_SETTINGS_MODULE" - "DJANGO_SU_NAME" - "FRONTEND_BASE_URL" - "LOGGING_LEVEL" - "REDIS_URI" - "JWT_KEY" - "STAGING_JWT_KEY" + "ACFTITAN_HOST" + "ACFTITAN_KEY" + "ACFTITAN_USERNAME" + "AMS_CLIENT_ID" + "AMS_CLIENT_SECRET" + "AMS_CONFIGURATION_ENDPOINT" + "BASE_URL" + "CLAMAV_NEEDED" + "CYPRESS_TOKEN" + "DJANGO_CONFIGURATION" + "DJANGO_DEBUG" + "DJANGO_SECRET_KEY" + "DJANGO_SETTINGS_MODULE" + "DJANGO_SU_NAME" + "FRONTEND_BASE_URL" + "LOGGING_LEVEL" + "JWT_KEY" + "STAGING_JWT_KEY" ) - echo "Setting environment variables for $CGAPPNAME_BACKEND" + echo "Setting environment variables for $1" for var_name in ${var_list[@]}; do # Intentionally unsetting variable if empty if [[ -z "${!var_name}" ]]; then - echo "WARNING: Empty value for $var_name. It will now be unset." - cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" - $cf_cmd - continue - elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then - sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') - cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + echo "WARNING: Empty value for $var_name. It will now be unset." + cf_cmd="cf unset-env $1 $var_name ${!var_name}" + $cf_cmd + continue + elif [[ ($var_name =~ 'STAGING_') && ($CF_SPACE = 'tanf-staging') ]]; then + sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') + cf_cmd="cf set-env $1 $sed_var_name ${!var_name}" else - cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" + cf_cmd="cf set-env $1 $var_name ${!var_name}" fi echo "Setting var : $var_name" @@ -78,80 +78,109 @@ set_cf_envs() } # Helper method to generate JWT cert and keys for new environment -generate_jwt_cert() -{ - echo "regenerating JWT cert/key" - yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256 - cf set-env "$CGAPPNAME_BACKEND" JWT_CERT "$(cat cert.pem)" - cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)" +generate_jwt_cert() { + echo "regenerating JWT cert/key" + yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256 + cf set-env "$CGAPPNAME_BACKEND" JWT_CERT "$(cat cert.pem)" + cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)" } -update_backend() -{ - cd tdrs-backend || exit - cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" - - if [ "$CF_SPACE" = "tanf-prod" ]; then - cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" - else - # Add environment varilables for clamav - cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan" - fi +update_backend() { + cd tdrs-backend || exit + cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" + cf unset-env "$CGAPPNAME_BACKEND" "CGAPPNAME_BACKEND" + cf unset-env "$CGAPPNAME_CELERY" "CGAPPNAME_BACKEND" + # Let Celery know backend app name for s3 file searching + cf set-env "$CGAPPNAME_BACKEND" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" + cf set-env "$CGAPPNAME_CELERY" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" + + if [[ $CF_SPACE == 'tanf-prod' ]]; then + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" + else + # Add environment varilables for clamav + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$space.apps.internal:9000/scan" + fi - if [ "$1" = "rolling" ] ; then - set_cf_envs - # Do a zero downtime deploy. This requires enough memory for - # two apps to exist in the org/space at one time. - cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1 + if [[ $1 == 'rolling' ]] ; then + set_cf_envs $CGAPPNAME_BACKEND + # Do a zero downtime deploy. This requires enough memory for + # two apps to exist in the org/space at one time. + cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1 + else + cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 + # set up JWT key if needed + if cf e "$CGAPPNAME_BACKEND" | grep -q JWT_KEY ; then + echo jwt cert already created else - cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 - # set up JWT key if needed - if cf e "$CGAPPNAME_BACKEND" | grep -q JWT_KEY ; then - echo jwt cert already created - else - generate_jwt_cert - fi + generate_jwt_cert fi + fi - set_cf_envs - - cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" + if [[ $2 == 'rolling' ]] ; then + set_cf_envs $CGAPPNAME_CELERY + # Do a zero downtime deploy. This requires enough memory for + # two apps to exist in the org/space at one time. + cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 --strategy rolling || exit 1 + else + cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 + fi - # Add network policy to allow frontend to access backend - cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 - - if [ "$CF_SPACE" = "tanf-prod" ]; then - # Add network policy to allow backend to access tanf-prod services - cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 - else - cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$env --protocol tcp --port 9000 - fi + if [[ ! "$CF_SPACE" == "tanf-prod" ]]; then + #allow dev envs to monitor celery through flower/prometheus + cf map-route "$CGAPPNAME_CELERY" app.cloud.gov --hostname "${CGAPPNAME_CELERY}" + fi + + set_cf_envs $CGAPPNAME_BACKEND + set_cf_envs $CGAPPNAME_CELERY + # Let Celery know backend app name for s3 file searching + cf set-env "$CGAPPNAME_BACKEND" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" + cf set-env "$CGAPPNAME_CELERY" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" + + cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" + + # Add network policy to allow frontend to access backend + cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 + + if [[ $CF_SPACE == 'tanf-prod' ]]; then + # Add network policy to allow backend to access tanf-prod services + cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 + else + cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$space --protocol tcp --port 9000 + fi - cd .. } bind_backend_to_services() { - echo "Binding services to app: $CGAPPNAME_BACKEND" + echo "Binding services to app: $CGAPPNAME_BACKEND" - if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then - # TODO: this is technical debt, we should either make staging mimic tanf-dev - # or make unique services for all apps but we have a services limit - # Introducing technical debt for release 3.0.0 specifically. - env="develop" - fi + if [[ $CGAPPNAME_BACKEND = 'tdp-backend-develop' ]]; then + # TODO: this is technical debt, we should either make staging mimic tanf-dev + # or make unique services for all apps but we have a services limit + # Introducing technical debt for release 3.0.0 specifically. + space="develop" + fi - cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${env}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}" - - # The below command is different because they cannot be shared like the 3 above services - cf bind-service "$CGAPPNAME_BACKEND" "es-${backend_app_name}" - - set_cf_envs + cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${space}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${space}" - echo "Restarting app: $CGAPPNAME_BACKEND" - cf restage "$CGAPPNAME_BACKEND" + cf bind-service "$CGAPPNAME_CELERY" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-datafiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-db-${space}" + # bind to redis + cf bind-service "$CGAPPNAME_BACKEND" "tdp-redis-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-redis-${space}" + # bind to elastic-search + cf bind-service "$CGAPPNAME_BACKEND" "es-${ENV}" + cf bind-service "$CGAPPNAME_CELERY" "es-${ENV}" + + set_cf_envs $CGAPPNAME_BACKEND + set_cf_envs $CGAPPNAME_CELERY + + echo "Restarting apps: $CGAPPNAME_BACKEND and $CGAPPNAME_CELERY" + cf restage "$CGAPPNAME_BACKEND" + cf restage "$CGAPPNAME_CELERY" } ############################## @@ -161,13 +190,13 @@ bind_backend_to_services() { # Determine the appropriate BASE_URL for the deployed instance based on the # provided Cloud.gov App Name DEFAULT_ROUTE="https://$CGAPPNAME_FRONTEND.app.cloud.gov" -if [ -n "$BASE_URL" ]; then +if [[ -n $BASE_URL ]]; then # Use Shell Parameter Expansion to replace localhost in the URL BASE_URL="${BASE_URL//http:\/\/localhost:8080/$DEFAULT_ROUTE}" -elif [ "$CF_SPACE" = "tanf-prod" ]; then +elif [[ $CF_SPACE == 'tanf-prod' ]]; then # Keep the base url set explicitly for production. BASE_URL="https://tanfdata.acf.hhs.gov/v1" -elif [ "$CF_SPACE" = "tanf-staging" ]; then +elif [[ $CF_SPACE == 'tanf-staging' ]]; then # use .acf.hss.gov domain for develop and staging. BASE_URL="https://$CGAPPNAME_FRONTEND.acf.hhs.gov/v1" else @@ -176,12 +205,12 @@ else fi DEFAULT_FRONTEND_ROUTE="${DEFAULT_ROUTE//backend/frontend}" -if [ -n "$FRONTEND_BASE_URL" ]; then +if [[ -n $FRONTEND_BASE_URL ]]; then FRONTEND_BASE_URL="${FRONTEND_BASE_URL//http:\/\/localhost:3000/$DEFAULT_FRONTEND_ROUTE}" -elif [ "$CF_SPACE" = "tanf-prod" ]; then +elif [[ $CF_SPACE == 'tanf-prod' ]]; then # Keep the base url set explicitly for production. - FRONTEND_BASE_URL="https://tanfdata.acf.hhs.gov" -elif [ "$CF_SPACE" = "tanf-staging" ]; then + FRONTEND_BASE_URL='https://tanfdata.acf.hhs.gov' +elif [[ $CF_SPACE == 'tanf-staging' ]]; then # use .acf.hss.gov domain for develop and staging. FRONTEND_BASE_URL="https://$CGAPPNAME_FRONTEND.acf.hhs.gov" else @@ -194,9 +223,9 @@ DJANGO_SECRET_KEY=$(python3 -c "from secrets import token_urlsafe; print(token_u # Dynamically set DJANGO_CONFIGURATION based on Cloud.gov Space DJANGO_SETTINGS_MODULE="tdpservice.settings.cloudgov" -if [ "$CF_SPACE" = "tanf-prod" ]; then +if [[ $CF_SPACE == 'tanf-prod' ]]; then DJANGO_CONFIGURATION="Production" -elif [ "$CF_SPACE" = "tanf-staging" ]; then +elif [[ $CF_SPACE == 'tanf-staging' ]]; then DJANGO_CONFIGURATION="Staging" else DJANGO_CONFIGURATION="Development" @@ -204,27 +233,54 @@ else CYPRESS_TOKEN=$CYPRESS_TOKEN fi -if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then - # Perform a rolling update for the backend and frontend deployments if - # specified, otherwise perform a normal deployment - update_backend 'rolling' -elif [ "$DEPLOY_STRATEGY" = "bind" ] ; then - # Bind the services the application depends on and restage the app. - bind_backend_to_services -elif [ "$DEPLOY_STRATEGY" = "initial" ]; then - # There is no app with this name, and the services need to be bound to it - # for it to work. the app will fail to start once, have the services bind, - # and then get restaged. - update_backend - bind_backend_to_services -elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then - # You want to redeploy the instance under the same name - # Delete the existing app (with out deleting the services) - # and perform the initial deployment strategy. - cf delete "$CGAPPNAME_BACKEND" -r -f - update_backend - bind_backend_to_services +APP_GUID=$(cf app $CGAPPNAME_BACKEND --guid || true) +CELERY_GUID=$(cf app $CGAPPNAME_CELERY --guid || true) + +if [[ $DEPLOY_STRATEGY == 'tbd' ]]; then + if [[ $APP_GUID == 'FAILED' ]]; then + DEPLOY_STRATEGY='initial' + else + DEPLOY_STRATEGY='rolling' + fi + echo "Setting backend deployment strategy: ${DEPLOY_STRATEGY}" +else + echo "Using given backend deployment strategy: ${DEPLOY_STRATEGY}" +fi + +if [[ $CELERY_DEPLOY_STRATEGY == 'tbd' ]]; then + if [[ $CELERY_GUID == 'FAILED' ]]; then + CELERY_DEPLOY_STRATEGY='initial' + else + CELERY_DEPLOY_STRATEGY='rolling' + fi + echo "Setting celery deployment strategy: ${CELERY_DEPLOY_STRATEGY}" +else + echo "Using given celery deployment strategy: ${CELERY_DEPLOY_STRATEGY}" +fi + +if [[ $DEPLOY_STRATEGY == 'rebuild' ]]; then + # You want to redeploy the instance under the same name + # Delete the existing app (with out deleting the services) + # and perform the initial deployment strategy. + cf delete "$CGAPPNAME_BACKEND" -r -f + cf delete "$CGAPPNAME_CELERY" -r -f +fi + +if [[ $DEPLOY_STRATEGY == 'bind' ]]; then + # Bind the services the application depends on and restage the app. + bind_backend_to_services +else + update_backend $DEPLOY_STRATEGY $CELERY_DEPLOY_STRATEGY +fi + +if [[ $DEPLOY_STRATEGY == 'initial' ]]; then + bind_backend_to_services +elif [[ $DEPLOY_STRATEGY == 'rebuild' ]]; then + bind_backend_to_services +elif [[ $CELERY_DEPLOY_STRATEGY == 'initial' ]]; then + bind_backend_to_services +elif [[ $CELERY_DEPLOY_STRATEGY == 'rebuild' ]]; then + bind_backend_to_services else - # No changes to deployment config, just deploy the changes and restart - update_backend + echo "no need to rebind to services" fi diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh index 96af218f2..67bd83954 100755 --- a/scripts/deploy-frontend.sh +++ b/scripts/deploy-frontend.sh @@ -2,20 +2,24 @@ # source deploy-util.sh -# The deployment strategy you wish to employ ( rolling update or setting up a new environment) -DEPLOY_STRATEGY=${1} +CF_SPACE=${1} +ENV=${2} -#The application name defined via the manifest yml for the frontend -CGHOSTNAME_FRONTEND=${2} -CGHOSTNAME_BACKEND=${3} -CF_SPACE=${4} -ENVIRONMENT=${5} +DEPLOY_STRATEGY=${3-'tbd'} -update_frontend() -{ +CGHOSTNAME_FRONTEND="tdp-frontend-${ENV}" +CGHOSTNAME_BACKEND="tdp-backend-${ENV}" + +[[ $ENV = "prod" ]] && BUILD_ENV="production" || BUILD_ENV="development" + +update_frontend() { echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY" echo FRONTEND_HOST: "$CGHOSTNAME_FRONTEND" echo BACKEND_HOST: "$CGHOSTNAME_BACKEND" + echo CF_SPACE: "$CF_SPACE" + echo ENVIRONMENT: "$ENV" + echo BUILD_ENV: "$BUILD_ENV" + cd tdrs-frontend || exit if [ "$CF_SPACE" = "tanf-prod" ]; then @@ -45,7 +49,7 @@ update_frontend() cf set-env "$CGHOSTNAME_FRONTEND" BACKEND_HOST "$CGHOSTNAME_BACKEND" - npm run build:$ENVIRONMENT + npm run build:$BUILD_ENV unlink .env.production mkdir deployment @@ -81,11 +85,15 @@ update_frontend() cd ../.. rm -r tdrs-frontend/deployment } +# NOTE: The || true is a no-op included to suppress exit codes which +# would cause the step to exit early due to use of pipefail -# perform a rolling update for the backend and frontend deployments if -# specified, otherwise perform a normal deployment -if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then - update_frontend 'rolling' -else +APP_GUID=$(cf app $CGHOSTNAME_FRONTEND --guid || true) + +if [ $APP_GUID == 'FAILED' ] || [ $DEPLOY_STRATEGY == 'initial' ]; then + DEPLOY_STRATEGY='initial' update_frontend +else + DEPLOY_STRATEGY='rolling' + update_frontend 'rolling' fi diff --git a/tdrs-backend/apt.yml b/tdrs-backend/apt.yml index f07aee4a3..035482695 100644 --- a/tdrs-backend/apt.yml +++ b/tdrs-backend/apt.yml @@ -6,4 +6,3 @@ repos: packages: - postgresql-client-12 - libjemalloc-dev - - redis diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh new file mode 100755 index 000000000..88b4aba21 --- /dev/null +++ b/tdrs-backend/celery_start.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +echo starting celery +celery -A tdpservice.settings worker -c 3 & +sleep 5 +echo "REDIS_URI: $REDIS_URI" +celery -A tdpservice.settings --broker=$REDIS_URI flower --port=8080 & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index d2cd5289c..b65a72e36 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -81,8 +81,7 @@ services: build: . command: > bash -c "./wait_for_services.sh && - ./gunicorn_start.sh && - celery -A tdpservice.settings worker -l info" + ./gunicorn_start.sh &&" ports: - "5555:5555" depends_on: @@ -90,6 +89,7 @@ services: - postgres - redis-server - elastic + - celery redis-server: image: "redis:alpine" @@ -99,6 +99,50 @@ services: volumes: - .:/tdpapp + celery: + restart: always + env_file: + - .env + environment: + - CLAMAV_NEEDED + - AV_SCAN_URL=http://clamav-rest:9000/scan + - DB_HOST=postgres + - DB_NAME=tdrs_test + - DB_PASSWORD=something_secure + - DB_PORT=5432 + - DB_USER=tdpuser + - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Local} + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-tdp-dev-insecure} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-tdpservice.settings.local} + - LOCALSTACK_HOST=localstack + - DJANGO_SU_NAME + - JWT_CERT_TEST + - JWT_KEY + - USE_LOCALSTACK + - LOGGING_LEVEL + - AMS_CLIENT_ID + - AMS_CLIENT_SECRET + - AMS_CONFIGURATION_ENDPOINT + - ACFTITAN_HOST + - ACFTITAN_KEY + - ACFTITAN_USERNAME + - REDIS_URI=redis://redis-server:6379 + - REDIS_SERVER_LOCAL=TRUE + - ACFTITAN_SFTP_PYTEST + - SENDGRID_API_KEY + volumes: + - .:/tdpapp + image: tdp + build: . + command: > + bash -c "./wait_for_services.sh && + ./celery_start.sh" + depends_on: + - localstack + - postgres + - redis-server + - elastic + volumes: localstack_data: postgres_data: diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index a6624688b..35fcb0e4a 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -96,7 +96,6 @@ services: - ACFTITAN_KEY - ACFTITAN_USERNAME - REDIS_URI=redis://redis-server:6379 - - REDIS_SERVER_LOCAL=TRUE - ACFTITAN_SFTP_PYTEST - CYPRESS_TOKEN - DJANGO_DEBUG @@ -110,7 +109,7 @@ services: ./manage.py makemigrations && ./manage.py migrate && ./manage.py populate_stts && - ./gunicorn_start.sh && celery -A tdpservice.settings worker -l info" + ./gunicorn_start.sh" ports: - "5555:5555" tty: true @@ -120,6 +119,7 @@ services: - postgres - redis-server - elastic + - celery redis-server: image: "redis:alpine" @@ -129,6 +129,48 @@ services: volumes: - .:/tdpapp + celery: + restart: always + environment: + - CLAMAV_NEEDED + - AV_SCAN_URL=http://clamav-rest:9000/scan + - DB_HOST=postgres + - DB_NAME=tdrs_test + - DB_PASSWORD=something_secure + - DB_PORT=5432 + - DB_USER=tdpuser + - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Local} + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-tdp-dev-insecure} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-tdpservice.settings.local} + - LOCALSTACK_HOST=localstack + - DJANGO_SU_NAME + - JWT_CERT_TEST + - JWT_KEY + - USE_LOCALSTACK + - LOGGING_LEVEL + - AMS_CLIENT_ID + - AMS_CLIENT_SECRET + - AMS_CONFIGURATION_ENDPOINT + - ACFTITAN_HOST + - ACFTITAN_KEY + - ACFTITAN_USERNAME + - REDIS_URI=redis://redis-server:6379 + - REDIS_SERVER_LOCAL=TRUE + - ACFTITAN_SFTP_PYTEST + - SENDGRID_API_KEY + volumes: + - .:/tdpapp + image: tdp + build: . + command: > + bash -c "./wait_for_services.sh && + ./celery_start.sh" + depends_on: + - localstack + - postgres + - redis-server + - elastic + volumes: localstack_data: postgres_data: diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 684e7eb24..5e4804bdf 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -2,28 +2,11 @@ # Apply database migrations set -e -echo "REDIS_SERVER" -echo "redis local: $REDIS_SERVER_LOCAL" -if [[ "$REDIS_SERVER_LOCAL" = "TRUE" || "$CIRCLE_JOB" = "backend-owasp-scan" ]]; then - echo "Run redis server on docker" -else - echo "Run redis server locally" - export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH - ( cd /home/vcap/deps/0/bin/; ./redis-server /home/vcap/app/redis.conf &) -fi - -# echo "Applying database migrations" python manage.py migrate #python manage.py populate_stts #python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -c 1 & -sleep 5 -# TODO: Uncomment the following line to add flower service when memory limitation is resolved -celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & - echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then gunicorn_params="-c gunicorn_dev_cfg.py" diff --git a/tdrs-backend/manifest.buildpack.yml b/tdrs-backend/manifest.buildpack.yml index fc9d1460c..54e8b5c98 100755 --- a/tdrs-backend/manifest.buildpack.yml +++ b/tdrs-backend/manifest.buildpack.yml @@ -4,8 +4,6 @@ applications: memory: 2G instances: 1 disk_quota: 2G - env: - REDIS_URI: redis://localhost:6379 buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 diff --git a/tdrs-backend/manifest.celery.yml b/tdrs-backend/manifest.celery.yml new file mode 100755 index 000000000..2f75241d1 --- /dev/null +++ b/tdrs-backend/manifest.celery.yml @@ -0,0 +1,12 @@ +version: 1 +applications: +- name: tdp-celery + memory: 2G + instances: 1 + disk_quota: 2G + buildpacks: + - https://github.com/cloudfoundry/apt-buildpack + - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 + command: "./celery_start.sh" + + diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 6955ca9ce..63fccb722 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,7 +1,9 @@ """Celery configuration file.""" from __future__ import absolute_import import os +import ssl import configurations +from django.conf import settings from celery import Celery @@ -17,5 +19,17 @@ # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') + +# disable ssl verification +if not settings.USE_LOCALSTACK: + app.conf.update( + broker_use_ssl={ + 'ssl_cert_reqs': ssl.CERT_NONE, + }, + redis_backend_use_ssl={ + 'ssl_cert_reqs': ssl.CERT_NONE, + }, + ) + # Load task modules from all registered Django apps. app.autodiscover_tasks() diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index b00f76fa9..23436733a 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -4,7 +4,6 @@ import os from requests_aws4auth import AWS4Auth from elasticsearch import RequestsHttpConnection - from tdpservice.settings.common import Common import logging @@ -16,7 +15,6 @@ def get_json_env_var(variable_name): os.getenv(variable_name, '{}') ) - def get_cloudgov_service_creds_by_instance_name(services, instance_name): """Retrieve credentials for a bound Cloud.gov service by instance name.""" return next( @@ -25,7 +23,6 @@ def get_cloudgov_service_creds_by_instance_name(services, instance_name): {} ) - class CloudGov(Common): """Base settings class for applications deployed in Cloud.gov.""" @@ -37,13 +34,14 @@ class CloudGov(Common): # Cloud.gov exposes variables for the application and bound services via # VCAP_APPLICATION and VCAP_SERVICES environment variables, respectively. cloudgov_app = get_json_env_var('VCAP_APPLICATION') - APP_NAME = cloudgov_app.get('application_name') + APP_NAME = os.getenv('CGAPPNAME_BACKEND', '{}') cloudgov_services = get_json_env_var('VCAP_SERVICES') cloudgov_space = cloudgov_app.get('space_name', 'tanf-dev') cloudgov_space_suffix = cloudgov_space.strip('tanf-') cloudgov_name = cloudgov_app.get('name').split("-")[-1] # converting "tdp-backend-name" to just "name" + services_basename = cloudgov_name if ( cloudgov_name == "develop" and cloudgov_space_suffix == "staging" ) else cloudgov_space_suffix @@ -149,6 +147,13 @@ class CloudGov(Common): }, } + # Redis + redis_settings = cloudgov_services['aws-elasticache-redis'][0]['credentials'] + REDIS_URI = f"rediss://:{redis_settings['password']}@{redis_settings['host']}:{redis_settings['port']}" + logger.debug("REDIS_URI: " + REDIS_URI) + + CELERY_BROKER_URL = REDIS_URI + '/0' + CELERY_RESULT_BACKEND = REDIS_URI + '/1' class Development(CloudGov): """Settings for applications deployed in the Cloud.gov dev space.""" diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index da1df5b10..0db1e94d5 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -86,8 +86,7 @@ data "cloudfoundry_service" "redis" { } resource "cloudfoundry_service_instance" "redis" { - for_each = toset(var.dev_app_names) - name = "tdp-redis-${each.value}" + name = "tdp-redis-develop" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] } diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf index 58173eaf1..fa44ceef2 100644 --- a/terraform/dev/variables.tf +++ b/terraform/dev/variables.tf @@ -39,6 +39,6 @@ variable "cf_user" { variable "dev_app_names" { type = list(string) - description = "list of app names deployed in the dev environment" + description = "list of app names deployed in the dev cf space" default = ["a11y", "qasp", "raft", "sandbox"] } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 6948ecd72..b22592505 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -75,3 +75,15 @@ resource "cloudfoundry_service_instance" "datafiles" { service_plan = data.cloudfoundry_service.s3.service_plans["basic"] recursive_delete = true } + +data "cloudfoundry_service" "redis" { + name = "aws-elasticache-redis" +} + +resource "cloudfoundry_service_instance" "redis" { + name = "tdp-redis-prod" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.redis.service_plans["PLACEHOLDER"] + # before prod deploy choose one of the following redis type: + # [redis-dev, redis-3node, redis-5node, redis-3node-large, redis-5node-large] +} diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 7b6d45a1f..29beb31f9 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -75,3 +75,15 @@ resource "cloudfoundry_service_instance" "datafiles" { service_plan = data.cloudfoundry_service.s3.service_plans["basic-sandbox"] recursive_delete = true } + +data "cloudfoundry_service" "redis" { + name = "aws-elasticache-redis" +} + +resource "cloudfoundry_service_instance" "redis" { + for_each = toset(var.staging_app_names) + name = "tdp-redis-${each.value}" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] +} + diff --git a/terraform/staging/variables.tf b/terraform/staging/variables.tf index 824c3ebf6..c61902eeb 100644 --- a/terraform/staging/variables.tf +++ b/terraform/staging/variables.tf @@ -36,3 +36,9 @@ variable "cf_app_name" { type = string description = "name of app" } + +variable "staging_app_names" { + type = list(string) + description = "list of app names deployed in the staging cf space" + default = ["develop", "staging"] +}