From 9c360913080b4add7db860b293887aae1f17d308 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 30 Nov 2023 13:41:19 -0700 Subject: [PATCH 01/41] WIP updated terraform and started celery setup. --- tdrs-backend/apt.yml | 1 - tdrs-backend/manifest.buildpack.yml | 2 -- tdrs-backend/manifest.celery.yml | 12 ++++++++++++ terraform/dev/main.tf | 15 +++++++++++++++ terraform/dev/variables.tf | 27 +++++++++++++++++++-------- 5 files changed, 46 insertions(+), 11 deletions(-) create mode 100755 tdrs-backend/manifest.celery.yml diff --git a/tdrs-backend/apt.yml b/tdrs-backend/apt.yml index f07aee4a3..035482695 100644 --- a/tdrs-backend/apt.yml +++ b/tdrs-backend/apt.yml @@ -6,4 +6,3 @@ repos: packages: - postgresql-client-12 - libjemalloc-dev - - redis diff --git a/tdrs-backend/manifest.buildpack.yml b/tdrs-backend/manifest.buildpack.yml index fc9d1460c..54e8b5c98 100755 --- a/tdrs-backend/manifest.buildpack.yml +++ b/tdrs-backend/manifest.buildpack.yml @@ -4,8 +4,6 @@ applications: memory: 2G instances: 1 disk_quota: 2G - env: - REDIS_URI: redis://localhost:6379 buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 diff --git a/tdrs-backend/manifest.celery.yml b/tdrs-backend/manifest.celery.yml new file mode 100755 index 000000000..fc9d1460c --- /dev/null +++ b/tdrs-backend/manifest.celery.yml @@ -0,0 +1,12 @@ +version: 1 +applications: +- name: tdp-backend + memory: 2G + instances: 1 + disk_quota: 2G + env: + REDIS_URI: redis://localhost:6379 + buildpacks: + - https://github.com/cloudfoundry/apt-buildpack + - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 + command: "./gunicorn_start.sh" diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index ba01d9cf3..79bc33b3a 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -76,3 +76,18 @@ resource "cloudfoundry_service_instance" "datafiles" { service_plan = data.cloudfoundry_service.s3.service_plans["basic-sandbox"] recursive_delete = true } + +### +# Provision Redis for each env +### + +data "cloudfoundry_service" "redis" { + name = "aws-elasticache-redis" +} + +resource "cloudfoundry_service_instance" "redis" { + for_each = toset(var.dev_app_names) + name = "${each.value}-redis" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] +} \ No newline at end of file diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf index eedbcab1a..f57264c5e 100644 --- a/terraform/dev/variables.tf +++ b/terraform/dev/variables.tf @@ -10,12 +10,22 @@ variable "cf_api_url" { default = "https://api.fr.cloud.gov" } +variable "cf_app_name" { + type = string + description = "name of app" +} + variable "cf_org_name" { type = string description = "cloud.gov organization name" default = "hhs-acf-ofa" } +variable "cf_password" { + type = string + description = "secret; cloud.gov deployer account password" +} + variable "cf_space_name" { type = string description = "cloud.gov space name" @@ -27,12 +37,13 @@ variable "cf_user" { description = "secret; cloud.gov deployer account user" } -variable "cf_password" { - type = string - description = "secret; cloud.gov deployer account password" -} - -variable "cf_app_name" { - type = string - description = "name of app" +variable "dev_app_names" { + type = list(string) + description = "list of app names deployed in the dev environment" + default = [ + "a11y", + "qasp", + "raft", + "sandbox" + ] } From fc082b576e91a44accd3377b2136ca8fabcd1be7 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 30 Nov 2023 13:43:53 -0700 Subject: [PATCH 02/41] WIP --- tdrs-backend/gunicorn_start.sh | 11 ----------- tdrs-backend/tdpservice/settings/celery.py | 12 ++++++++++++ tdrs-backend/tdpservice/settings/cloudgov.py | 8 ++++++++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 684e7eb24..72c9267ac 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -2,17 +2,6 @@ # Apply database migrations set -e -echo "REDIS_SERVER" -echo "redis local: $REDIS_SERVER_LOCAL" -if [[ "$REDIS_SERVER_LOCAL" = "TRUE" || "$CIRCLE_JOB" = "backend-owasp-scan" ]]; then - echo "Run redis server on docker" -else - echo "Run redis server locally" - export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH - ( cd /home/vcap/deps/0/bin/; ./redis-server /home/vcap/app/redis.conf &) -fi - -# echo "Applying database migrations" python manage.py migrate #python manage.py populate_stts diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 6955ca9ce..1a635bd69 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,6 +1,7 @@ """Celery configuration file.""" from __future__ import absolute_import import os +import ssl import configurations from celery import Celery @@ -17,5 +18,16 @@ # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') + +# disable ssl verification +app.conf.update( + broker_use_ssl={ + 'ssl_cert_reqs': ssl.CERT_NONE, + }, + redis_backend_use_ssl={ + 'ssl_cert_reqs': ssl.CERT_NONE, + }, +) + # Load task modules from all registered Django apps. app.autodiscover_tasks() diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index b7def9383..3a43edb36 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -149,6 +149,14 @@ class CloudGov(Common): }, } + # Redis + redis_settings = cloudgov_services['aws-elasticache-redis'][0]['credentials'] + REDIS_URI = f"rediss://:{redis_settings['password']}@{redis_settings['host']}:{redis_settings['port']}" + logger.debug("REDIS_URI: " + REDIS_URI) + + CELERY_BROKER_URL = REDIS_URI + '/0' + CELERY_RESULT_BACKEND = REDIS_URI + '/1' + class Development(CloudGov): """Settings for applications deployed in the Cloud.gov dev space.""" From f14c946f6ddade8ec2dba8e5dba28aad119af960 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 7 Dec 2023 12:12:39 -0700 Subject: [PATCH 03/41] creating redis services, separate celery instance, and new connection info --- scripts/deploy-backend.sh | 59 +++++++++++--------- tdrs-backend/celery_start.sh | 7 +++ tdrs-backend/docker-compose.yml | 1 - tdrs-backend/gunicorn_start.sh | 6 -- tdrs-backend/manifest.celery.yml | 6 +- tdrs-backend/tdpservice/settings/cloudgov.py | 3 +- terraform/dev/main.tf | 2 +- 7 files changed, 47 insertions(+), 37 deletions(-) create mode 100644 tdrs-backend/celery_start.sh diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index ec372396a..c835b6bc0 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -7,32 +7,31 @@ # The deployment strategy you wish to employ ( rolling update or setting up a new environment) DEPLOY_STRATEGY=${1} -#The application name defined via the manifest yml for the frontend -CGAPPNAME_FRONTEND=${2} -CGAPPNAME_BACKEND=${3} -CF_SPACE=${4} +ENV=${2} +CF_SPACE=${3} +CGAPPNAME_FRONTEND="tdp-frontend-${ENV}" +CGAPPNAME_BACKEND="tdp-backend-${ENV}" +CGAPPNAME_CELERY="tdp-celery-${ENV}" strip() { # Usage: strip "string" "pattern" printf '%s\n' "${1##$2}" } -# The cloud.gov space defined via environment variable (e.g., "tanf-dev", "tanf-staging") -env=$(strip $CF_SPACE "tanf-") -backend_app_name=$(echo $CGAPPNAME_BACKEND | cut -d"-" -f3) +# The cloud.gov space defined via CF_SPACE environment variable (e.g., "tanf-dev", "tanf-staging") +space=$(strip $CF_SPACE "tanf-") echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY" echo BACKEND_HOST: "$CGAPPNAME_BACKEND" echo CF_SPACE: "$CF_SPACE" -echo env: "$env" -echo backend_app_name: "$backend_app_name" +echo space: "$space" +echo environment: "$ENV" ############################## # Function Decls ############################## -set_cf_envs() -{ +set_cf_envs() { var_list=( "ACFTITAN_HOST" "ACFTITAN_KEY" @@ -78,16 +77,14 @@ set_cf_envs() } # Helper method to generate JWT cert and keys for new environment -generate_jwt_cert() -{ +generate_jwt_cert() { echo "regenerating JWT cert/key" yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256 cf set-env "$CGAPPNAME_BACKEND" JWT_CERT "$(cat cert.pem)" cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)" } -update_backend() -{ +update_backend() { cd tdrs-backend || exit cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" @@ -95,7 +92,7 @@ update_backend() cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" else # Add environment varilables for clamav - cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan" + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$space.apps.internal:9000/scan" fi if [ "$1" = "rolling" ] ; then @@ -104,8 +101,10 @@ update_backend() # Do a zero downtime deploy. This requires enough memory for # two apps to exist in the org/space at one time. cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1 + cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 --strategy rolling || exit 1 else cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 + cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 # set up JWT key if needed if cf e "$CGAPPNAME_BACKEND" | grep -q JWT_KEY ; then echo jwt cert already created @@ -125,7 +124,7 @@ update_backend() # Add network policy to allow backend to access tanf-prod services cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 else - cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$env --protocol tcp --port 9000 + cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$space --protocol tcp --port 9000 fi cd .. @@ -138,21 +137,29 @@ bind_backend_to_services() { # TODO: this is technical debt, we should either make staging mimic tanf-dev # or make unique services for all apps but we have a services limit # Introducing technical debt for release 3.0.0 specifically. - env="develop" + space="develop" fi - cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${env}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}" - - # The below command is different because they cannot be shared like the 3 above services - cf bind-service "$CGAPPNAME_BACKEND" "es-${backend_app_name}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${space}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${space}" + + f bind-service "$CGAPPNAME_CELERY" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-datafiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-db-${space}" + + # bind to redis + cf bind-service "$CGAPPNAME_BACKEND" "tdp-redis-${ENV}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-redis-${ENV}" + # bind to elastic-search + cf bind-service "$CGAPPNAME_BACKEND" "es-${ENV}" + cf bind-service "$CGAPPNAME_CELERY" "es-${ENV}" set_cf_envs - echo "Restarting app: $CGAPPNAME_BACKEND" + echo "Restarting apps: $CGAPPNAME_BACKEND and $CGAPPNAME_CELERY" cf restage "$CGAPPNAME_BACKEND" - + cf restage "$CGAPPNAME_CELERY" } ############################## diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh new file mode 100644 index 000000000..bb279eae2 --- /dev/null +++ b/tdrs-backend/celery_start.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +celery -A tdpservice.settings worker -c 1 & +sleep 5 +# TODO: Uncomment the following line to add flower service when memory limitation is resolved +celery -A tdpservice.settings --broker=$REDIS_URI flower & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & \ No newline at end of file diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 69e08bc64..9101d21d5 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -83,7 +83,6 @@ services: - ACFTITAN_KEY - ACFTITAN_USERNAME - REDIS_URI=redis://redis-server:6379 - - REDIS_SERVER_LOCAL=TRUE - ACFTITAN_SFTP_PYTEST - CYPRESS_TOKEN - DJANGO_DEBUG diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 72c9267ac..5e4804bdf 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -7,12 +7,6 @@ python manage.py migrate #python manage.py populate_stts #python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -c 1 & -sleep 5 -# TODO: Uncomment the following line to add flower service when memory limitation is resolved -celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & - echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then gunicorn_params="-c gunicorn_dev_cfg.py" diff --git a/tdrs-backend/manifest.celery.yml b/tdrs-backend/manifest.celery.yml index fc9d1460c..0497f1096 100755 --- a/tdrs-backend/manifest.celery.yml +++ b/tdrs-backend/manifest.celery.yml @@ -1,6 +1,6 @@ version: 1 applications: -- name: tdp-backend +- name: tdp-celery memory: 2G instances: 1 disk_quota: 2G @@ -9,4 +9,6 @@ applications: buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 - command: "./gunicorn_start.sh" + command: "./celery_start.sh" + + diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 3a43edb36..453653665 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -25,7 +25,6 @@ def get_cloudgov_service_creds_by_instance_name(services, instance_name): {} ) - class CloudGov(Common): """Base settings class for applications deployed in Cloud.gov.""" @@ -44,6 +43,7 @@ class CloudGov(Common): cloudgov_space = cloudgov_app.get('space_name', 'tanf-dev') cloudgov_space_suffix = cloudgov_space.strip('tanf-') cloudgov_name = cloudgov_app.get('name').split("-")[-1] # converting "tdp-backend-name" to just "name" + services_basename = cloudgov_name if ( cloudgov_name == "develop" and cloudgov_space_suffix == "staging" ) else cloudgov_space_suffix @@ -139,6 +139,7 @@ class CloudGov(Common): 'es' ) + # Elastic ELASTICSEARCH_DSL = { 'default': { diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 79bc33b3a..c728a7f90 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -87,7 +87,7 @@ data "cloudfoundry_service" "redis" { resource "cloudfoundry_service_instance" "redis" { for_each = toset(var.dev_app_names) - name = "${each.value}-redis" + name = "tdp-redis-${each.value}" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] } \ No newline at end of file From b0d581ed8293010fd7dff6d733eee8ad8d51d52e Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 8 Dec 2023 11:08:17 -0700 Subject: [PATCH 04/41] update pipeline and deploy script logic to deploy celery and make more DRY --- .circleci/deployment/commands.yml | 71 +++++-------------------------- .circleci/deployment/jobs.yml | 13 ++---- scripts/deploy-backend.sh | 29 ++++++++++--- scripts/deploy-frontend.sh | 38 ++++++++++------- 4 files changed, 62 insertions(+), 89 deletions(-) diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index 43adb60e3..ef332498e 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -2,12 +2,7 @@ deploy-cloud-dot-gov: parameters: environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend + default: raft type: string cf-password: default: CF_PASSWORD_DEV @@ -21,9 +16,6 @@ cf-username: default: CF_USERNAME_DEV type: env_var_name - frontend-appname: - default: tdp-frontend - type: string steps: - checkout - sudo-check @@ -34,14 +26,11 @@ cf-space: <> cf-username: <> - deploy-backend: - backend-appname: <> - frontend-appname: <> cf-space: <> - - deploy-frontend: environment: <> - backend-appname: <> - frontend-appname: <> + - deploy-frontend: cf-space: <> + environment: <> clamav-cloud-dot-gov: parameters: @@ -73,15 +62,12 @@ deploy-backend: parameters: - backend-appname: - default: tdp-backend - type: string - frontend-appname: - default: tdp-frontend - type: string cf-space: default: tanf-dev type: string + environment: + default: raft + type: string steps: - get-app-deploy-strategy: appname: <> @@ -89,10 +75,8 @@ name: Deploy backend application command: | bash ./scripts/deploy-backend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> \ - <> + <> \ + <> deploy-clamav: steps: @@ -104,58 +88,25 @@ deploy-frontend: parameters: - environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend - type: string - frontend-appname: - default: tdp-frontend - type: string -# So the frontend knows what space its in for the banner. -# I am unclear if the domain is a reliable metric to make this function -# It seems like it might not be working cf-space: + type: string default: dev + environment: type: string + default: raft steps: - install-nodejs: node-version: "16.13" - disable-npm-audit - install-nodejs-packages: app-dir: tdrs-frontend - - get-app-deploy-strategy: - appname: <> - run: name: Deploy frontend application command: | bash ./scripts/deploy-frontend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> \ <> \ <> - get-app-deploy-strategy: - parameters: - appname: - type: string - steps: - - run: - name: Determine deploy strategy - command: | - # NOTE: The || true is a no-op included to suppress exit codes which - # would cause the step to exit early due to use of pipefail - APP_GUID=$(cf app <> --guid || true) - if [ "$APP_GUID" == "FAILED" ]; then - echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV - else - echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV - fi - deploy-infrastructure: parameters: tf-path: diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 9aa40dfa7..6b468d10f 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -7,16 +7,14 @@ working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - backend-appname: tdp-backend-<< parameters.target_env >> - frontend-appname: tdp-frontend-<< parameters.target_env >> + environment: << parameters.target_env >> deploy-staging: executor: docker-executor working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - backend-appname: tdp-backend-staging - frontend-appname: tdp-frontend-staging + environment: staging cf-password: CF_PASSWORD_STAGING cf-space: tanf-staging cf-username: CF_USERNAME_STAGING @@ -26,8 +24,7 @@ working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - backend-appname: tdp-backend-develop - frontend-appname: tdp-frontend-develop + environment: develop cf-password: CF_PASSWORD_STAGING cf-space: tanf-staging cf-username: CF_USERNAME_STAGING @@ -145,9 +142,7 @@ working_directory: ~/tdp-deploy steps: - deploy-cloud-dot-gov: - environment: production - backend-appname: tdp-backend-prod - frontend-appname: tdp-frontend-prod + environment: prod cf-password: CF_PASSWORD_PROD cf-space: tanf-prod cf-username: CF_USERNAME_PROD diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index d19ac8ecb..5962adbdf 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -4,11 +4,11 @@ # Global Variable Decls ############################## -# The deployment strategy you wish to employ ( rolling update or setting up a new environment) -DEPLOY_STRATEGY=${1} - +CF_SPACE=${1} ENV=${2} -CF_SPACE=${3} + +DEPLOY_STRATEGY=${3-tbd} + CGAPPNAME_FRONTEND="tdp-frontend-${ENV}" CGAPPNAME_BACKEND="tdp-backend-${ENV}" CGAPPNAME_CELERY="tdp-celery-${ENV}" @@ -211,6 +211,24 @@ else CYPRESS_TOKEN=$CYPRESS_TOKEN fi +APP_GUID=$(cf app $CGHOSTNAME_BACKEND --guid || true) +CELERY_GUID=$(cf app $CGHOSTNAME_CELERY --guid || true) + +# if celery or backend missing, remove other and perform initial deploy +if [ "$DEPLOY_STRATEGY" = "tbd" ] ; then + if [ $APP_GUID == 'FAILED' ] && [ $CELERY_GUID == 'FAILED' ]; then + DEPLOY_STRATEGY='initial' + elif [ $APP_GUID == 'FAILED' ]; then + cf delete "$CGAPPNAME_CELERY" -r -f + DEPLOY_STRATEGY='initial' + elif [ $CELERY_GUID == 'FAILED' ]; then + cf delete "$CGAPPNAME_BACKEND" -r -f + DEPLOY_STRATEGY='initial' + else + DEPLOY_STRATEGY='rolling' + fi +fi + if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then # Perform a rolling update for the backend and frontend deployments if # specified, otherwise perform a normal deployment @@ -218,7 +236,7 @@ if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then elif [ "$DEPLOY_STRATEGY" = "bind" ] ; then # Bind the services the application depends on and restage the app. bind_backend_to_services -elif [ "$DEPLOY_STRATEGY" = "initial" ]; then +elif [ "$DEPLOY_STRATEGY" = "initial" ] || ; then # There is no app with this name, and the services need to be bound to it # for it to work. the app will fail to start once, have the services bind, # and then get restaged. @@ -229,6 +247,7 @@ elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then # Delete the existing app (with out deleting the services) # and perform the initial deployment strategy. cf delete "$CGAPPNAME_BACKEND" -r -f + cf delete "$CGAPPNAME_CELERY" -r -f update_backend bind_backend_to_services else diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh index 96af218f2..5139b7ba9 100755 --- a/scripts/deploy-frontend.sh +++ b/scripts/deploy-frontend.sh @@ -2,20 +2,24 @@ # source deploy-util.sh -# The deployment strategy you wish to employ ( rolling update or setting up a new environment) -DEPLOY_STRATEGY=${1} +CF_SPACE=${1} +ENV=${2} -#The application name defined via the manifest yml for the frontend -CGHOSTNAME_FRONTEND=${2} -CGHOSTNAME_BACKEND=${3} -CF_SPACE=${4} -ENVIRONMENT=${5} +DEPLOY_STRATEGY=${3-'tbd'} -update_frontend() -{ +CGAPPNAME_FRONTEND="tdp-frontend-${ENV}" +CGAPPNAME_BACKEND="tdp-backend-${ENV}" + +[[ $ENV = "prod" ]] && BUILD_ENV="production" || BUILD_ENV="development" + +update_frontend() { echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY" echo FRONTEND_HOST: "$CGHOSTNAME_FRONTEND" echo BACKEND_HOST: "$CGHOSTNAME_BACKEND" + echo CF_SPACE: "$CF_SPACE" + echo ENVIRONMENT: "$ENV" + echo BUILD_ENV: "$BUILD_ENV" + cd tdrs-frontend || exit if [ "$CF_SPACE" = "tanf-prod" ]; then @@ -45,7 +49,7 @@ update_frontend() cf set-env "$CGHOSTNAME_FRONTEND" BACKEND_HOST "$CGHOSTNAME_BACKEND" - npm run build:$ENVIRONMENT + npm run build:$BUILD_ENV unlink .env.production mkdir deployment @@ -81,11 +85,15 @@ update_frontend() cd ../.. rm -r tdrs-frontend/deployment } +# NOTE: The || true is a no-op included to suppress exit codes which +# would cause the step to exit early due to use of pipefail -# perform a rolling update for the backend and frontend deployments if -# specified, otherwise perform a normal deployment -if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then - update_frontend 'rolling' -else +APP_GUID=$(cf app $CGHOSTNAME_FRONTEND --guid || true) + +if [ $APP_GUID == 'FAILED' ] || [ $DEPLOY_STRATEGY == 'initial' ]; then + DEPLOY_STRATEGY='initial' update_frontend +else + DEPLOY_STRATEGY='rolling' + update_frontend 'rolling' fi From 5a99192f083749f49ba2b49b6a86e79f143c2e4c Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 8 Dec 2023 11:19:53 -0700 Subject: [PATCH 05/41] removed deploy strategy command from pipeline since it's now in scripts --- .circleci/deployment/commands.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index ef332498e..f97354e99 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -69,8 +69,6 @@ default: raft type: string steps: - - get-app-deploy-strategy: - appname: <> - run: name: Deploy backend application command: | From 9e4e926aaa77ef143c1015bbe4a234c27b4ae22f Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 8 Dec 2023 11:41:38 -0700 Subject: [PATCH 06/41] fixed spacing in variables.tf --- terraform/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index c728a7f90..03ad2e524 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -82,7 +82,7 @@ resource "cloudfoundry_service_instance" "datafiles" { ### data "cloudfoundry_service" "redis" { - name = "aws-elasticache-redis" + name = "aws-elasticache-redis" } resource "cloudfoundry_service_instance" "redis" { From a8910f6f0da79b7a96c8f7009d727c45bd415816 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 8 Dec 2023 11:47:09 -0700 Subject: [PATCH 07/41] fixed spacing in variables.tf --- terraform/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 03ad2e524..0e019b48a 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -82,7 +82,7 @@ resource "cloudfoundry_service_instance" "datafiles" { ### data "cloudfoundry_service" "redis" { - name = "aws-elasticache-redis" + name = "aws-elasticache-redis" } resource "cloudfoundry_service_instance" "redis" { From 332525c9e9461c3f27a48dfe48192769e8774f24 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 8 Dec 2023 11:49:37 -0700 Subject: [PATCH 08/41] fixed spacing in main.tf --- terraform/dev/variables.tf | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf index f57264c5e..58173eaf1 100644 --- a/terraform/dev/variables.tf +++ b/terraform/dev/variables.tf @@ -40,10 +40,5 @@ variable "cf_user" { variable "dev_app_names" { type = list(string) description = "list of app names deployed in the dev environment" - default = [ - "a11y", - "qasp", - "raft", - "sandbox" - ] + default = ["a11y", "qasp", "raft", "sandbox"] } From 24a4ba6320ad969b2e46a684beed87912c78627e Mon Sep 17 00:00:00 2001 From: George Hudson Date: Mon, 11 Dec 2023 15:25:17 -0700 Subject: [PATCH 09/41] fixed syntax errors --- scripts/deploy-backend.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 5962adbdf..f1bc1f2ec 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -22,6 +22,7 @@ space=$(strip $CF_SPACE "tanf-") echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY" echo BACKEND_HOST: "$CGAPPNAME_BACKEND" +echo CELERY_HOST: "$CGAPPNAME_CELERY" echo CF_SPACE: "$CF_SPACE" echo space: "$space" echo environment: "$ENV" @@ -215,7 +216,7 @@ APP_GUID=$(cf app $CGHOSTNAME_BACKEND --guid || true) CELERY_GUID=$(cf app $CGHOSTNAME_CELERY --guid || true) # if celery or backend missing, remove other and perform initial deploy -if [ "$DEPLOY_STRATEGY" = "tbd" ] ; then +if [ "$DEPLOY_STRATEGY" = "tbd" ]; then if [ $APP_GUID == 'FAILED' ] && [ $CELERY_GUID == 'FAILED' ]; then DEPLOY_STRATEGY='initial' elif [ $APP_GUID == 'FAILED' ]; then @@ -227,16 +228,18 @@ if [ "$DEPLOY_STRATEGY" = "tbd" ] ; then else DEPLOY_STRATEGY='rolling' fi +else + echo "Using given deployment strategy: ${DEPLOY_STRATEGY}" fi -if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then +if [ "$DEPLOY_STRATEGY" = "rolling" ]; then # Perform a rolling update for the backend and frontend deployments if # specified, otherwise perform a normal deployment update_backend 'rolling' -elif [ "$DEPLOY_STRATEGY" = "bind" ] ; then +elif [ "$DEPLOY_STRATEGY" = "bind" ]; then # Bind the services the application depends on and restage the app. bind_backend_to_services -elif [ "$DEPLOY_STRATEGY" = "initial" ] || ; then +elif [ "$DEPLOY_STRATEGY" = "initial" ]; then # There is no app with this name, and the services need to be bound to it # for it to work. the app will fail to start once, have the services bind, # and then get restaged. From 6b336ecb2849d770782799703b77c0f37616b327 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 12 Dec 2023 11:45:22 -0700 Subject: [PATCH 10/41] updated logic for deploy backend --- scripts/deploy-backend.sh | 211 ++++++++++++++++++++------------------ 1 file changed, 112 insertions(+), 99 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index f1bc1f2ec..84b330882 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -8,6 +8,7 @@ CF_SPACE=${1} ENV=${2} DEPLOY_STRATEGY=${3-tbd} +CELERY_DEPLOY_STRATEGY=${4-tbd} CGAPPNAME_FRONTEND="tdp-frontend-${ENV}" CGAPPNAME_BACKEND="tdp-backend-${ENV}" @@ -34,25 +35,25 @@ echo environment: "$ENV" set_cf_envs() { var_list=( - "ACFTITAN_HOST" - "ACFTITAN_KEY" - "ACFTITAN_USERNAME" - "AMS_CLIENT_ID" - "AMS_CLIENT_SECRET" - "AMS_CONFIGURATION_ENDPOINT" - "BASE_URL" - "CLAMAV_NEEDED" - "CYPRESS_TOKEN" - "DJANGO_CONFIGURATION" - "DJANGO_DEBUG" - "DJANGO_SECRET_KEY" - "DJANGO_SETTINGS_MODULE" - "DJANGO_SU_NAME" - "FRONTEND_BASE_URL" - "LOGGING_LEVEL" - "REDIS_URI" - "JWT_KEY" - "STAGING_JWT_KEY" + "ACFTITAN_HOST" + "ACFTITAN_KEY" + "ACFTITAN_USERNAME" + "AMS_CLIENT_ID" + "AMS_CLIENT_SECRET" + "AMS_CONFIGURATION_ENDPOINT" + "BASE_URL" + "CLAMAV_NEEDED" + "CYPRESS_TOKEN" + "DJANGO_CONFIGURATION" + "DJANGO_DEBUG" + "DJANGO_SECRET_KEY" + "DJANGO_SETTINGS_MODULE" + "DJANGO_SU_NAME" + "FRONTEND_BASE_URL" + "LOGGING_LEVEL" + "REDIS_URI" + "JWT_KEY" + "STAGING_JWT_KEY" ) echo "Setting environment variables for $CGAPPNAME_BACKEND" @@ -60,13 +61,13 @@ set_cf_envs() { for var_name in ${var_list[@]}; do # Intentionally unsetting variable if empty if [[ -z "${!var_name}" ]]; then - echo "WARNING: Empty value for $var_name. It will now be unset." - cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" - $cf_cmd - continue + echo "WARNING: Empty value for $var_name. It will now be unset." + cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" + $cf_cmd + continue elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then - sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') - cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') + cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" else cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" fi @@ -79,55 +80,61 @@ set_cf_envs() { # Helper method to generate JWT cert and keys for new environment generate_jwt_cert() { - echo "regenerating JWT cert/key" - yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256 - cf set-env "$CGAPPNAME_BACKEND" JWT_CERT "$(cat cert.pem)" - cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)" + echo "regenerating JWT cert/key" + yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256 + cf set-env "$CGAPPNAME_BACKEND" JWT_CERT "$(cat cert.pem)" + cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)" } update_backend() { - cd tdrs-backend || exit - cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" - - if [ "$CF_SPACE" = "tanf-prod" ]; then - cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" - else - # Add environment varilables for clamav - cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$space.apps.internal:9000/scan" - fi + cd tdrs-backend || exit + cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" + + if [ $CF_SPACE == 'tanf-prod' ]; then + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" + else + # Add environment varilables for clamav + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$space.apps.internal:9000/scan" + fi - if [ "$1" = "rolling" ] ; then - set_cf_envs - # Do a zero downtime deploy. This requires enough memory for - # two apps to exist in the org/space at one time. - cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1 - cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 --strategy rolling || exit 1 + if [ $1 == 'rolling' ] ; then + set_cf_envs + # Do a zero downtime deploy. This requires enough memory for + # two apps to exist in the org/space at one time. + cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1 + else + cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 + # set up JWT key if needed + if cf e "$CGAPPNAME_BACKEND" | grep -q JWT_KEY ; then + echo jwt cert already created else - cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 - cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 - # set up JWT key if needed - if cf e "$CGAPPNAME_BACKEND" | grep -q JWT_KEY ; then - echo jwt cert already created - else - generate_jwt_cert - fi + generate_jwt_cert fi + fi + if [ $2 == 'rolling' ] ; then set_cf_envs - - cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" + # Do a zero downtime deploy. This requires enough memory for + # two apps to exist in the org/space at one time. + cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 --strategy rolling || exit 1 + else + cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 + fi - # Add network policy to allow frontend to access backend - cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 - - if [ "$CF_SPACE" = "tanf-prod" ]; then - # Add network policy to allow backend to access tanf-prod services - cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 - else - cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$space --protocol tcp --port 9000 - fi + set_cf_envs + + cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" + + # Add network policy to allow frontend to access backend + cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 + + if [ $CF_SPACE == 'tanf-prod' ]; then + # Add network policy to allow backend to access tanf-prod services + cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 + else + cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$space --protocol tcp --port 9000 + fi - cd .. } bind_backend_to_services() { @@ -212,48 +219,54 @@ else CYPRESS_TOKEN=$CYPRESS_TOKEN fi -APP_GUID=$(cf app $CGHOSTNAME_BACKEND --guid || true) -CELERY_GUID=$(cf app $CGHOSTNAME_CELERY --guid || true) +APP_GUID=$(cf app $CGAPPNAME_BACKEND --guid || true) +CELERY_GUID=$(cf app $CGAPPNAME_CELERY --guid || true) -# if celery or backend missing, remove other and perform initial deploy -if [ "$DEPLOY_STRATEGY" = "tbd" ]; then - if [ $APP_GUID == 'FAILED' ] && [ $CELERY_GUID == 'FAILED' ]; then - DEPLOY_STRATEGY='initial' - elif [ $APP_GUID == 'FAILED' ]; then - cf delete "$CGAPPNAME_CELERY" -r -f - DEPLOY_STRATEGY='initial' - elif [ $CELERY_GUID == 'FAILED' ]; then - cf delete "$CGAPPNAME_BACKEND" -r -f +if [ $DEPLOY_STRATEGY == 'tbd' ]; then + if [ $APP_GUID == 'FAILED' ]; then DEPLOY_STRATEGY='initial' else DEPLOY_STRATEGY='rolling' fi + echo "Setting backend deployment strategy: ${DEPLOY_STRATEGY}" +else + echo "Using given backend deployment strategy: ${DEPLOY_STRATEGY}" +fi + +if [ $CELERY_DEPLOY_STRATEGY == 'tbd' ]; then + if [ $CELERY_GUID == 'FAILED' ]; then + CELERY_DEPLOY_STRATEGY='initial' + else + CELERY_DEPLOY_STRATEGY='rolling' + fi + echo "Setting celery deployment strategy: ${CELERY_DEPLOY_STRATEGY}" +else + echo "Using given celery deployment strategy: ${CELERY_DEPLOY_STRATEGY}" +fi + +if [ $DEPLOY_STRATEGY == 'rebuild' ]; then + # You want to redeploy the instance under the same name + # Delete the existing app (with out deleting the services) + # and perform the initial deployment strategy. + cf delete "$CGAPPNAME_BACKEND" -r -f + cf delete "$CGAPPNAME_CELERY" -r -f +fi + +if [ $DEPLOY_STRATEGY == 'bind' ]; then + # Bind the services the application depends on and restage the app. + bind_backend_to_services else - echo "Using given deployment strategy: ${DEPLOY_STRATEGY}" + update_backend $DEPLOY_STRATEGY $CELERY_DEPLOY_STRATEGY fi -if [ "$DEPLOY_STRATEGY" = "rolling" ]; then - # Perform a rolling update for the backend and frontend deployments if - # specified, otherwise perform a normal deployment - update_backend 'rolling' -elif [ "$DEPLOY_STRATEGY" = "bind" ]; then - # Bind the services the application depends on and restage the app. - bind_backend_to_services -elif [ "$DEPLOY_STRATEGY" = "initial" ]; then - # There is no app with this name, and the services need to be bound to it - # for it to work. the app will fail to start once, have the services bind, - # and then get restaged. - update_backend - bind_backend_to_services -elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then - # You want to redeploy the instance under the same name - # Delete the existing app (with out deleting the services) - # and perform the initial deployment strategy. - cf delete "$CGAPPNAME_BACKEND" -r -f - cf delete "$CGAPPNAME_CELERY" -r -f - update_backend - bind_backend_to_services +if [ $DEPLOY_STRATEGY == 'initial' ]; then + bind_backend_to_services +elif [ $DEPLOY_STRATEGY == 'rebuild' ]; then + bind_backend_to_services +elif [ $CELERY_DEPLOY_STRATEGY == 'initial' ]; then + bind_backend_to_services +elif [ $CELERY_DEPLOY_STRATEGY == 'rebuild' ]; then + bind_backend_to_services else - # No changes to deployment config, just deploy the changes and restart - update_backend + echo "no need to rebind to services" fi From 535bc30b8cf7d8b7d0b639daf9a035dedccf6bc3 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 12 Dec 2023 14:25:09 -0700 Subject: [PATCH 11/41] cleaned up spaces, etc. --- tdrs-backend/celery_start.sh | 2 +- tdrs-backend/manifest.celery.yml | 2 -- terraform/dev/main.tf | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index bb279eae2..541e20011 100644 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -4,4 +4,4 @@ celery -A tdpservice.settings worker -c 1 & sleep 5 # TODO: Uncomment the following line to add flower service when memory limitation is resolved celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & \ No newline at end of file +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & diff --git a/tdrs-backend/manifest.celery.yml b/tdrs-backend/manifest.celery.yml index 0497f1096..2f75241d1 100755 --- a/tdrs-backend/manifest.celery.yml +++ b/tdrs-backend/manifest.celery.yml @@ -4,8 +4,6 @@ applications: memory: 2G instances: 1 disk_quota: 2G - env: - REDIS_URI: redis://localhost:6379 buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 0e019b48a..da1df5b10 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -90,4 +90,4 @@ resource "cloudfoundry_service_instance" "redis" { name = "tdp-redis-${each.value}" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] -} \ No newline at end of file +} From 59e0c8be68192cf91aa43a1bfedc42614f1f39c8 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 13 Dec 2023 08:57:25 -0700 Subject: [PATCH 12/41] cleaned up more spaces, format of if statements, and indentation etc. --- scripts/deploy-backend.sh | 100 +++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 84b330882..eb3c6ba80 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -15,8 +15,8 @@ CGAPPNAME_BACKEND="tdp-backend-${ENV}" CGAPPNAME_CELERY="tdp-celery-${ENV}" strip() { - # Usage: strip "string" "pattern" - printf '%s\n' "${1##$2}" + # Usage: strip "string" "pattern" + printf '%s\n' "${1##$2}" } # The cloud.gov space defined via CF_SPACE environment variable (e.g., "tanf-dev", "tanf-staging") space=$(strip $CF_SPACE "tanf-") @@ -65,7 +65,7 @@ set_cf_envs() { cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" $cf_cmd continue - elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then + elif [[ ($var_name =~ 'STAGING_') && ($CF_SPACE = 'tanf-staging') ]]; then sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" else @@ -90,14 +90,14 @@ update_backend() { cd tdrs-backend || exit cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" - if [ $CF_SPACE == 'tanf-prod' ]; then + if [[ $CF_SPACE == 'tanf-prod' ]]; then cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" else # Add environment varilables for clamav cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$space.apps.internal:9000/scan" fi - if [ $1 == 'rolling' ] ; then + if [[ $1 == 'rolling' ]] ; then set_cf_envs # Do a zero downtime deploy. This requires enough memory for # two apps to exist in the org/space at one time. @@ -112,7 +112,7 @@ update_backend() { fi fi - if [ $2 == 'rolling' ] ; then + if [[ $2 == 'rolling' ]] ; then set_cf_envs # Do a zero downtime deploy. This requires enough memory for # two apps to exist in the org/space at one time. @@ -128,7 +128,7 @@ update_backend() { # Add network policy to allow frontend to access backend cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 - if [ $CF_SPACE == 'tanf-prod' ]; then + if [[ $CF_SPACE == 'tanf-prod' ]]; then # Add network policy to allow backend to access tanf-prod services cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 else @@ -138,35 +138,35 @@ update_backend() { } bind_backend_to_services() { - echo "Binding services to app: $CGAPPNAME_BACKEND" + echo "Binding services to app: $CGAPPNAME_BACKEND" - if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then - # TODO: this is technical debt, we should either make staging mimic tanf-dev - # or make unique services for all apps but we have a services limit - # Introducing technical debt for release 3.0.0 specifically. - space="develop" - fi + if [[ $CGAPPNAME_BACKEND = 'tdp-backend-develop' ]]; then + # TODO: this is technical debt, we should either make staging mimic tanf-dev + # or make unique services for all apps but we have a services limit + # Introducing technical debt for release 3.0.0 specifically. + space="develop" + fi + + cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${space}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${space}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${space}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${space}" - cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${space}" + f bind-service "$CGAPPNAME_CELERY" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-datafiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-db-${space}" - f bind-service "$CGAPPNAME_CELERY" "tdp-staticfiles-${space}" - cf bind-service "$CGAPPNAME_CELERY" "tdp-datafiles-${space}" - cf bind-service "$CGAPPNAME_CELERY" "tdp-db-${space}" + # bind to redis + cf bind-service "$CGAPPNAME_BACKEND" "tdp-redis-${ENV}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-redis-${ENV}" + # bind to elastic-search + cf bind-service "$CGAPPNAME_BACKEND" "es-${ENV}" + cf bind-service "$CGAPPNAME_CELERY" "es-${ENV}" - # bind to redis - cf bind-service "$CGAPPNAME_BACKEND" "tdp-redis-${ENV}" - cf bind-service "$CGAPPNAME_CELERY" "tdp-redis-${ENV}" - # bind to elastic-search - cf bind-service "$CGAPPNAME_BACKEND" "es-${ENV}" - cf bind-service "$CGAPPNAME_CELERY" "es-${ENV}" - - set_cf_envs + set_cf_envs - echo "Restarting apps: $CGAPPNAME_BACKEND and $CGAPPNAME_CELERY" - cf restage "$CGAPPNAME_BACKEND" - cf restage "$CGAPPNAME_CELERY" + echo "Restarting apps: $CGAPPNAME_BACKEND and $CGAPPNAME_CELERY" + cf restage "$CGAPPNAME_BACKEND" + cf restage "$CGAPPNAME_CELERY" } ############################## @@ -176,13 +176,13 @@ bind_backend_to_services() { # Determine the appropriate BASE_URL for the deployed instance based on the # provided Cloud.gov App Name DEFAULT_ROUTE="https://$CGAPPNAME_FRONTEND.app.cloud.gov" -if [ -n "$BASE_URL" ]; then +if [[ -n $BASE_URL ]]; then # Use Shell Parameter Expansion to replace localhost in the URL BASE_URL="${BASE_URL//http:\/\/localhost:8080/$DEFAULT_ROUTE}" -elif [ "$CF_SPACE" = "tanf-prod" ]; then +elif [[ $CF_SPACE == 'tanf-prod' ]]; then # Keep the base url set explicitly for production. BASE_URL="https://tanfdata.acf.hhs.gov/v1" -elif [ "$CF_SPACE" = "tanf-staging" ]; then +elif [[ $CF_SPACE == 'tanf-staging' ]]; then # use .acf.hss.gov domain for develop and staging. BASE_URL="https://$CGAPPNAME_FRONTEND.acf.hhs.gov/v1" else @@ -191,12 +191,12 @@ else fi DEFAULT_FRONTEND_ROUTE="${DEFAULT_ROUTE//backend/frontend}" -if [ -n "$FRONTEND_BASE_URL" ]; then +if [[ -n $FRONTEND_BASE_URL ]]; then FRONTEND_BASE_URL="${FRONTEND_BASE_URL//http:\/\/localhost:3000/$DEFAULT_FRONTEND_ROUTE}" -elif [ "$CF_SPACE" = "tanf-prod" ]; then +elif [[ $CF_SPACE == 'tanf-prod' ]]; then # Keep the base url set explicitly for production. - FRONTEND_BASE_URL="https://tanfdata.acf.hhs.gov" -elif [ "$CF_SPACE" = "tanf-staging" ]; then + FRONTEND_BASE_URL='https://tanfdata.acf.hhs.gov' +elif [[ $CF_SPACE == 'tanf-staging' ]]; then # use .acf.hss.gov domain for develop and staging. FRONTEND_BASE_URL="https://$CGAPPNAME_FRONTEND.acf.hhs.gov" else @@ -209,9 +209,9 @@ DJANGO_SECRET_KEY=$(python3 -c "from secrets import token_urlsafe; print(token_u # Dynamically set DJANGO_CONFIGURATION based on Cloud.gov Space DJANGO_SETTINGS_MODULE="tdpservice.settings.cloudgov" -if [ "$CF_SPACE" = "tanf-prod" ]; then +if [[ $CF_SPACE == 'tanf-prod' ]]; then DJANGO_CONFIGURATION="Production" -elif [ "$CF_SPACE" = "tanf-staging" ]; then +elif [[ $CF_SPACE == 'tanf-staging' ]]; then DJANGO_CONFIGURATION="Staging" else DJANGO_CONFIGURATION="Development" @@ -222,8 +222,8 @@ fi APP_GUID=$(cf app $CGAPPNAME_BACKEND --guid || true) CELERY_GUID=$(cf app $CGAPPNAME_CELERY --guid || true) -if [ $DEPLOY_STRATEGY == 'tbd' ]; then - if [ $APP_GUID == 'FAILED' ]; then +if [[ $DEPLOY_STRATEGY == 'tbd' ]]; then + if [[ $APP_GUID == 'FAILED' ]]; then DEPLOY_STRATEGY='initial' else DEPLOY_STRATEGY='rolling' @@ -233,8 +233,8 @@ else echo "Using given backend deployment strategy: ${DEPLOY_STRATEGY}" fi -if [ $CELERY_DEPLOY_STRATEGY == 'tbd' ]; then - if [ $CELERY_GUID == 'FAILED' ]; then +if [[ $CELERY_DEPLOY_STRATEGY == 'tbd' ]]; then + if [[ $CELERY_GUID == 'FAILED' ]]; then CELERY_DEPLOY_STRATEGY='initial' else CELERY_DEPLOY_STRATEGY='rolling' @@ -244,7 +244,7 @@ else echo "Using given celery deployment strategy: ${CELERY_DEPLOY_STRATEGY}" fi -if [ $DEPLOY_STRATEGY == 'rebuild' ]; then +if [[ $DEPLOY_STRATEGY == 'rebuild' ]]; then # You want to redeploy the instance under the same name # Delete the existing app (with out deleting the services) # and perform the initial deployment strategy. @@ -252,20 +252,20 @@ if [ $DEPLOY_STRATEGY == 'rebuild' ]; then cf delete "$CGAPPNAME_CELERY" -r -f fi -if [ $DEPLOY_STRATEGY == 'bind' ]; then +if [[ $DEPLOY_STRATEGY == 'bind' ]]; then # Bind the services the application depends on and restage the app. bind_backend_to_services else update_backend $DEPLOY_STRATEGY $CELERY_DEPLOY_STRATEGY fi -if [ $DEPLOY_STRATEGY == 'initial' ]; then +if [[ $DEPLOY_STRATEGY == 'initial' ]]; then bind_backend_to_services -elif [ $DEPLOY_STRATEGY == 'rebuild' ]; then +elif [[ $DEPLOY_STRATEGY == 'rebuild' ]]; then bind_backend_to_services -elif [ $CELERY_DEPLOY_STRATEGY == 'initial' ]; then +elif [[ $CELERY_DEPLOY_STRATEGY == 'initial' ]]; then bind_backend_to_services -elif [ $CELERY_DEPLOY_STRATEGY == 'rebuild' ]; then +elif [[ $CELERY_DEPLOY_STRATEGY == 'rebuild' ]]; then bind_backend_to_services else echo "no need to rebind to services" From 56f9eeb0372aabe75a473d080d0f04aa9ab2ffb6 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Mon, 18 Dec 2023 09:11:24 -0700 Subject: [PATCH 13/41] made celery_start.sh executable --- tdrs-backend/celery_start.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tdrs-backend/celery_start.sh diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh old mode 100644 new mode 100755 From 38990b021a0b690e4ba2552d151e0b099a576f45 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Mon, 18 Dec 2023 09:41:15 -0700 Subject: [PATCH 14/41] set env vars to celery as well --- scripts/deploy-backend.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index eb3c6ba80..1679695c0 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -56,20 +56,20 @@ set_cf_envs() { "STAGING_JWT_KEY" ) - echo "Setting environment variables for $CGAPPNAME_BACKEND" + echo "Setting environment variables for $1" for var_name in ${var_list[@]}; do # Intentionally unsetting variable if empty if [[ -z "${!var_name}" ]]; then echo "WARNING: Empty value for $var_name. It will now be unset." - cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" + cf_cmd="cf unset-env $1 $var_name ${!var_name}" $cf_cmd continue elif [[ ($var_name =~ 'STAGING_') && ($CF_SPACE = 'tanf-staging') ]]; then sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') - cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + cf_cmd="cf set-env $1 $sed_var_name ${!var_name}" else - cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" + cf_cmd="cf set-env $1 $var_name ${!var_name}" fi echo "Setting var : $var_name" @@ -98,7 +98,7 @@ update_backend() { fi if [[ $1 == 'rolling' ]] ; then - set_cf_envs + set_cf_envs $CGAPPNAME_BACKEND # Do a zero downtime deploy. This requires enough memory for # two apps to exist in the org/space at one time. cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1 @@ -113,7 +113,7 @@ update_backend() { fi if [[ $2 == 'rolling' ]] ; then - set_cf_envs + set_cf_envs $CGAPPNAME_CELERY # Do a zero downtime deploy. This requires enough memory for # two apps to exist in the org/space at one time. cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 --strategy rolling || exit 1 @@ -121,7 +121,8 @@ update_backend() { cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 fi - set_cf_envs + set_cf_envs $CGAPPNAME_BACKEND + set_cf_envs $CGAPPNAME_CELERY cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" @@ -162,7 +163,8 @@ bind_backend_to_services() { cf bind-service "$CGAPPNAME_BACKEND" "es-${ENV}" cf bind-service "$CGAPPNAME_CELERY" "es-${ENV}" - set_cf_envs + set_cf_envs $CGAPPNAME_BACKEND + set_cf_envs $CGAPPNAME_CELERY echo "Restarting apps: $CGAPPNAME_BACKEND and $CGAPPNAME_CELERY" cf restage "$CGAPPNAME_BACKEND" From aca5d8feb9c8013977f236759f189ebf7893e563 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Mon, 18 Dec 2023 12:12:24 -0700 Subject: [PATCH 15/41] added cf bind-services for staticfiles (fixed typo). --- scripts/deploy-backend.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 1679695c0..1c3450b32 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -152,7 +152,7 @@ bind_backend_to_services() { cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${space}" cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${space}" - f bind-service "$CGAPPNAME_CELERY" "tdp-staticfiles-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-staticfiles-${space}" cf bind-service "$CGAPPNAME_CELERY" "tdp-datafiles-${space}" cf bind-service "$CGAPPNAME_CELERY" "tdp-db-${space}" From 4a2e4d905aae7803ba0de095a2d1532922121d20 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Mon, 18 Dec 2023 12:50:45 -0700 Subject: [PATCH 16/41] removed REDIS_URI from ENV list because we should be grabbing from VCAP_SERVICES --- scripts/deploy-backend.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 1c3450b32..baf2bc57b 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -51,7 +51,6 @@ set_cf_envs() { "DJANGO_SU_NAME" "FRONTEND_BASE_URL" "LOGGING_LEVEL" - "REDIS_URI" "JWT_KEY" "STAGING_JWT_KEY" ) From 6495c78383424c6ab4479908bdb18c3a4fe16c22 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 19 Dec 2023 13:49:22 -0700 Subject: [PATCH 17/41] added debug info --- tdrs-backend/celery_start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 541e20011..16c926598 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash +echo starting celery celery -A tdpservice.settings worker -c 1 & sleep 5 # TODO: Uncomment the following line to add flower service when memory limitation is resolved +echo "REDIS_URI: $REDIS_URI" celery -A tdpservice.settings --broker=$REDIS_URI flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & From fa97c70ef26e703cd3bf42bfb1d6b8e16cc0e1ad Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 20 Dec 2023 10:23:54 -0700 Subject: [PATCH 18/41] trying out unsetting REDIS_URI --- scripts/deploy-backend.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index baf2bc57b..7e068c982 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -122,6 +122,9 @@ update_backend() { set_cf_envs $CGAPPNAME_BACKEND set_cf_envs $CGAPPNAME_CELERY + + cf unset-env "$CGAPPNAME_BACKEND" "REDIS_URI" + cf unset-env "$CGAPPNAME_CELERY" "REDIS_URI" cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" From 849f42eff0aa7e234eaf020ec05ed4cee7c85570 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 20 Dec 2023 11:26:56 -0700 Subject: [PATCH 19/41] trying to hardcode REDIS_URI in start script --- tdrs-backend/celery_start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 16c926598..1743bc455 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +export REDIS_URI=rediss://xtJnXoZpeLNID24pQhpPb9HSr@master.prd-3fa9afec-b332-43c8-9a46-0aaa72c49d53.pw1qnn.usgw1.cache.amazonaws.com:6379 echo starting celery celery -A tdpservice.settings worker -c 1 & sleep 5 From c672ef17262f89a292bced9a12fcf691329bf9af Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 20 Dec 2023 13:14:34 -0700 Subject: [PATCH 20/41] removing unset REDIS_URI commands and hardcoded redis --- scripts/deploy-backend.sh | 3 --- tdrs-backend/celery_start.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 7e068c982..baf2bc57b 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -122,9 +122,6 @@ update_backend() { set_cf_envs $CGAPPNAME_BACKEND set_cf_envs $CGAPPNAME_CELERY - - cf unset-env "$CGAPPNAME_BACKEND" "REDIS_URI" - cf unset-env "$CGAPPNAME_CELERY" "REDIS_URI" cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 1743bc455..34fdd557a 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export REDIS_URI=rediss://xtJnXoZpeLNID24pQhpPb9HSr@master.prd-3fa9afec-b332-43c8-9a46-0aaa72c49d53.pw1qnn.usgw1.cache.amazonaws.com:6379 +# export REDIS_URI=rediss://xtJnXoZpeLNID24pQhpPb9HSr@master.prd-3fa9afec-b332-43c8-9a46-0aaa72c49d53.pw1qnn.usgw1.cache.amazonaws.com:6379 echo starting celery celery -A tdpservice.settings worker -c 1 & sleep 5 From 55aa406163fbf6dd57fb296b68b045552b4a6148 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 22 Dec 2023 12:05:49 -0700 Subject: [PATCH 21/41] setting last celery task to not be backgrounded. --- tdrs-backend/celery_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 34fdd557a..2e64d3992 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -7,4 +7,4 @@ sleep 5 # TODO: Uncomment the following line to add flower service when memory limitation is resolved echo "REDIS_URI: $REDIS_URI" celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler From 606e60c3aed63f1ad98a94dcff183a0c61d98ff3 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 26 Dec 2023 11:03:26 -0700 Subject: [PATCH 22/41] no route needed or port health checking for celery worker. --- tdrs-backend/manifest.celery.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/manifest.celery.yml b/tdrs-backend/manifest.celery.yml index 2f75241d1..3b27f3211 100755 --- a/tdrs-backend/manifest.celery.yml +++ b/tdrs-backend/manifest.celery.yml @@ -4,6 +4,8 @@ applications: memory: 2G instances: 1 disk_quota: 2G + no-route: true + health-check-type: process buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 From 387f91b623aa033b1675e8cfac7484e518a87c4f Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 2 Jan 2024 09:06:20 -0700 Subject: [PATCH 23/41] Fixed misnamed variables --- scripts/deploy-frontend.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh index 5139b7ba9..67bd83954 100755 --- a/scripts/deploy-frontend.sh +++ b/scripts/deploy-frontend.sh @@ -7,8 +7,8 @@ ENV=${2} DEPLOY_STRATEGY=${3-'tbd'} -CGAPPNAME_FRONTEND="tdp-frontend-${ENV}" -CGAPPNAME_BACKEND="tdp-backend-${ENV}" +CGHOSTNAME_FRONTEND="tdp-frontend-${ENV}" +CGHOSTNAME_BACKEND="tdp-backend-${ENV}" [[ $ENV = "prod" ]] && BUILD_ENV="production" || BUILD_ENV="development" From 57e91b3edb367124764643f4f4b2247cc896d763 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 3 Jan 2024 11:00:37 -0700 Subject: [PATCH 24/41] added backend appname to env for celery and backend to share during document saving and searching in s3 --- scripts/deploy-backend.sh | 3 +++ tdrs-backend/tdpservice/settings/cloudgov.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index baf2bc57b..76259e8c3 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -122,6 +122,9 @@ update_backend() { set_cf_envs $CGAPPNAME_BACKEND set_cf_envs $CGAPPNAME_CELERY + # Let Celery know backend app name for s3 file searching + cf set-env "$CGAPPNAME_BACKEND" CGAPPNAME_BACKEND $CGAPPNAME_BACKEND + cf set-env "$CGAPPNAME_CELERY" CGAPPNAME_BACKEND $CGAPPNAME_BACKEND cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 935fafe09..6cd2b45c8 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -36,7 +36,7 @@ class CloudGov(Common): # Cloud.gov exposes variables for the application and bound services via # VCAP_APPLICATION and VCAP_SERVICES environment variables, respectively. cloudgov_app = get_json_env_var('VCAP_APPLICATION') - APP_NAME = cloudgov_app.get('application_name') + APP_NAME = get_json_env_var('CGAPPNAME_BACKEND') cloudgov_services = get_json_env_var('VCAP_SERVICES') From 2e8fa9fcf42034b762c9c8c6d5909303a6584f3c Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 3 Jan 2024 11:17:12 -0700 Subject: [PATCH 25/41] put quotes around env vars --- scripts/deploy-backend.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 76259e8c3..5a0aa58c4 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -123,8 +123,8 @@ update_backend() { set_cf_envs $CGAPPNAME_BACKEND set_cf_envs $CGAPPNAME_CELERY # Let Celery know backend app name for s3 file searching - cf set-env "$CGAPPNAME_BACKEND" CGAPPNAME_BACKEND $CGAPPNAME_BACKEND - cf set-env "$CGAPPNAME_CELERY" CGAPPNAME_BACKEND $CGAPPNAME_BACKEND + cf set-env "$CGAPPNAME_BACKEND" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" + cf set-env "$CGAPPNAME_CELERY" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" From 71552f4e5c83e83eb9a3305e8987ba4e946758c8 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 3 Jan 2024 11:44:38 -0700 Subject: [PATCH 26/41] pass env backend name before and after. --- scripts/deploy-backend.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 5a0aa58c4..b01c8a7c0 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -88,6 +88,11 @@ generate_jwt_cert() { update_backend() { cd tdrs-backend || exit cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" + cf unset-env "$CGAPPNAME_BACKEND" "CGAPPNAME_BACKEND" + cf unset-env "$CGAPPNAME_CELERY" "CGAPPNAME_BACKEND" + # Let Celery know backend app name for s3 file searching + cf set-env "$CGAPPNAME_BACKEND" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" + cf set-env "$CGAPPNAME_CELERY" CGAPPNAME_BACKEND "$CGAPPNAME_BACKEND" if [[ $CF_SPACE == 'tanf-prod' ]]; then cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" From 663f761d6f4c4065727401b25e1a48693fc750b6 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 3 Jan 2024 12:11:42 -0700 Subject: [PATCH 27/41] don't need to serialize a basic string for appname --- tdrs-backend/tdpservice/settings/cloudgov.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 6cd2b45c8..ec80ed19e 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -36,7 +36,7 @@ class CloudGov(Common): # Cloud.gov exposes variables for the application and bound services via # VCAP_APPLICATION and VCAP_SERVICES environment variables, respectively. cloudgov_app = get_json_env_var('VCAP_APPLICATION') - APP_NAME = get_json_env_var('CGAPPNAME_BACKEND') + APP_NAME = os.getenv('CGAPPNAME_BACKEND', '{}') cloudgov_services = get_json_env_var('VCAP_SERVICES') From c21a04ecb51ffa9fc20814583bcd8044f65346d0 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 3 Jan 2024 18:32:44 -0700 Subject: [PATCH 28/41] added some documentation for Celery and Redis. --- docs/Technical-Documentation/redis-and-celery.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/Technical-Documentation/redis-and-celery.md diff --git a/docs/Technical-Documentation/redis-and-celery.md b/docs/Technical-Documentation/redis-and-celery.md new file mode 100644 index 000000000..e69de29bb From f10e2f56e251666ba679dac8efddd4722a29f663 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 4 Jan 2024 12:23:05 -0700 Subject: [PATCH 29/41] updated docker compose files to create celery to mimic new celery instance --- .../redis-and-celery.md | 11 +++++ tdrs-backend/docker-compose.local.yml | 48 ++++++++++++++++++- tdrs-backend/docker-compose.yml | 47 +++++++++++++++++- tdrs-backend/tdpservice/settings/celery.py | 17 +++---- 4 files changed, 112 insertions(+), 11 deletions(-) diff --git a/docs/Technical-Documentation/redis-and-celery.md b/docs/Technical-Documentation/redis-and-celery.md index e69de29bb..5c09a65ca 100644 --- a/docs/Technical-Documentation/redis-and-celery.md +++ b/docs/Technical-Documentation/redis-and-celery.md @@ -0,0 +1,11 @@ +# Redis Service and Celery Instance + +We use a CloudFoundry Redis service and a separate instance to run celery to run background processes like parsing data-file documents that have been submitted and have passed the [ClamAV scan](./clamav.md). + +## Redis Deployment + +As part of the move towards each environment being self-contained, one redis service is created per environment, deployed through the [CircleCI pipeline](./circle-ci.md), defined using [terraform](../../terraform/README.md). + +## Celery Deployment + +Celery is deployed at the same time as the backend through the [CircleCI pipeline](./circle-ci.md), with the details configured in the [celery manifest](../../tdrs-backend/manifest.celery.yml) and the [deploy-backend script](../../scripts/deploy-backend.sh) \ No newline at end of file diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index 3c8e76317..06c282a08 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -80,8 +80,7 @@ services: build: . command: > bash -c "./wait_for_services.sh && - ./gunicorn_start.sh && - celery -A tdpservice.settings worker -l info" + ./gunicorn_start.sh &&" ports: - "5555:5555" depends_on: @@ -89,6 +88,7 @@ services: - postgres - redis-server - elastic + - celery redis-server: image: "redis:alpine" @@ -98,6 +98,50 @@ services: volumes: - .:/tdpapp + celery: + restart: always + env_file: + - .env + environment: + - CLAMAV_NEEDED + - AV_SCAN_URL=http://clamav-rest:9000/scan + - DB_HOST=postgres + - DB_NAME=tdrs_test + - DB_PASSWORD=something_secure + - DB_PORT=5432 + - DB_USER=tdpuser + - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Local} + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-tdp-dev-insecure} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-tdpservice.settings.local} + - LOCALSTACK_HOST=localstack + - DJANGO_SU_NAME + - JWT_CERT_TEST + - JWT_KEY + - USE_LOCALSTACK + - LOGGING_LEVEL + - AMS_CLIENT_ID + - AMS_CLIENT_SECRET + - AMS_CONFIGURATION_ENDPOINT + - ACFTITAN_HOST + - ACFTITAN_KEY + - ACFTITAN_USERNAME + - REDIS_URI=redis://redis-server:6379 + - REDIS_SERVER_LOCAL=TRUE + - ACFTITAN_SFTP_PYTEST + - SENDGRID_API_KEY + volumes: + - .:/tdpapp + image: tdp + build: . + command: > + bash -c "./wait_for_services.sh && + ./celery_start.sh" + depends_on: + - localstack + - postgres + - redis-server + - elastic + volumes: localstack_data: postgres_data: diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 9101d21d5..bece067be 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -96,7 +96,7 @@ services: ./manage.py makemigrations && ./manage.py migrate && ./manage.py populate_stts && - ./gunicorn_start.sh && celery -A tdpservice.settings worker -l info" + ./gunicorn_start.sh" ports: - "5555:5555" tty: true @@ -106,6 +106,7 @@ services: - postgres - redis-server - elastic + - celery redis-server: image: "redis:alpine" @@ -115,6 +116,50 @@ services: volumes: - .:/tdpapp + celery: + restart: always + env_file: + - .env + environment: + - CLAMAV_NEEDED + - AV_SCAN_URL=http://clamav-rest:9000/scan + - DB_HOST=postgres + - DB_NAME=tdrs_test + - DB_PASSWORD=something_secure + - DB_PORT=5432 + - DB_USER=tdpuser + - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Local} + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-tdp-dev-insecure} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-tdpservice.settings.local} + - LOCALSTACK_HOST=localstack + - DJANGO_SU_NAME + - JWT_CERT_TEST + - JWT_KEY + - USE_LOCALSTACK + - LOGGING_LEVEL + - AMS_CLIENT_ID + - AMS_CLIENT_SECRET + - AMS_CONFIGURATION_ENDPOINT + - ACFTITAN_HOST + - ACFTITAN_KEY + - ACFTITAN_USERNAME + - REDIS_URI=redis://redis-server:6379 + - REDIS_SERVER_LOCAL=TRUE + - ACFTITAN_SFTP_PYTEST + - SENDGRID_API_KEY + volumes: + - .:/tdpapp + image: tdp + build: . + command: > + bash -c "./wait_for_services.sh && + ./celery_start.sh" + depends_on: + - localstack + - postgres + - redis-server + - elastic + volumes: localstack_data: postgres_data: diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 1a635bd69..bc7703007 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -20,14 +20,15 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # disable ssl verification -app.conf.update( - broker_use_ssl={ - 'ssl_cert_reqs': ssl.CERT_NONE, - }, - redis_backend_use_ssl={ - 'ssl_cert_reqs': ssl.CERT_NONE, - }, -) +if not settings.USE_LOCALSTACK + app.conf.update( + broker_use_ssl={ + 'ssl_cert_reqs': ssl.CERT_NONE, + }, + redis_backend_use_ssl={ + 'ssl_cert_reqs': ssl.CERT_NONE, + }, + ) # Load task modules from all registered Django apps. app.autodiscover_tasks() From 673a3389d77e558469d7171c92cbd5c3328e49e1 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 4 Jan 2024 15:23:49 -0700 Subject: [PATCH 30/41] remove .env requirement for circleci celery build --- tdrs-backend/docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index bece067be..9eabd8f3b 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -96,7 +96,7 @@ services: ./manage.py makemigrations && ./manage.py migrate && ./manage.py populate_stts && - ./gunicorn_start.sh" + ./gunicorn_start.shg" ports: - "5555:5555" tty: true @@ -118,8 +118,6 @@ services: celery: restart: always - env_file: - - .env environment: - CLAMAV_NEEDED - AV_SCAN_URL=http://clamav-rest:9000/scan From ae02550adf731b185ffa8844243d24efda10c22f Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 4 Jan 2024 15:30:55 -0700 Subject: [PATCH 31/41] fix typo" --- tdrs-backend/tdpservice/settings/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index bc7703007..df9864dd5 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -20,7 +20,7 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # disable ssl verification -if not settings.USE_LOCALSTACK +if not settings.USE_LOCALSTACK: app.conf.update( broker_use_ssl={ 'ssl_cert_reqs': ssl.CERT_NONE, From 7c1015f0e6597281011e4629db25034725b741bf Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 4 Jan 2024 15:38:59 -0700 Subject: [PATCH 32/41] import settings --- tdrs-backend/tdpservice/settings/celery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index df9864dd5..8ee07120a 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -3,6 +3,7 @@ import os import ssl import configurations +import settings from celery import Celery From 1d08d18900eb6e8e77e9e3dc3660ef150a25fa08 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 5 Jan 2024 11:13:27 -0700 Subject: [PATCH 33/41] import settings --- tdrs-backend/tdpservice/settings/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 8ee07120a..63fccb722 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -3,7 +3,7 @@ import os import ssl import configurations -import settings +from django.conf import settings from celery import Celery From c091a2cff5544b641cfe1dc8ea4537717052ac7d Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 5 Jan 2024 11:33:22 -0700 Subject: [PATCH 34/41] linting --- tdrs-backend/tdpservice/settings/cloudgov.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index ec80ed19e..23436733a 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -4,7 +4,6 @@ import os from requests_aws4auth import AWS4Auth from elasticsearch import RequestsHttpConnection - from tdpservice.settings.common import Common import logging @@ -16,7 +15,6 @@ def get_json_env_var(variable_name): os.getenv(variable_name, '{}') ) - def get_cloudgov_service_creds_by_instance_name(services, instance_name): """Retrieve credentials for a bound Cloud.gov service by instance name.""" return next( @@ -43,7 +41,7 @@ class CloudGov(Common): cloudgov_space = cloudgov_app.get('space_name', 'tanf-dev') cloudgov_space_suffix = cloudgov_space.strip('tanf-') cloudgov_name = cloudgov_app.get('name').split("-")[-1] # converting "tdp-backend-name" to just "name" - + services_basename = cloudgov_name if ( cloudgov_name == "develop" and cloudgov_space_suffix == "staging" ) else cloudgov_space_suffix @@ -139,7 +137,6 @@ class CloudGov(Common): 'es' ) - # Elastic ELASTICSEARCH_DSL = { 'default': { @@ -158,7 +155,6 @@ class CloudGov(Common): CELERY_BROKER_URL = REDIS_URI + '/0' CELERY_RESULT_BACKEND = REDIS_URI + '/1' - class Development(CloudGov): """Settings for applications deployed in the Cloud.gov dev space.""" From 4e13d1a6bb95d6ae926c8e72a141324a9f913cb4 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 9 Jan 2024 11:29:17 -0700 Subject: [PATCH 35/41] fixed typo, removed unneeded comments --- tdrs-backend/celery_start.sh | 2 -- tdrs-backend/docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 2e64d3992..4ce1b03c7 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -1,10 +1,8 @@ #!/usr/bin/env bash -# export REDIS_URI=rediss://xtJnXoZpeLNID24pQhpPb9HSr@master.prd-3fa9afec-b332-43c8-9a46-0aaa72c49d53.pw1qnn.usgw1.cache.amazonaws.com:6379 echo starting celery celery -A tdpservice.settings worker -c 1 & sleep 5 -# TODO: Uncomment the following line to add flower service when memory limitation is resolved echo "REDIS_URI: $REDIS_URI" celery -A tdpservice.settings --broker=$REDIS_URI flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 9eabd8f3b..bdc9f030d 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -96,7 +96,7 @@ services: ./manage.py makemigrations && ./manage.py migrate && ./manage.py populate_stts && - ./gunicorn_start.shg" + ./gunicorn_start.sh" ports: - "5555:5555" tty: true From 6f326574fa544bd1201046a3c4c4cc0c99cd1b45 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Wed, 10 Jan 2024 12:21:00 -0700 Subject: [PATCH 36/41] use 3 concurrent celery workers on the app --- tdrs-backend/celery_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 4ce1b03c7..13c2500e9 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash echo starting celery -celery -A tdpservice.settings worker -c 1 & +celery -A tdpservice.settings worker -c 3 & sleep 5 echo "REDIS_URI: $REDIS_URI" celery -A tdpservice.settings --broker=$REDIS_URI flower & From ed66a8874ff6d1d1b46929a9fd00468af9e0647b Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 11 Jan 2024 13:53:26 -0700 Subject: [PATCH 37/41] allowing routing to port 8080 and configuring flower to run on 8080 in order to expose the flower /metrics endpoint --- tdrs-backend/celery_start.sh | 2 +- tdrs-backend/manifest.celery.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 13c2500e9..88b4aba21 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -4,5 +4,5 @@ echo starting celery celery -A tdpservice.settings worker -c 3 & sleep 5 echo "REDIS_URI: $REDIS_URI" -celery -A tdpservice.settings --broker=$REDIS_URI flower & +celery -A tdpservice.settings --broker=$REDIS_URI flower --port=8080 & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/tdrs-backend/manifest.celery.yml b/tdrs-backend/manifest.celery.yml index 3b27f3211..2f75241d1 100755 --- a/tdrs-backend/manifest.celery.yml +++ b/tdrs-backend/manifest.celery.yml @@ -4,8 +4,6 @@ applications: memory: 2G instances: 1 disk_quota: 2G - no-route: true - health-check-type: process buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.8.3 From e88d422fddbf969f4360aedf8e2fa16ce04b79ee Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 11 Jan 2024 14:14:50 -0700 Subject: [PATCH 38/41] map route for celery --- scripts/deploy-backend.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index b01c8a7c0..d6d2cc9ff 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -125,6 +125,9 @@ update_backend() { cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 fi + #allow deve envs to monitor celery through flower/prometheus + cf map-route "$CGAPPNAME_CELERY" app.cloud.gov --hostname "${CGAPPNAME_CELERY}" + set_cf_envs $CGAPPNAME_BACKEND set_cf_envs $CGAPPNAME_CELERY # Let Celery know backend app name for s3 file searching From 7f8833a8f97bd13318d31e7a68b0fd9b10bd87d0 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 11 Jan 2024 14:54:17 -0700 Subject: [PATCH 39/41] wrapped map routing for celery to non-prod space --- scripts/deploy-backend.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index d6d2cc9ff..0a085abe0 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -125,8 +125,10 @@ update_backend() { cf push "$CGAPPNAME_CELERY" --no-route -f manifest.celery.yml -t 180 fi - #allow deve envs to monitor celery through flower/prometheus - cf map-route "$CGAPPNAME_CELERY" app.cloud.gov --hostname "${CGAPPNAME_CELERY}" + if [[ ! "$CF_SPACE" == "tanf-prod" ]]; then + #allow dev envs to monitor celery through flower/prometheus + cf map-route "$CGAPPNAME_CELERY" app.cloud.gov --hostname "${CGAPPNAME_CELERY}" + fi set_cf_envs $CGAPPNAME_BACKEND set_cf_envs $CGAPPNAME_CELERY From ff069d2a6e47f449c1b4dcb16d5dc2e5680aefba Mon Sep 17 00:00:00 2001 From: George Hudson Date: Fri, 12 Jan 2024 14:55:11 -0700 Subject: [PATCH 40/41] updated tf for staging and prod spaces --- terraform/dev/variables.tf | 2 +- terraform/production/main.tf | 12 ++++++++++++ terraform/staging/main.tf | 12 ++++++++++++ terraform/staging/variables.tf | 6 ++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf index 58173eaf1..fa44ceef2 100644 --- a/terraform/dev/variables.tf +++ b/terraform/dev/variables.tf @@ -39,6 +39,6 @@ variable "cf_user" { variable "dev_app_names" { type = list(string) - description = "list of app names deployed in the dev environment" + description = "list of app names deployed in the dev cf space" default = ["a11y", "qasp", "raft", "sandbox"] } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 6948ecd72..b22592505 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -75,3 +75,15 @@ resource "cloudfoundry_service_instance" "datafiles" { service_plan = data.cloudfoundry_service.s3.service_plans["basic"] recursive_delete = true } + +data "cloudfoundry_service" "redis" { + name = "aws-elasticache-redis" +} + +resource "cloudfoundry_service_instance" "redis" { + name = "tdp-redis-prod" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.redis.service_plans["PLACEHOLDER"] + # before prod deploy choose one of the following redis type: + # [redis-dev, redis-3node, redis-5node, redis-3node-large, redis-5node-large] +} diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 7b6d45a1f..29beb31f9 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -75,3 +75,15 @@ resource "cloudfoundry_service_instance" "datafiles" { service_plan = data.cloudfoundry_service.s3.service_plans["basic-sandbox"] recursive_delete = true } + +data "cloudfoundry_service" "redis" { + name = "aws-elasticache-redis" +} + +resource "cloudfoundry_service_instance" "redis" { + for_each = toset(var.staging_app_names) + name = "tdp-redis-${each.value}" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] +} + diff --git a/terraform/staging/variables.tf b/terraform/staging/variables.tf index 824c3ebf6..c61902eeb 100644 --- a/terraform/staging/variables.tf +++ b/terraform/staging/variables.tf @@ -36,3 +36,9 @@ variable "cf_app_name" { type = string description = "name of app" } + +variable "staging_app_names" { + type = list(string) + description = "list of app names deployed in the staging cf space" + default = ["develop", "staging"] +} From 10cadcdcc8c703e90c8fecc258825fdaf11ec6ff Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 30 Jan 2024 10:59:47 -0500 Subject: [PATCH 41/41] one redis per space --- scripts/deploy-backend.sh | 4 ++-- terraform/dev/main.tf | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 0a085abe0..cb2270930 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -169,8 +169,8 @@ bind_backend_to_services() { cf bind-service "$CGAPPNAME_CELERY" "tdp-db-${space}" # bind to redis - cf bind-service "$CGAPPNAME_BACKEND" "tdp-redis-${ENV}" - cf bind-service "$CGAPPNAME_CELERY" "tdp-redis-${ENV}" + cf bind-service "$CGAPPNAME_BACKEND" "tdp-redis-${space}" + cf bind-service "$CGAPPNAME_CELERY" "tdp-redis-${space}" # bind to elastic-search cf bind-service "$CGAPPNAME_BACKEND" "es-${ENV}" cf bind-service "$CGAPPNAME_CELERY" "es-${ENV}" diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index da1df5b10..0db1e94d5 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -86,8 +86,7 @@ data "cloudfoundry_service" "redis" { } resource "cloudfoundry_service_instance" "redis" { - for_each = toset(var.dev_app_names) - name = "tdp-redis-${each.value}" + name = "tdp-redis-develop" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.redis.service_plans["redis-dev"] }