From b58d479dbe8878fe0d18ea3d17f207577462cb5c Mon Sep 17 00:00:00 2001 From: Andrew-Kang-G Date: Sun, 18 Feb 2024 15:41:34 +0900 Subject: [PATCH] feature : Nginx NOT stopped running although there is critical errors on Consul or Registrator, even they die. --- .docker/nginx/Dockerfile | 2 +- .docker/nginx/entrypoint.sh | 50 ++++++++-- README.md | 6 +- activate.sh | 13 ++- docker-compose-consul.yml | 2 +- emergency-nginx-restart.sh | 1 + reset.sh | 9 +- run.sh | 5 +- use-nginx.sh | 181 ++++++++++++++++++++++++++++++++++++ util.sh | 9 +- 10 files changed, 261 insertions(+), 17 deletions(-) diff --git a/.docker/nginx/Dockerfile b/.docker/nginx/Dockerfile index 6199aa2..bc39556 100644 --- a/.docker/nginx/Dockerfile +++ b/.docker/nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:latest +FROM nginx:1.25.4 RUN apt-get update -qqy && apt-get -qqy --force-yes install curl runit wget unzip vim logrotate && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/* diff --git a/.docker/nginx/entrypoint.sh b/.docker/nginx/entrypoint.sh index 7a7bdd9..ae30681 100644 --- a/.docker/nginx/entrypoint.sh +++ b/.docker/nginx/entrypoint.sh @@ -52,13 +52,22 @@ fi echo "[INSIDE_NGINX_CONTAINER][NOTICE] Locate the template file for ${protocol}." sleep 3 cp -f /ctmpl/${protocol}/nginx.conf.ctmpl /etc/consul-templates +cp -f /ctmpl/${protocol}/nginx.conf.contingency /etc/consul-templates -sed -i -e "s/###EXPOSE_PORT###/${expose_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "expose_port (${expose_port}) replacement failure." && exit 1) -sed -i -e "s/###APP_PORT###/${app_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "app_port (${app_port}) replacement failure." && exit 1) +sed -i -e "s/###EXPOSE_PORT###/${expose_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "expose_port (${expose_port}) replacement (ctmpl) failure." && exit 1) +sed -i -e "s/###EXPOSE_PORT###/${expose_port}/" /etc/consul-templates/nginx.conf.contingency || (echo "expose_port (${expose_port}) replacement (contingency) failure." && exit 1) -sed -i -e "s/###PROJECT_NAME###/${project_name}/g" /etc/consul-templates/nginx.conf.ctmpl || (echo "project_name (${project_name}) replacement failure." && exit 1) -sed -i -e "s/###CONSUL_KEY###/${consul_key}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "consul_key (${consul_key}) replacement failure." && exit 1) -sed -i -e "s/###NGINX_CLIENT_MAX_BODY_SIZE###/${nginx_client_max_body_size}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "nginx_client_max_body_size (${nginx_client_max_body_size}) replacement failure." && exit 1) +sed -i -e "s/###APP_PORT###/${app_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "app_port (${app_port}) replacement (ctmpl) failure." && exit 1) +sed -i -e "s/###APP_PORT###/${app_port}/" /etc/consul-templates/nginx.conf.contingency || (echo "app_port (${app_port}) replacement (contingency) failure." && exit 1) + +sed -i -e "s/###PROJECT_NAME###/${project_name}/g" /etc/consul-templates/nginx.conf.ctmpl || (echo "project_name (${project_name}) replacement (ctmpl) failure." && exit 1) +sed -i -e "s/###PROJECT_NAME###/${project_name}/g" /etc/consul-templates/nginx.conf.contingency || (echo "project_name (${project_name}) replacement (contingency) failure." && exit 1) + +sed -i -e "s/###CONSUL_KEY###/${consul_key}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "consul_key (${consul_key}) replacement (ctmpl) failure." && exit 1) +sed -i -e "s/###CONSUL_KEY###/${consul_key}/" /etc/consul-templates/nginx.conf.contingency || (echo "consul_key (${consul_key}) replacement (contingency) failure." && exit 1) + +sed -i -e "s/###NGINX_CLIENT_MAX_BODY_SIZE###/${nginx_client_max_body_size}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "nginx_client_max_body_size (${nginx_client_max_body_size}) replacement (ctmpl) failure." && exit 1) +sed -i -e "s/###NGINX_CLIENT_MAX_BODY_SIZE###/${nginx_client_max_body_size}/" /etc/consul-templates/nginx.conf.contingency || (echo "nginx_client_max_body_size (${nginx_client_max_body_size}) replacement (contingency) failure." && exit 1) use_nginx_restricted_location=$(printenv USE_NGINX_RESTRICTED_LOCATION) nginx_restricted_location=$(printenv NGINX_RESTRICTED_LOCATION) @@ -90,9 +99,29 @@ if [[ ${use_nginx_restricted_location} = 'true' ]]; then proxy_connect_timeout 75s; \ }" /etc/consul-templates/nginx.conf.ctmpl + sed -i -e "/###USE_NGINX_RESTRICTED_LOCATION###/c \ + location ${nginx_restricted_location} { \ + add_header Pragma no-cache; \ + add_header Cache-Control no-cache; \ + \ + auth_basic \"Restricted\"; \ + auth_basic_user_file /etc/nginx/custom-files/.htpasswd; \ + \ + proxy_pass ${protocol}://${project_name}-###APP_STATE###:${app_port}; \ + proxy_set_header Host \$http_host; \ + proxy_set_header X-Scheme \$scheme; \ + proxy_set_header X-Forwarded-Protocol \$scheme; \ + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; \ + proxy_set_header X-Real-IP \$remote_addr; \ + proxy_http_version 1.1; \ + proxy_read_timeout 300s; \ + proxy_connect_timeout 75s; \ + }" /etc/consul-templates/nginx.conf.contingency + else - sed -i -e "s/###USE_NGINX_RESTRICTED_LOCATION###//" /etc/consul-templates/nginx.conf.ctmpl || (echo "use_nginx_restricted_location=false (${use_nginx_restricted_location}) replacement failure." && exit 1) + sed -i -e "s/###USE_NGINX_RESTRICTED_LOCATION###//" /etc/consul-templates/nginx.conf.ctmpl || (echo "use_nginx_restricted_location=false (${use_nginx_restricted_location}) replacement (ctmpl) failure." && exit 1) + sed -i -e "s/###USE_NGINX_RESTRICTED_LOCATION###//" /etc/consul-templates/nginx.conf.contingency || (echo "use_nginx_restricted_location=false (${use_nginx_restricted_location}) replacement (contingency) failure." && exit 1) fi if [[ ${protocol} = 'https' ]]; then @@ -149,7 +178,8 @@ if [[ ${protocol} = 'https' ]]; then # sleep 1 #sed -i -e "s/###APP_HOST###/${app_host}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "APP_HOST on .env failed to be applied." && exit 1) #sleep 1 - sed -i -e "s/###COMMERCIAL_SSL_NAME###/${commercial_ssl_name}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "commercial_ssl_name (${commercial_ssl_name}) on .env failed to be applied." && exit 1) + sed -i -e "s/###COMMERCIAL_SSL_NAME###/${commercial_ssl_name}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "commercial_ssl_name (${commercial_ssl_name}) on .env failed to be applied. (ctmpl)" && exit 1) + sed -i -e "s/###COMMERCIAL_SSL_NAME###/${commercial_ssl_name}/" /etc/consul-templates/nginx.conf.contingency || (echo "commercial_ssl_name (${commercial_ssl_name}) on .env failed to be applied. (contingency)" && exit 1) fi @@ -174,6 +204,12 @@ for retry_count in {1..5}; do echo "[INSIDE_NGINX_CONTAINER][NOTICE] Retry four times with a three-second interval... (retrying ${retry_count} times...)" sleep 3 done +echo "[INSIDE_NGINX_CONTAINER][NOTICE] Creating 'nginx.conf.contingency.blue', 'nginx.conf.contingency.green' ..." +cp -f /etc/consul-templates/nginx.conf.contingency /etc/consul-templates/nginx.conf.contingency.blue || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.blue" && exit 1) +sed -i -e "s/###APP_STATE###/blue/" /etc/consul-templates/nginx.conf.contingency.blue || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.blue (2)" && exit 1) +cp -f /etc/consul-templates/nginx.conf.contingency /etc/consul-templates/nginx.conf.contingency.green || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.green" && exit 1) +sed -i -e "s/###APP_STATE###/green/" /etc/consul-templates/nginx.conf.contingency.green || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.green (2)" && exit 1) + echo "[INSIDE_NGINX_CONTAINER][NOTICE] Applying the Nginx template..." bash /etc/service/consul-template/run/consul-template.service echo "[INSIDE_NGINX_CONTAINER][NOTICE] Start the Nginx." diff --git a/README.md b/README.md index 51b1b7a..f9fc868 100644 --- a/README.md +++ b/README.md @@ -184,11 +184,15 @@ bash check-current-states.sh | grep -o '[^_]state : [^,]*,' ```shell # Automatically set the safe state & down and up Nginx bash emergency-nginx-down-and-up.sh + # In case you need to manually set the Nginx to point to 'blue' or 'green' bash emergency-nginx-down-and-up.sh blue ## OR bash emergency-nginx-down-and-up.sh green +# If the script above fails, recreate & reset all about Nginx settings. +bash emergency-nginx-restart.sh + # If the script above fails, set *NGINX_RESTART to be true on .env. and.. sudo bash run.sh @@ -268,7 +272,7 @@ bash emergency-consul-down-and-up.sh apply_ports_onto_nginx_yaml apply_docker_compose_volumes_onto_app_nginx_yaml create_nginx_ctmpl - + create_nginx_contingency_conf backup_nginx_to_previous_images fi diff --git a/activate.sh b/activate.sh index 20e7dfa..4ee4cae 100644 --- a/activate.sh +++ b/activate.sh @@ -14,7 +14,12 @@ new_upstream=$3 consul_key_value_store=$4 echo "[NOTICE] new_state : ${new_state}, old_state : ${old_state}, new_upstream : ${new_upstream}, consul_key_value_store : ${consul_key_value_store}" -was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw) || (echo "[EMERGENCY] Errors on Nginx or Consul Network. 1) Nginx : check logs above & docker logs -f... 2) Consul : This usually occurs when the physical machine has been restarted. (Solution : run bash 'stop-all-containers.sh' & 'emergency-consul-down-and-up.sh' & 'run.sh')" && exit 1) +# was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw) || (echo "[EMERGENCY] Errors on Nginx or Consul Network. 1) Nginx : check logs above & docker logs -f... 2) Consul : This usually occurs when the physical machine has been restarted. (Solution : run bash 'stop-all-containers.sh' & 'emergency-consul-down-and-up.sh' & 'run.sh')" && exit 1) +was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw) || { + echo "[EMERGENCY] Errors on Nginx or Consul Network. Run Nginx Contingency Plan." + was_state="${old_state}" +} + echo "[NOTICE] CONSUL (${consul_key_value_store}) is currently pointing to : ${was_state}" if [[ ${old_state} != ${was_state} ]]; then echo "[WARNING] Was State (${was_state}, currently pointed from CONSUL) is different from Old State (${old_state}, checked at the first stage of the mother script.)" @@ -44,7 +49,11 @@ done echo "[NOTICE] Activate ${new_state} CONSUL. (old Nginx pids: ${pid_was})" echo "[NOTICE] ${new_state} is stored in CONSUL." -docker exec ${project_name}-nginx curl -X PUT -d ${new_state} ${consul_key_value_store} >/dev/null +docker exec ${project_name}-nginx curl -X PUT -d ${new_state} ${consul_key_value_store} >/dev/null || { + echo "[EMERGENCY] Set ${new_state} on nginx.conf according to the Nginx Contingency Plan." + docker exec ${project_name}-nginx cp -f /etc/consul-templates/nginx.conf.contingency.${new_state} /etc/nginx/conf.d/nginx.conf + docker exec ${project_name}-nginx sh -c 'service nginx reload || service nginx restart || [EMERGENCY] Nginx Contingency Plan failed as well. Correct /etc/nginx/conf.d/nginx.conf directly and Run "service nginx restart".' +} sleep 1 diff --git a/docker-compose-consul.yml b/docker-compose-consul.yml index af77997..f90b95f 100644 --- a/docker-compose-consul.yml +++ b/docker-compose-consul.yml @@ -7,7 +7,7 @@ services: consul: hostname: consul container_name: consul - image: hashicorp/consul:1.14.11 # https://github.com/hashicorp/consul/issues/17973 + image: hashicorp/consul:1.14.11 restart: always environment: - CONSUL_LOCAL_CONFIG={"disable_update_check":true} diff --git a/emergency-nginx-restart.sh b/emergency-nginx-restart.sh index ac68588..8646b02 100644 --- a/emergency-nginx-restart.sh +++ b/emergency-nginx-restart.sh @@ -41,6 +41,7 @@ apply_env_service_name_onto_nginx_yaml apply_ports_onto_nginx_yaml apply_docker_compose_volumes_onto_app_nginx_yaml create_nginx_ctmpl +create_nginx_contingency_conf # build load_nginx_docker_image # run diff --git a/reset.sh b/reset.sh index 9e0bea9..4cc518e 100644 --- a/reset.sh +++ b/reset.sh @@ -14,8 +14,13 @@ consul_key_value_store=$1 state=$2 new_state=$3 -echo "[NOTICE] Be stored as ${state} in Consul." -docker exec ${project_name}-nginx curl -X PUT -d ${state} ${consul_key_value_store} > /dev/null +echo "[NOTICE] Point Nginx back to ${state} from reset.sh." +docker exec ${project_name}-nginx curl -X PUT -d ${state} ${consul_key_value_store} > /dev/null || { + echo "[EMERGENCY] Set ${state} on nginx.conf according to the Nginx Contingency Plan." + docker exec ${project_name}-nginx cp -f /etc/consul-templates/nginx.conf.contingency.${state} /etc/nginx/conf.d/nginx.conf + docker exec ${project_name}-nginx sh -c 'service nginx reload || service nginx restart || [EMERGENCY] Nginx Contingency Plan failed as well. Correct /etc/nginx/conf.d/nginx.conf directly and Run "service nginx restart".' +} + echo "[NOTICE] Stopping the ${new_state} ${orchestration_type}" if [[ ${orchestration_type} != 'stack' ]]; then diff --git a/run.sh b/run.sh index 7e83fa1..e0e6096 100644 --- a/run.sh +++ b/run.sh @@ -152,7 +152,8 @@ load_all_containers(){ fi - check_supporting_containers_loaded || (echo "[ERROR] Fail in loading supporting containers." && exit 1) + check_necessary_supporting_containers_loaded || (echo "[ERROR] Fail in loading necessary supporting containers." && exit 1) + check_supporting_containers_loaded || (echo "[WARN] Fail in loading supporting containers. We will conduct the Nginx Contingency Plan.") } @@ -196,7 +197,7 @@ _main() { apply_ports_onto_nginx_yaml apply_docker_compose_volumes_onto_app_nginx_yaml create_nginx_ctmpl - + create_nginx_contingency_conf backup_nginx_to_previous_images fi diff --git a/use-nginx.sh b/use-nginx.sh index daf3d1b..44d4821 100644 --- a/use-nginx.sh +++ b/use-nginx.sh @@ -243,10 +243,191 @@ server { EOF done + fi +} + +create_nginx_contingency_conf(){ + local proxy_hostname= + if [[ ${orchestration_type} == 'stack' ]]; then + proxy_hostname="###PROJECT_NAME###-###APP_STATE###_###PROJECT_NAME###-###APP_STATE###" + else + proxy_hostname="###PROJECT_NAME###-###APP_STATE###" fi + if [[ ${protocol} = 'http' ]]; then + + echo "[NOTICE] NGINX template (.docker/nginx/ctmpl/${protocol}/nginx.conf.contingency) is now being created." + + cat > .docker/nginx/ctmpl/http/nginx.conf.contingency <> .docker/nginx/ctmpl/http/nginx.conf.contingency < .docker/nginx/ctmpl/https/nginx.conf.contingency <> .docker/nginx/ctmpl/https/nginx.conf.contingency <