feature : Nginx NOT stopped running although there is critical errors…

… on Consul or Registrator, even they die.

.docker/nginx/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:latest
+FROM nginx:1.25.4
 
 RUN apt-get update -qqy && apt-get -qqy --force-yes install curl runit wget unzip vim logrotate && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/*

.docker/nginx/entrypoint.sh

@@ -52,13 +52,22 @@ fi
 echo "[INSIDE_NGINX_CONTAINER][NOTICE] Locate the template file for ${protocol}."
 sleep 3
 cp -f /ctmpl/${protocol}/nginx.conf.ctmpl /etc/consul-templates
+cp -f /ctmpl/${protocol}/nginx.conf.contingency /etc/consul-templates
 
-sed -i -e "s/###EXPOSE_PORT###/${expose_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "expose_port (${expose_port}) replacement failure." && exit 1)
-sed -i -e "s/###APP_PORT###/${app_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "app_port (${app_port}) replacement failure." && exit 1)
+sed -i -e "s/###EXPOSE_PORT###/${expose_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "expose_port (${expose_port}) replacement (ctmpl) failure." && exit 1)
+sed -i -e "s/###EXPOSE_PORT###/${expose_port}/" /etc/consul-templates/nginx.conf.contingency || (echo "expose_port (${expose_port}) replacement (contingency) failure." && exit 1)
 
-sed -i -e "s/###PROJECT_NAME###/${project_name}/g" /etc/consul-templates/nginx.conf.ctmpl || (echo "project_name (${project_name}) replacement failure." && exit 1)
-sed -i -e "s/###CONSUL_KEY###/${consul_key}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "consul_key (${consul_key}) replacement failure." && exit 1)
-sed -i -e "s/###NGINX_CLIENT_MAX_BODY_SIZE###/${nginx_client_max_body_size}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "nginx_client_max_body_size (${nginx_client_max_body_size}) replacement failure." && exit 1)
+sed -i -e "s/###APP_PORT###/${app_port}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "app_port (${app_port}) replacement (ctmpl) failure." && exit 1)
+sed -i -e "s/###APP_PORT###/${app_port}/" /etc/consul-templates/nginx.conf.contingency || (echo "app_port (${app_port}) replacement (contingency) failure." && exit 1)
+
+sed -i -e "s/###PROJECT_NAME###/${project_name}/g" /etc/consul-templates/nginx.conf.ctmpl || (echo "project_name (${project_name}) replacement (ctmpl) failure." && exit 1)
+sed -i -e "s/###PROJECT_NAME###/${project_name}/g" /etc/consul-templates/nginx.conf.contingency || (echo "project_name (${project_name}) replacement (contingency) failure." && exit 1)
+
+sed -i -e "s/###CONSUL_KEY###/${consul_key}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "consul_key (${consul_key}) replacement (ctmpl) failure." && exit 1)
+sed -i -e "s/###CONSUL_KEY###/${consul_key}/" /etc/consul-templates/nginx.conf.contingency || (echo "consul_key (${consul_key}) replacement (contingency) failure." && exit 1)
+
+sed -i -e "s/###NGINX_CLIENT_MAX_BODY_SIZE###/${nginx_client_max_body_size}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "nginx_client_max_body_size (${nginx_client_max_body_size}) replacement (ctmpl) failure." && exit 1)
+sed -i -e "s/###NGINX_CLIENT_MAX_BODY_SIZE###/${nginx_client_max_body_size}/" /etc/consul-templates/nginx.conf.contingency || (echo "nginx_client_max_body_size (${nginx_client_max_body_size}) replacement (contingency) failure." && exit 1)
 
 use_nginx_restricted_location=$(printenv USE_NGINX_RESTRICTED_LOCATION)
 nginx_restricted_location=$(printenv NGINX_RESTRICTED_LOCATION)
@@ -90,9 +99,29 @@ if [[ ${use_nginx_restricted_location} = 'true' ]]; then
           proxy_connect_timeout 75s; \
       }" /etc/consul-templates/nginx.conf.ctmpl
 
+  sed -i -e "/###USE_NGINX_RESTRICTED_LOCATION###/c \
+      location ${nginx_restricted_location} { \
+          add_header Pragma no-cache; \
+          add_header Cache-Control no-cache; \
+  \
+                          auth_basic           \"Restricted\"; \
+                          auth_basic_user_file /etc/nginx/custom-files/.htpasswd; \
+  \
+          proxy_pass ${protocol}://${project_name}-###APP_STATE###:${app_port}; \
+          proxy_set_header Host \$http_host; \
+          proxy_set_header X-Scheme \$scheme; \
+          proxy_set_header X-Forwarded-Protocol \$scheme; \
+          proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; \
+          proxy_set_header X-Real-IP \$remote_addr; \
+          proxy_http_version 1.1; \
+          proxy_read_timeout 300s; \
+          proxy_connect_timeout 75s; \
+      }" /etc/consul-templates/nginx.conf.contingency
+
 
 else
-  sed -i -e "s/###USE_NGINX_RESTRICTED_LOCATION###//" /etc/consul-templates/nginx.conf.ctmpl || (echo "use_nginx_restricted_location=false (${use_nginx_restricted_location}) replacement failure." && exit 1)
+  sed -i -e "s/###USE_NGINX_RESTRICTED_LOCATION###//" /etc/consul-templates/nginx.conf.ctmpl || (echo "use_nginx_restricted_location=false (${use_nginx_restricted_location}) replacement (ctmpl) failure." && exit 1)
+  sed -i -e "s/###USE_NGINX_RESTRICTED_LOCATION###//" /etc/consul-templates/nginx.conf.contingency || (echo "use_nginx_restricted_location=false (${use_nginx_restricted_location}) replacement (contingency) failure." && exit 1)
 fi
 
 if [[ ${protocol} = 'https' ]]; then
@@ -149,7 +178,8 @@ if [[ ${protocol} = 'https' ]]; then
     # sleep 1
     #sed -i -e "s/###APP_HOST###/${app_host}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "APP_HOST on .env failed to be applied." && exit 1)
     #sleep 1
-    sed -i -e "s/###COMMERCIAL_SSL_NAME###/${commercial_ssl_name}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "commercial_ssl_name (${commercial_ssl_name}) on .env failed to be applied." && exit 1)
+    sed -i -e "s/###COMMERCIAL_SSL_NAME###/${commercial_ssl_name}/" /etc/consul-templates/nginx.conf.ctmpl || (echo "commercial_ssl_name (${commercial_ssl_name}) on .env failed to be applied. (ctmpl)" && exit 1)
+    sed -i -e "s/###COMMERCIAL_SSL_NAME###/${commercial_ssl_name}/" /etc/consul-templates/nginx.conf.contingency || (echo "commercial_ssl_name (${commercial_ssl_name}) on .env failed to be applied. (contingency)" && exit 1)
 fi
 
 
@@ -174,6 +204,12 @@ for retry_count in {1..5}; do
   echo "[INSIDE_NGINX_CONTAINER][NOTICE] Retry four times with a three-second interval... (retrying ${retry_count} times...)"
   sleep 3
 done
+echo "[INSIDE_NGINX_CONTAINER][NOTICE] Creating 'nginx.conf.contingency.blue', 'nginx.conf.contingency.green' ..."
+cp -f /etc/consul-templates/nginx.conf.contingency /etc/consul-templates/nginx.conf.contingency.blue || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.blue" && exit 1)
+sed -i -e "s/###APP_STATE###/blue/" /etc/consul-templates/nginx.conf.contingency.blue || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.blue (2)" && exit 1)
+cp -f /etc/consul-templates/nginx.conf.contingency /etc/consul-templates/nginx.conf.contingency.green || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.green" && exit 1)
+sed -i -e "s/###APP_STATE###/green/" /etc/consul-templates/nginx.conf.contingency.green || (echo "Failed in creating /etc/consul-templates/nginx.conf.contingency.green (2)" && exit 1)
+
 echo "[INSIDE_NGINX_CONTAINER][NOTICE] Applying the Nginx template..."
 bash /etc/service/consul-template/run/consul-template.service
 echo "[INSIDE_NGINX_CONTAINER][NOTICE] Start the Nginx."

README.md

@@ -184,11 +184,15 @@ bash check-current-states.sh | grep -o '[^_]state : [^,]*,'
 ```shell
 # Automatically set the safe state & down and up Nginx
 bash emergency-nginx-down-and-up.sh
+
 # In case you need to manually set the Nginx to point to 'blue' or 'green'
 bash emergency-nginx-down-and-up.sh blue
 ## OR
 bash emergency-nginx-down-and-up.sh green
 
+# If the script above fails, recreate & reset all about Nginx settings.
+bash emergency-nginx-restart.sh
+
 # If the script above fails, set *NGINX_RESTART to be true on .env. and..
 sudo bash run.sh
 
@@ -268,7 +272,7 @@ bash emergency-consul-down-and-up.sh
     apply_ports_onto_nginx_yaml
     apply_docker_compose_volumes_onto_app_nginx_yaml
     create_nginx_ctmpl
-
+    create_nginx_contingency_conf
     backup_nginx_to_previous_images
   fi
 

activate.sh

@@ -14,7 +14,12 @@ new_upstream=$3
 consul_key_value_store=$4
 
 echo "[NOTICE] new_state : ${new_state}, old_state : ${old_state}, new_upstream : ${new_upstream}, consul_key_value_store : ${consul_key_value_store}"
-was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw) || (echo "[EMERGENCY] Errors on Nginx or Consul Network. 1) Nginx : check logs above & docker logs -f... 2) Consul : This usually occurs when the physical machine has been restarted. (Solution : run bash 'stop-all-containers.sh' & 'emergency-consul-down-and-up.sh' & 'run.sh')" && exit 1)
+# was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw) || (echo "[EMERGENCY] Errors on Nginx or Consul Network. 1) Nginx : check logs above & docker logs -f... 2) Consul : This usually occurs when the physical machine has been restarted. (Solution : run bash 'stop-all-containers.sh' & 'emergency-consul-down-and-up.sh' & 'run.sh')" && exit 1)
+was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw) || {
+    echo "[EMERGENCY] Errors on Nginx or Consul Network. Run Nginx Contingency Plan."
+    was_state="${old_state}"
+}
+
 echo "[NOTICE] CONSUL (${consul_key_value_store}) is currently pointing to : ${was_state}"
 if [[ ${old_state} != ${was_state} ]]; then
   echo "[WARNING] Was State (${was_state}, currently pointed from CONSUL) is different from Old State (${old_state}, checked at the first stage of the mother script.)"
@@ -44,7 +49,11 @@ done
 
 echo "[NOTICE] Activate ${new_state} CONSUL. (old Nginx pids: ${pid_was})"
 echo "[NOTICE] ${new_state} is stored in CONSUL."
-docker exec ${project_name}-nginx curl -X PUT -d ${new_state} ${consul_key_value_store} >/dev/null
+docker exec ${project_name}-nginx curl -X PUT -d ${new_state} ${consul_key_value_store} >/dev/null || {
+    echo "[EMERGENCY] Set ${new_state} on nginx.conf according to the Nginx Contingency Plan."
+    docker exec ${project_name}-nginx cp -f /etc/consul-templates/nginx.conf.contingency.${new_state} /etc/nginx/conf.d/nginx.conf
+    docker exec ${project_name}-nginx sh -c 'service nginx reload || service nginx restart || [EMERGENCY] Nginx Contingency Plan failed as well. Correct /etc/nginx/conf.d/nginx.conf directly and Run "service nginx restart".'
+}
 
 sleep 1
 

docker-compose-consul.yml

@@ -7,7 +7,7 @@ services:
   consul:
     hostname: consul
     container_name: consul
-    image: hashicorp/consul:1.14.11 # https://github.com/hashicorp/consul/issues/17973
+    image: hashicorp/consul:1.14.11
     restart: always
     environment:
       - CONSUL_LOCAL_CONFIG={"disable_update_check":true}

emergency-nginx-restart.sh

@@ -41,6 +41,7 @@ apply_env_service_name_onto_nginx_yaml
 apply_ports_onto_nginx_yaml
 apply_docker_compose_volumes_onto_app_nginx_yaml
 create_nginx_ctmpl
+create_nginx_contingency_conf
 # build
 load_nginx_docker_image
 # run

reset.sh

@@ -14,8 +14,13 @@ consul_key_value_store=$1
 state=$2
 new_state=$3
 
-echo "[NOTICE] Be stored as ${state} in Consul."
-docker exec ${project_name}-nginx curl -X PUT -d ${state} ${consul_key_value_store} > /dev/null
+echo "[NOTICE] Point Nginx back to ${state} from reset.sh."
+docker exec ${project_name}-nginx curl -X PUT -d ${state} ${consul_key_value_store} > /dev/null || {
+   echo "[EMERGENCY] Set ${state} on nginx.conf according to the Nginx Contingency Plan."
+   docker exec ${project_name}-nginx cp -f /etc/consul-templates/nginx.conf.contingency.${state} /etc/nginx/conf.d/nginx.conf
+   docker exec ${project_name}-nginx sh -c 'service nginx reload || service nginx restart || [EMERGENCY] Nginx Contingency Plan failed as well. Correct /etc/nginx/conf.d/nginx.conf directly and Run "service nginx restart".'
+}
+
 
 echo "[NOTICE] Stopping the ${new_state} ${orchestration_type}"
 if [[ ${orchestration_type} != 'stack' ]]; then

run.sh

@@ -152,7 +152,8 @@ load_all_containers(){
 
   fi
 
-  check_supporting_containers_loaded || (echo "[ERROR] Fail in loading supporting containers." && exit 1)
+  check_necessary_supporting_containers_loaded || (echo "[ERROR] Fail in loading necessary supporting containers." && exit 1)
+  check_supporting_containers_loaded || (echo "[WARN] Fail in loading supporting containers. We will conduct the Nginx Contingency Plan.")
 
 }
 
@@ -196,7 +197,7 @@ _main() {
     apply_ports_onto_nginx_yaml
     apply_docker_compose_volumes_onto_app_nginx_yaml
     create_nginx_ctmpl
-
+    create_nginx_contingency_conf
     backup_nginx_to_previous_images
   fi
 

use-nginx.sh

@@ -243,10 +243,191 @@ server {
 EOF
    done
 
+   fi
+}
+
+create_nginx_contingency_conf(){
 
+   local proxy_hostname=
 
+   if [[ ${orchestration_type} == 'stack' ]]; then
+     proxy_hostname="###PROJECT_NAME###-###APP_STATE###_###PROJECT_NAME###-###APP_STATE###"
+   else
+     proxy_hostname="###PROJECT_NAME###-###APP_STATE###"
    fi
 
+    if [[ ${protocol} = 'http' ]]; then
+
+    echo "[NOTICE] NGINX template (.docker/nginx/ctmpl/${protocol}/nginx.conf.contingency) is now being created."
+
+    cat > .docker/nginx/ctmpl/http/nginx.conf.contingency <<EOF
+
+server {
+
+     listen ###EXPOSE_PORT### default_server;
+     listen [::]:###EXPOSE_PORT### default_server;
+
+     server_name localhost;
+
+     error_page 497 http://\$host:\$server_port\$request_uri;
+
+     client_max_body_size ###NGINX_CLIENT_MAX_BODY_SIZE###;
+
+     location / {
+         add_header Pragma no-cache;
+         add_header Cache-Control no-cache;
+
+         proxy_pass http://$proxy_hostname:###APP_PORT###;
+
+         proxy_set_header Host \$http_host;
+         proxy_set_header X-Scheme \$scheme;
+         proxy_set_header X-Forwarded-Protocol \$scheme;
+         proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+         proxy_set_header X-Real-IP \$remote_addr;
+         proxy_http_version 1.1;
+         proxy_read_timeout 300s;
+         proxy_connect_timeout 75s;
+     }
+
+     ###USE_NGINX_RESTRICTED_LOCATION###
+
+     access_log /var/log/nginx/access.log;
+     error_log /var/log/nginx/error.log;
+}
+EOF
+
+   for i in "${additional_ports[@]}"
+   do
+        cat >> .docker/nginx/ctmpl/http/nginx.conf.contingency <<EOF
+
+server {
+
+     listen $i default_server;
+     listen [::]:$i default_server;
+
+     server_name localhost;
+
+     error_page 497 http://\$host:\$server_port\$request_uri;
+
+     client_max_body_size ###NGINX_CLIENT_MAX_BODY_SIZE###;
+
+     location / {
+         add_header Pragma no-cache;
+         add_header Cache-Control no-cache;
+
+         proxy_pass http://$proxy_hostname:$i;
+
+         proxy_set_header Host \$http_host;
+         proxy_set_header X-Scheme \$scheme;
+         proxy_set_header X-Forwarded-Protocol \$scheme;
+         proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+         proxy_set_header X-Real-IP \$remote_addr;
+         proxy_http_version 1.1;
+         proxy_read_timeout 300s;
+         proxy_connect_timeout 75s;
+    }
+
+     access_log /var/log/nginx/access.log;
+     error_log /var/log/nginx/error.log;
+}
+EOF
+   done
+
+   else
+
+    echo "[NOTICE] NGINX template (.docker/nginx/contingency/${protocol}/nginx.conf.contingency) is now being created."
+
+    cat > .docker/nginx/ctmpl/https/nginx.conf.contingency <<EOF
+server {
+
+    listen ###EXPOSE_PORT### default_server ssl;
+    listen [::]:###EXPOSE_PORT### default_server ssl;
+
+    http2 on;
+    server_name localhost;
+
+    error_page 497 https://\$host:\$server_port\$request_uri;
+
+    client_max_body_size ###NGINX_CLIENT_MAX_BODY_SIZE###;
+
+
+    ssl_certificate /etc/nginx/ssl/###COMMERCIAL_SSL_NAME###.chained.crt;
+    ssl_certificate_key /etc/nginx/ssl/###COMMERCIAL_SSL_NAME###.key;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
+
+
+    location / {
+        add_header Pragma no-cache;
+        add_header Cache-Control no-cache;
+
+        proxy_pass https://$proxy_hostname:###APP_PORT###;
+
+        proxy_set_header Host \$http_host;
+        proxy_set_header X-Scheme \$scheme;
+        proxy_set_header X-Forwarded-Protocol \$scheme;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_http_version 1.1;
+        proxy_read_timeout 300s;
+        proxy_connect_timeout 75s;
+    }
+
+    ###USE_NGINX_RESTRICTED_LOCATION###
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+}
+EOF
+
+   for i in "${additional_ports[@]}"
+   do
+        cat >> .docker/nginx/ctmpl/https/nginx.conf.contingency <<EOF
+
+server {
+    listen $i default_server ssl;
+    listen [::]:$i default_server ssl;
+
+    http2 on;
+
+    server_name localhost;
+
+    error_page 497 https://\$host:\$server_port\$request_uri;
+
+    client_max_body_size ###NGINX_CLIENT_MAX_BODY_SIZE###;
+
+
+    ssl_certificate /etc/nginx/ssl/###COMMERCIAL_SSL_NAME###.chained.crt;
+    ssl_certificate_key /etc/nginx/ssl/###COMMERCIAL_SSL_NAME###.key;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
+
+    location / {
+        add_header Pragma no-cache;
+        add_header Cache-Control no-cache;
+
+        proxy_pass https://$proxy_hostname:$i;
+
+        proxy_set_header Host \$http_host;
+        proxy_set_header X-Scheme \$scheme;
+        proxy_set_header X-Forwarded-Protocol \$scheme;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_http_version 1.1;
+        proxy_read_timeout 300s;
+        proxy_connect_timeout 75s;
+    }
+
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+}
+EOF
+   done
+
+   fi
 }
 
 load_nginx_docker_image(){

util.sh

@@ -431,7 +431,14 @@ check_one_container_loaded(){
     fi
 }
 check_supporting_containers_loaded(){
-  all_container_names=("consul" "registrator" "${project_name}-nginx")
+  all_container_names=("consul" "registrator")
+  for name in "${all_container_names[@]}"; do
+    check_one_container_loaded ${name}
+  done
+}
+
+check_necessary_supporting_containers_loaded(){
+  all_container_names=("${project_name}-nginx")
   for name in "${all_container_names[@]}"; do
     check_one_container_loaded ${name}
   done