fix : change logic for setting which state to deploy & ! emergency-ng…

…inx-restart.sh & feature : add tests module

Proirtize cosul-pointing. No use any more 'check_availability_inside_container_speed_mode' in 'cache_all_states'.
For, emergency-nginx-restart.sh, run docker-compose down first and then dokcer-compose up -d

activate.sh

@@ -15,6 +15,9 @@ consul_key_value_store=$4
 echo "[NOTICE] new_state : ${new_state}, old_state : ${old_state}, new_upstream : ${new_upstream}, consul_key_value_store : ${consul_key_value_store}"
 was_state=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw)
 echo "[NOTICE] CONSUL (${consul_key_value_store}) is currently pointing to : ${was_state}"
+if [[ ${old_state} != ${was_state} ]]; then
+  echo "[WARNING] Was State (${was_state}, currently pointed from CONSUL) is different from Old State (${old_state}, checked at the first stage of the mother script.)"
+fi
 
 # The meaning of "${pid_was} != '-'" is that when Nginx has fully started, the BLUE-GREEN change operation is performed in CONSUL.
 echo "[NOTICE] Check if Nginx is completely UP."

emergency-nginx-restart.sh

@@ -33,7 +33,8 @@ fi
 nginx_restart(){
 
    echo "[NOTICE] Re-Run NGINX as a container."
-   PROJECT_NAME=${project_name} docker-compose -f docker-compose-${project_name}-nginx.yml up -d ${project_name}-nginx || echo "[ERROR] Critical - ${project_name}-nginx UP failure"
+   docker-compose -f docker-compose-${project_name}-nginx.yml down
+   PROJECT_NAME=${project_name} docker-compose -f docker-compose-${project_name}-nginx.yml up -d || echo "[ERROR] Critical - ${project_name}-nginx UP failure"
 
    ./activate.sh ${state_a} ${state_b} ${state_a_upstream} ${consul_key_value_store}
 }

run.sh

@@ -26,27 +26,10 @@ backup_app_to_previous_images(){
       docker tag ${project_name}:new ${project_name}:previous && return  || echo "[NOTICE] There is no 'new' tag image to backup the app."
   fi
 
+  echo "[NOTICE] Since there is no 'new' tag image for the app, depending on 'cache_all_states' previously run in the 'cache_global_vars' stage, we will back ${state} up."
+  docker tag ${project_name}:${state} ${project_name}:previous && return || echo "[NOTICE] No ${state} tagged image."
 
-  echo "[NOTICE] Since there is no 'new' tag image for the app, we will check the blue or green container and use the image of the container that is running properly as the backup image."
-  if [[ $(docker exec ${project_name}-blue printenv SERVICE_NAME 2> /dev/null) == 'blue' ]]
-  then
-      echo "[NOTICE] Checking if the blue container is running..."
-      if [[ $(check_availability_inside_container_speed_mode 'blue' 10 5 | tail -n 1) == 'true' ]]; then
-          echo "[NOTICE] Docker tag 'blue' 'previous'"
-          docker tag ${project_name}:blue ${project_name}:previous && return || echo "[NOTICE] No 'blue' tagged image."
-      fi
-  fi
-
-  if [[ $(docker exec ${project_name}-green printenv SERVICE_NAME 2> /dev/null) == 'green' ]]
-  then
-      echo "[NOTICE] Checking if the green container is running..."
-      if [[ $(check_availability_inside_container_speed_mode 'green' 10 5 | tail -n 1) == 'true' ]]; then
-        echo "[NOTICE] Docker tag 'green' 'previous'"
-        docker tag ${project_name}:green ${project_name}:previous && return || echo "[NOTICE] No 'green' tagged image."
-      fi
-  fi
-
-  echo "[NOTICE] Since there are no 'new', 'blue', and 'green' images, we will attempt to back up the latest image as previous"
+  echo "[NOTICE] Since there is no ${state} images, we will attempt to back up the latest image as previous"
   docker tag ${project_name}:latest ${project_name}:previous || echo "[NOTICE] No 'latest' tagged image."
 
 }
@@ -246,7 +229,8 @@ nginx_restart(){
    docker network rm ${project_name}_app || echo "[DEBUG] NA"
 
    echo "[NOTICE] Run NGINX as a container."
-   PROJECT_NAME=${project_name} docker-compose -f docker-compose-${project_name}-nginx.yml up -d ${project_name}-nginx || echo "[ERROR] Critical - ${project_name}-nginx UP failure"
+   docker-compose -f docker-compose-${project_name}-nginx.yml down
+   PROJECT_NAME=${project_name} docker-compose -f docker-compose-${project_name}-nginx.yml up -d || echo "[ERROR] Critical - ${project_name}-nginx UP failure"
 }
 
 consul_restart(){
@@ -261,7 +245,8 @@ consul_restart(){
 
     docker network create consul || echo "[NOTICE] Consul Network has already been created. You can ignore this message, or if you want to restart it, please terminate other projects that share the Consul network."
 
-    echo "[NOTICE] Run CONSUL container"
+    echo "[NOTICE] Re-run CONSUL container"
+    docker-compose -f docker-compose-consul.yml down
     # https://github.com/hashicorp/consul/issues/17973
     docker-compose -p consul -f docker-compose-consul.yml up -d || echo "[NOTICE] Consul has already been created. You can ignore this message."
     sleep 10

tests/spring-sample-h-auth/run-and-kill-jar-and-check-states.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+sed -i -e "s/\r$//g" $(basename $0)
+set -eu
+
+cd ../../
+
+sudo chmod a+x *.sh
+
+echo "[NOTICE] Substituting CRLF with LF to prevent possible CRLF errors..."
+bash prevent-crlf.sh
+git config apply.whitespace nowarn
+git config core.filemode false
+
+container=$(docker ps --format '{{.Names}}' | grep "spring-sample-h-auth")
+if [ -z "$container" ]; then
+  cp -f .env.java.real .env
+  sudo bash run.sh
+fi
+
+sleep 3
+source ./util.sh
+
+cache_global_vars
+
+consul_pointing=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw 2>/dev/null || echo "failed")
+
+echo "[TEST][NOTICE] ! Kill the jar in ${project_name}-${consul_pointing}"
+docker exec ${project_name}-${consul_pointing} bash -c "kill 9 7"
+
+# Print state checking process
+cache_all_states

util.sh

@@ -7,78 +7,133 @@ git config core.filemode false
 
 cache_all_states() {
 
-  echo '[NOTICE] Checking which container, blue or green, is running.'
+  echo '[NOTICE] Checking which container, blue or green, is running. (Priority :  Where Consul Pointing > Which Container Running > Which Container Restarting)'
 
-  blue_is_up=$(docker exec ${project_name}-blue echo 'yes' 2>/dev/null || echo 'no')
-  if [[ ${blue_is_up} == 'yes' ]]; then
-      if [[ $(check_availability_inside_container_speed_mode 'blue' 10 5 | tail -n 1) == 'true' ]]; then
+  local consul_pointing
+  consul_pointing=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw 2>/dev/null || echo "failed")
+  local blue_status
+  blue_status=$(docker inspect --format='{{.State.Status}}' ${project_name}-blue 2>/dev/null || echo "unknown")
+  local green_status
+  green_status=$(docker inspect --format='{{.State.Status}}' ${project_name}-green 2>/dev/null || echo "unknown")
 
-        echo '[DEBUG] Checking State : A (Blue is currently running)'
+  echo "[NOTICE] ! Base Check : consul_pointing(${consul_pointing}), blue_status(${blue_status}), green_status(${green_status})"
 
-        state='blue'
-        new_state='green'
-        new_upstream=${green_upstream}
+  if [[ ${consul_pointing} == 'blue' ]]; then
+
+      if [[ ${blue_status} == 'running' ]]; then
+
+            echo '[DEBUG] Checking State : Blue-A (Blue is pointed & currently running)'
+
+            state='blue'
+            new_state='green'
+            new_upstream=${green_upstream}
 
       else
 
-        green_is_up=$(docker exec ${project_name}-green echo 'yes' 2>/dev/null || echo 'no')
-        if [[ ${green_is_up} == 'yes' ]]; then
-            if [[ $(check_availability_inside_container_speed_mode 'green' 10 5 | tail -n 1) == 'true' ]]; then
+          if [[ ${consul_pointing} == 'green' ]]; then
 
-              echo '[DEBUG] Checking State : B (Green is currently running)'
+                if [[ ${green_status} == 'running' ]]; then
 
-              state='green'
-              new_state='blue'
-              new_upstream=${blue_upstream}
-            else
+                    echo '[DEBUG] Checking State : Green-A (Green is pointed & currently running)'
 
-              echo '[DEBUG] Checking State : C (Blue and Green are up but both failed in the Internal Integrity Check. Whichever is OK to be deployed.)'
+                    state='green'
+                    new_state='blue'
+                    new_upstream=${blue_upstream}
 
-              state='blue'
-              new_state='green'
-              new_upstream=${green_upstream}
+                else
 
-            fi
-        else
-              echo '[DEBUG] Checking State : D (Both failed in the Internal Integrity Check, but Blue is currently only up.)'
+                    if [[ ${green_status} == 'restarting' ]]; then
 
-              state='blue'
-              new_state='green'
-              new_upstream=${green_upstream}
-        fi
+                        echo '[DEBUG] Checking State : Green-B (Green is pointed & currently restarting)'
 
-      fi
-  else
+                        state='green'
+                        new_state='blue'
+                        new_upstream=${blue_upstream}
+
+                    else
 
-      green_is_up=$(docker exec ${project_name}-green echo 'yes' 2>/dev/null || echo 'no')
-      if [[ ${green_is_up} == 'yes' ]]; then
-          if [[ $(check_availability_inside_container_speed_mode 'green' 10 5 | tail -n 1) == 'true' ]]; then
+                        echo "[DEBUG] Checking State : Green-C (Green is pointed & Green is currently ${green_status})"
 
-            echo '[DEBUG] Checking State : E (Green is currently running)'
+                        state='green'
+                        new_state='blue'
+                        new_upstream=${blue_upstream}
+
+                    fi
+
+                fi
 
-            state='green'
-            new_state='blue'
-            new_upstream=${blue_upstream}
           else
 
-            echo '[DEBUG] cache_all_states : F (Both failed in the Internal Integrity Check, but Green is currently only up.)'
+                if [[ ${blue_status} == 'restarting' ]]; then
+
+                    echo '[DEBUG] Checking State : Blue-B (Blue is pointed & currently restarting)'
+
+                    state='blue'
+                    new_state='green'
+                    new_upstream=${green_upstream}
+
+                else
+
+                    echo "[DEBUG] Checking State : Blue-C (Blue is pointed & Blue is currently ${blue_status})"
+
+                    state='blue'
+                    new_state='green'
+                    new_upstream=${green_upstream}
+
+                fi
+
 
-            state='green'
-            new_state='blue'
-            new_upstream=${blue_upstream}
           fi
+
+      fi
+
+  else
+
+      if [[ ${consul_pointing} == 'green' ]]; then
+
+            if [[ ${green_status} == 'running' ]]; then
+
+                echo '[DEBUG] Checking State : Green-A (Green is pointed & currently running)'
+
+                state='green'
+                new_state='blue'
+                new_upstream=${blue_upstream}
+
+            else
+
+                if [[ ${green_status} == 'restarting' ]]; then
+
+                    echo '[DEBUG] Checking State : Green-B (Green is pointed & currently restarting)'
+
+                    state='green'
+                    new_state='blue'
+                    new_upstream=${blue_upstream}
+
+                else
+
+                    echo "[DEBUG] Checking State : Green-C (Green is pointed & currently ${green_status})"
+
+                    state='green'
+                    new_state='blue'
+                    new_upstream=${blue_upstream}
+
+                fi
+
+            fi
+
       else
-          echo '[DEBUG] cache_all_states : G (Both failed in the Internal Integrity Check and are down. Whichever is OK to be deployed.)'
 
-          state='green'
-          new_state='blue'
-          new_upstream=${blue_upstream}
+            echo "[DEBUG] Checking State : Undefined"
+
+            state='blue'
+            new_state='green'
+            new_upstream=${green_upstream}
 
       fi
 
   fi
 
-  echo "[NOTICE] Finally, ! ${new_state} will be deployed."
+  echo "[NOTICE] Finally, ! will be deployed to ${new_state}."
 }
 
 set_expose_and_app_port(){