diff --git a/README.md b/README.md index 6f4fee2..2685244 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ For all echo messages or properties .env, the following terms indicate... - DOWN : ```docker-compose down``` - RESTART : ```docker build & docker-compose down & docker-compose up ``` - ex) NGINX_RESTART on .env means docker build & down & up for NGINX - +- safe : set a new state(=blue or green) without stopping or causing errors on your web App. ## How to Start with a Node Sample (Local). A Node.js sample project (https://github.com/hagopj13/node-express-boilerplate) that has been receiving a lot of stars, comes with an MIT License and serves as an example for demonstrating how to use Docker-Blue-Green-Runner. @@ -155,9 +155,21 @@ CONSUL_RESTART=false # The value must be json or yaml type, which is injected into docker-compose-app-${app_env}.yml DOCKER_COMPOSE_ENVIRONMENT={"MONGODB_URL":"mongodb://host.docker.internal:27017/node-boilerplate","NODE_ENV":"development"} ``` + +## Check states +```shell +bash check-current-states.sh + +# an output sample below +# [DEBUG] ! Setting which (Blue OR Green) to deploy the App as... (Final Check) : blue_score : 80, green_score : 0, state : blue, new_state : green, state_for_emergency : blue, new_upstream : https://laravel_crud_boilerplate-green:8080. +# The higher the score a state receives, the more likely it is to be the currently running state. So the updated App should be deployed as the non-occupied state(=new_state). +# For the emergency script, there is another safer priority added over the results of scores. So, the 'state_for_emergency' is basically the same as the 'state' but can differ. +``` + ## Emergency - Nginx (like when Nginx is NOT booted OR 502 error...) ```shell +# Automatically set the safe state & down and up Nginx bash emergency-nginx-down-and-up.sh # In case you need to manually set the Nginx to point to 'blue' or 'green' bash emergency-nginx-down-and-up.sh blue @@ -205,50 +217,60 @@ bash ./rollback.sh ## Structure ```shell - # [A] Get mandatory variables - check_necessary_commands - cache_global_vars - # The 'cache_all_states' in 'cache_global_vars' function decides which state should be deployed. If this is called later at a point in this script, states could differ. - local initially_cached_old_state=${state} - check_env_integrity - - echo "[NOTICE] We will now deploy '${project_name}' in a way of 'Blue-Green'" - - # [B] Set mandatory files - # These are all about passing variables from the .env to the docker-compose-${project_name}-local.yml - initiate_docker_compose - apply_env_service_name_onto_app_yaml - apply_ports_onto_nginx_yaml - apply_docker_compose_environment_onto_app_yaml - - # Refer to .env.*.real - if [[ ${app_env} == 'real' ]]; then - apply_docker_compose_volumes_onto_app_real_yaml - fi +# [A] Get mandatory variables +check_necessary_commands +cache_global_vars +# The 'cache_all_states' in 'cache_global_vars' function decides which state should be deployed. If this is called later at a point in this script, states could differ. +local initially_cached_old_state=${state} +check_env_integrity + +echo "[NOTICE] Finally, !! Deploy the App as !! ${new_state} !!, we will now deploy '${project_name}' in a way of 'Blue-Green'" + +# [B] Set mandatory files +# These are all about passing variables from the .env to the docker-compose-${project_name}-local.yml +initiate_docker_compose +apply_env_service_name_onto_app_yaml +apply_ports_onto_nginx_yaml +apply_docker_compose_environment_onto_app_yaml + +# Refer to .env.*.real +if [[ ${app_env} == 'real' ]]; then + apply_docker_compose_volumes_onto_app_real_yaml +fi - apply_docker_compose_volumes_onto_app_nginx_yaml +apply_docker_compose_volumes_onto_app_nginx_yaml - create_nginx_ctmpl +create_nginx_ctmpl - if [[ ${skip_building_app_image} != 'true' ]]; then - backup_app_to_previous_images - backup_nginx_to_previous_images - fi +if [[ ${skip_building_app_image} != 'true' ]]; then + backup_app_to_previous_images +fi +backup_nginx_to_previous_images - if [[ ${app_env} == 'local' ]]; then - give_host_group_id_full_permissions - fi +if [[ ${app_env} == 'local' ]]; then - if [[ ${docker_layer_corruption_recovery} == 'true' ]]; then - terminate_whole_system - fi + give_host_group_id_full_permissions +#else - # [B] Build Docker images for the App, Nginx, Consul - if [[ ${skip_building_app_image} != 'true' ]]; then - load_app_docker_image - load_consul_docker_image - load_nginx_docker_image - fi + # create_host_folders_if_not_exists +fi + +if [[ ${docker_layer_corruption_recovery} == 'true' ]]; then + terminate_whole_system +fi + +# [B] Build Docker images for the App, Nginx, Consul +if [[ ${skip_building_app_image} != 'true' ]]; then + load_app_docker_image +fi + load_consul_docker_image + load_nginx_docker_image + +local cached_new_state=${new_state} +cache_all_states +if [[ ${cached_new_state} != "${new_state}" ]]; then + (echo "[ERROR] Just checked all states shortly after the Docker Images had been done built. The state the App was supposed to be deployed as has been changed. (Original : ${cached_new_state}, New : ${new_state}). For the safety, we exit..." && exit 1) +fi # Run 'docker-compose up' for 'App', 'Consul (Service Mesh)' and 'Nginx' and # Check if the App is properly working from the inside of the App's container using 'wait-for-it.sh ( https://github.com/vishnubob/wait-for-it )' and conducting a health check with settings defined on .env. @@ -302,6 +324,7 @@ fi echo "[NOTICE] Delete : images." docker rmi $(docker images -f "dangling=true" -q) || echo "[NOTICE] Any images in use will not be deleted." +echo "[NOTICE] APP_URL : ${app_url}" ``` ## Test diff --git a/check-current-states.sh b/check-current-states.sh new file mode 100644 index 0000000..3191752 --- /dev/null +++ b/check-current-states.sh @@ -0,0 +1,11 @@ +#!/bin/bash +sed -i -e "s/\r$//g" $(basename $0) +set -e + +git config apply.whitespace nowarn +git config core.filemode false + +source ./util.sh + +cache_global_vars +cache_all_states \ No newline at end of file diff --git a/emergency-nginx-down-and-up.sh b/emergency-nginx-down-and-up.sh index 6aab82f..9ad3ff3 100644 --- a/emergency-nginx-down-and-up.sh +++ b/emergency-nginx-down-and-up.sh @@ -40,6 +40,6 @@ nginx_down_and_up(){ } -echo "[NOTICE] Finally, !! to ${state_a}, we will now deploy '${project_name}' in a way of 'Blue-Green'" +echo "[NOTICE] Finally, !! Deploy the App as !! ${state_a} !!, we will now deploy '${project_name}' in a way of 'Blue-Green'" nginx_down_and_up ./activate.sh ${state_a} ${state_b} ${state_upstream} ${consul_key_value_store} \ No newline at end of file diff --git a/run.sh b/run.sh index bc14d33..9413ddd 100644 --- a/run.sh +++ b/run.sh @@ -313,41 +313,6 @@ load_all_containers(){ } -check_availability_out_of_container(){ - - echo "[NOTICE] Check the http status code from the outside of the container." >&2 - sleep 1 - - for retry_count in {1..8} - do - status=$(curl ${app_url}/${app_health_check_path} -o /dev/null -k -Isw '%{http_code}' --connect-timeout 10) - available_status_cnt=$(echo ${status} | egrep -i '^2[0-9]+|3[0-9]+$' | wc -l) - - if [[ ${available_status_cnt} -lt 1 ]]; then - - echo "Bad HTTP response in the ${new_state} app: ${status}" >&2 - - if [[ ${retry_count} -eq 7 ]] - then - echo "[ERROR] Health Check Failed. (If you are not accessing an external domain (=closed network setting environment), you need to check if APP_URL is the value retrieved by ifconfig on the Ubuntu host. Access to the ip output by the WIN ipconfig command may fail. Or you need to check the network firewall." >&2 - echo "false" - return - fi - - else - echo "[NOTICE] Success. (Status (2xx, 3xx) : ${status})" >&2 - break - fi - - echo "[NOTICE] Retry once every 3 seconds for a total of 8 times..." >&2 - sleep 3 - done - - echo 'true' - return - -} - backup_to_new_images(){ echo "[NOTICE] docker tag latest new" @@ -366,7 +331,7 @@ _main() { local initially_cached_old_state=${state} check_env_integrity - echo "[NOTICE] Finally, !! to ${new_state}, we will now deploy '${project_name}' in a way of 'Blue-Green'" + echo "[NOTICE] Finally, !! Deploy the App as !! ${new_state} !!, we will now deploy '${project_name}' in a way of 'Blue-Green'" # [B] Set mandatory files # These are all about passing variables from the .env to the docker-compose-${project_name}-local.yml @@ -408,6 +373,12 @@ _main() { load_consul_docker_image load_nginx_docker_image + local cached_new_state=${new_state} + cache_all_states + if [[ ${cached_new_state} != "${new_state}" ]]; then + (echo "[ERROR] Just checked all states shortly after the Docker Images had been done built. The state the App was supposed to be deployed as has been changed. (Original : ${cached_new_state}, New : ${new_state}). For the safety, we exit..." && exit 1) + fi + # [C] Docker-compose up the App, Nginx, Consul & * Internal Integrity Check for the App load_all_containers @@ -445,6 +416,8 @@ _main() { echo "[NOTICE] Delete : images." docker rmi $(docker images -f "dangling=true" -q) || echo "[NOTICE] Any images in use will not be deleted." + + echo "[NOTICE] APP_URL : ${app_url}" } _main diff --git a/tests/spring-sample-h-auth/run-and-kill-jar-and-after-seconds-auto-recovered.sh b/tests/spring-sample-h-auth/run-and-kill-jar-and-after-seconds-auto-recovered.sh index f90f222..5d694c3 100644 --- a/tests/spring-sample-h-auth/run-and-kill-jar-and-after-seconds-auto-recovered.sh +++ b/tests/spring-sample-h-auth/run-and-kill-jar-and-after-seconds-auto-recovered.sh @@ -2,8 +2,6 @@ sed -i -e "s/\r$//g" $(basename $0) set -eu -echo "[TEST][NOTICE] Check DB is connected (refer to Samples on README)." - cd ../../ sudo chmod a+x *.sh diff --git a/tests/spring-sample-h-auth/run-and-kill-jar-and-state-is-restarting-or-running.sh b/tests/spring-sample-h-auth/run-and-kill-jar-and-state-is-restarting-or-running.sh index 5cb1200..8b29626 100644 --- a/tests/spring-sample-h-auth/run-and-kill-jar-and-state-is-restarting-or-running.sh +++ b/tests/spring-sample-h-auth/run-and-kill-jar-and-state-is-restarting-or-running.sh @@ -2,8 +2,6 @@ sed -i -e "s/\r$//g" $(basename $0) set -eu -echo "[TEST][NOTICE] Check DB is connected (refer to Samples on README)." - cd ../../ sudo chmod a+x *.sh diff --git a/tests/spring-sample-h-auth/run-and-make-consul-pointing-error-and-recovered.sh b/tests/spring-sample-h-auth/run-and-make-consul-pointing-error-and-recovered.sh new file mode 100644 index 0000000..e7050bf --- /dev/null +++ b/tests/spring-sample-h-auth/run-and-make-consul-pointing-error-and-recovered.sh @@ -0,0 +1,51 @@ +#!/bin/bash +sed -i -e "s/\r$//g" $(basename $0) +set -eu + +cd ../../ + +sudo chmod a+x *.sh + +echo "[NOTICE] Substituting CRLF with LF to prevent possible CRLF errors..." +bash prevent-crlf.sh +git config apply.whitespace nowarn +git config core.filemode false + +container=$(docker ps --format '{{.Names}}' | grep "spring-sample-h-auth-[bg]") +if [ -z "$container" ]; then + echo "[NOTICE] There is NO spring-sample-h-auth container, now we will build it." + cp -f .env.java.real .env + sudo bash run.sh +else + echo "[NOTICE] $container exists." +fi + +sleep 3 +source ./util.sh + +cache_global_vars + +consul_pointing=$(docker exec ${project_name}-nginx curl ${consul_key_value_store}?raw 2>/dev/null || echo "failed") +the_opposite_of_consul_pointing='' +if [[ ${consul_pointing} == 'blue' ]]; then + the_opposite_of_consul_pointing='green' +else + the_opposite_of_consul_pointing='blue' +fi + +echo "[TEST][DEBUG] the_opposite_of_consul_pointing : ${the_opposite_of_consul_pointing}" + +echo "[TEST][NOTICE] To make a Nginx error, get consul_pointing to the wrong(=NOT running) container" +bash emergency-nginx-down-and-up.sh ${the_opposite_of_consul_pointing} || echo "" +#echo "[TEST][NOTICE] Run 'emergency-nginx-down-and-up.sh'" +#bash emergency-nginx-down-and-up.sh + +echo "[TEST][NOTICE] Run check_availability_out_of_container" +cache_global_vars +re=$(check_availability_out_of_container | tail -n 1); + +if [[ ${re} != 'true' ]]; then + echo "[TEST][NOTICE] : FAILURE" +else + echo "[TEST][NOTICE] : SUCCESS" +fi \ No newline at end of file diff --git a/util.sh b/util.sh index 22c852a..35e8b88 100644 --- a/util.sh +++ b/util.sh @@ -16,134 +16,96 @@ cache_all_states() { local green_status green_status=$(docker inspect --format='{{.State.Status}}' ${project_name}-green 2>/dev/null || echo "unknown") - echo "[NOTICE] ! Base Check : consul_pointing(${consul_pointing}), blue_status(${blue_status}), green_status(${green_status})" + echo "[DEBUG] ! Setting which (Blue OR Green) to deploy the App as... (Base Check) : consul_pointing(${consul_pointing}), blue_status(${blue_status}), green_status(${green_status})" - if [[ ${consul_pointing} == 'blue' ]]; then + local blue_score=0 + local green_score=0 - if [[ ${blue_status} == 'running' ]]; then - - echo '[DEBUG] Checking State : Blue-A (Blue is pointed & currently running)' - - state='blue' - state_for_emergency=${state} - new_state='green' - new_upstream=${green_upstream} - - else - - if [[ ${consul_pointing} == 'green' ]]; then - - if [[ ${green_status} == 'running' ]]; then - - echo '[DEBUG] Checking State : Green-A (Green is pointed & currently running)' - - state='green' - state_for_emergency=${state} - new_state='blue' - new_upstream=${blue_upstream} - - else - - if [[ ${green_status} == 'restarting' ]]; then - - echo '[DEBUG] Checking State : Green-B (Green is pointed & currently restarting)' - - state='green' - state_for_emergency='blue' - new_state='blue' - new_upstream=${blue_upstream} - - else - - echo "[DEBUG] Checking State : Green-C (Green is pointed & Green is currently ${green_status})" - - state='green' - state_for_emergency='blue' - new_state='blue' - new_upstream=${blue_upstream} - - fi + if [[ "$consul_pointing" == "blue" ]]; then + blue_score=$((blue_score + 50)) + elif [[ "$consul_pointing" == "green" ]]; then + green_score=$((green_score + 50)) + fi - fi + case "$blue_status" in + "running") + blue_score=$((blue_score + 30)) + ;; + "restarting") + blue_score=$((blue_score + 29)) + ;; + "created") + blue_score=$((blue_score + 28)) + ;; + "exited") + blue_score=$((blue_score + 27)) + ;; + "paused") + blue_score=$((blue_score + 26)) + ;; + "dead") + blue_score=$((blue_score + 25)) + ;; + *) + ;; + esac + + + case "$green_status" in + "running") + green_score=$((green_score + 30)) + ;; + "restarting") + green_score=$((green_score + 29)) + ;; + "created") + green_score=$((green_score + 28)) + ;; + "exited") + green_score=$((green_score + 27)) + ;; + "paused") + green_score=$((green_score + 26)) + ;; + "dead") + green_score=$((green_score + 25)) + ;; + *) + ;; + esac + + # 최종 결과 출력 + if [[ $blue_score -gt $green_score ]]; then + + state='blue' + if [[ ("$blue_status" == "unknown" || "$blue_status" == "exited" || "$blue_status" == "paused" || "$blue_status" == "dead") && "$green_status" == "running" ]]; then + state_for_emergency='green' + else + state_for_emergency=${state} + fi + new_state='green' + new_upstream=${green_upstream} + + elif [[ $green_score -gt $blue_score ]]; then + + state='green' + if [[ ("$green_status" == "unknown" || "$green_status" == "exited" || "$green_status" == "paused" || "$green_status" == "dead") && "$blue_status" == "running" ]]; then + state_for_emergency='blue' else - - if [[ ${blue_status} == 'restarting' ]]; then - - echo '[DEBUG] Checking State : Blue-B (Blue is pointed & currently restarting)' - - state='blue' - state_for_emergency='green' - new_state='green' - new_upstream=${green_upstream} - - else - - echo "[DEBUG] Checking State : Blue-C (Blue is pointed & Blue is currently ${blue_status})" - - state='blue' - state_for_emergency='green' - new_state='green' - new_upstream=${green_upstream} - - fi - - + state_for_emergency=${state} fi - - fi + new_state='blue' + new_upstream=${blue_upstream} else - - if [[ ${consul_pointing} == 'green' ]]; then - - if [[ ${green_status} == 'running' ]]; then - - echo '[DEBUG] Checking State : Green-A (Green is pointed & currently running)' - - state='green' - state_for_emergency=${state} - new_state='blue' - new_upstream=${blue_upstream} - - else - - if [[ ${green_status} == 'restarting' ]]; then - - echo '[DEBUG] Checking State : Green-B (Green is pointed & currently restarting)' - - state='green' - state_for_emergency='blue' - new_state='blue' - new_upstream=${blue_upstream} - - else - - echo "[DEBUG] Checking State : Green-C (Green is pointed & currently ${green_status})" - - state='green' - state_for_emergency='blue' - new_state='blue' - new_upstream=${blue_upstream} - - fi - - fi - - else - - echo "[DEBUG] Checking State : Undefined" - - state='blue' - state_for_emergency='blue' - new_state='green' - new_upstream=${green_upstream} - - fi - + state='green' + state_for_emergency=${state} + new_state='blue' + new_upstream=${blue_upstream} fi - echo "[DEBUG] state : ${state}, new_state : ${new_state}, state_for_emergency : ${state_for_emergency}." + echo "[DEBUG] ! Setting which (Blue OR Green) to deploy the App as... (Final Check) : blue_score : ${blue_score}, green_score : ${green_score}, state : ${state}, new_state : ${new_state}, state_for_emergency : ${state_for_emergency}, new_upstream : ${new_upstream}." } set_expose_and_app_port(){ @@ -939,3 +901,39 @@ check_availability_inside_container_speed_mode(){ return fi } + + +check_availability_out_of_container(){ + + echo "[NOTICE] Check the http status code from the outside of the container." >&2 + sleep 1 + + for retry_count in {1..6} + do + status=$(curl ${app_url}/${app_health_check_path} -o /dev/null -k -Isw '%{http_code}' --connect-timeout 10) + available_status_cnt=$(echo ${status} | egrep -i '^2[0-9]+|3[0-9]+$' | wc -l) + + if [[ ${available_status_cnt} -lt 1 ]]; then + + echo "Bad HTTP response in the ${new_state} app: ${status}" >&2 + + if [[ ${retry_count} -eq 5 ]] + then + echo "[ERROR] Health Check Failed. (If you are not accessing an external domain (=closed network setting environment), you need to check if APP_URL is the value retrieved by ifconfig on the Ubuntu host. Access to the ip output by the WIN ipconfig command may fail. Or you need to check the network firewall." >&2 + echo "false" + return + fi + + else + echo "[NOTICE] Success. (Status (2xx, 3xx) : ${status})" >&2 + break + fi + + echo "[NOTICE] Retry once every 3 seconds for a total of 8 times..." >&2 + sleep 3 + done + + echo 'true' + return + +}