vpn-health-checker

#!/bin/bash


### BEGIN INIT INFO
# Provides:          VPN Gateway to AWS VPC
# Required-Start:    $network $remote_fs $syslog
# Required-Stop:     $syslog $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start VPN Customer Gateway at boot time
# Description:       Start VPN Customer Gateway at boot time
### END INIT INFO

prog=$(basename "$0")
# if DEBUG is set (to anything but '0') send a copy of the output to /dev/tty so the caller
# can see the messages without checking the system logs
if [ "$DEBUG" ] && [ "$DEBUG" ] != 0; then
    logger="logger -s -t $prog"
else
    logger="logger -t $prog"
fi

if [ "$(id -u)" -ne 0 ]; then
    echo "permission denied (must be superuser)" |
      logger -s -p daemon.error -t "$prog" 2>&1
    exit 4
fi

RETVAL=0
SERVICEDIR=/etc/vpn-health-checker/
SCRIPTFILE=${SERVICEDIR}vpn-health-checker
CONFIGFILE=${SERVICEDIR}vpn-health-checker.conf
PIDFILE=/var/run/vpn-health-checker.pid

if [ ! -d "${SERVICEDIR}" ];then
  echo "Creating service directory..."
  mkdir -p ${SERVICEDIR}
  if [ $? -eq 0 ];then
    echo "Service dirctory successfully created"
  fi
fi

setup_healthcheck() {
if [ -f $SCRIPTFILE ] && [ -f $CONFIGFILE ]; then
  $SCRIPTFILE
else
  if [ ! -f $SCRIPTFILE ];then 
    cat <<'EOF' > $SCRIPTFILE
#!/bin/bash
# This script will run a continuous ping in the background to target IP over your VPN tunnel. If pings fail, then openswan will failover the tunnels.

SERVICEDIR=/etc/vpn-health-checker/
SCRIPTFILE=${SERVICEDIR}vpn-health-checker
CONFIGFILE=${SERVICEDIR}vpn-health-checker.conf

AWSREGION=`curl http://169.254.169.254/latest/dynamic/instance-identity/document|grep region|awk -F\" '{print $4}'`

source $CONFIGFILE
if [ ! $? -eq 0 ]; then
    echo "Configuration file not found"
    exit 1
fi

exec 1> >(logger -s -t $(basename $0)) 2>&1

function fdate {
    date '+%c'
}

function send_metric_errors {
    aws cloudwatch put-metric-data --metric-name VpnHealthCheckErrors --namespace VPN --value $1 --dimensions "VpnName=${environment_identifier}"  --region ${AWSREGION}
}

function log {
    # $1 log message
    echo "$(fdate) ${environment_identifier} ${1}" | tee -a $vpn_logger
}

log "INFO: Starting VPN Healthcheck"

function down_tunnel {
    local tunnel=$1
    ipsec auto --down ${tunnel}
    ipsec auto --delete ${tunnel}
    ipsec auto --unroute ${tunnel}
}

function up_tunnel {
    local tunnel=$1
    ipsec auto --add ${tunnel}
    timeout 30 ipsec auto --up ${tunnel}
    if [ $? -ne 0 ]; then
        return 124
    fi
    ipsec auto --route ${tunnel}
}

function clear_connection {
    down_tunnel $tunnel2
    down_tunnel $tunnel1
    sleep 2
    up_tunnel $tunnel1
    echo "${tunnel1}" > ${peer_state}
}

function valid_ip(){
    local  ip=$1
    local  stat=1

    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
        OIFS=$IFS
        IFS='.'
        ip=($ip)
        IFS=$OIFS
        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
        stat=$?
    fi
    return $stat
}

function failover_decision() {
    local result=$1
    if [[ ${result} -eq 0 ]]; then 
        log "INFO: This tunnel is still good. I can still ping the second target IP. No failover necessary"
    else
        /etc/init.d/ipsec start
        if grep -Fxq "${tunnel1}" ${peer_state}; then
            log "WARNING: VPN Healthcheck failed for ${tunnel1}, attempting failover to ${tunnel2}"
            send_metric_errors 1
            down_tunnel $tunnel1 
            down_tunnel $tunnel2
            sleep 2
            up_tunnel $tunnel2
        
            if [[ $? -eq 0 ]]; then
                log "INFO: Successfully switched to ${tunnel2}, resuming healthchecks"
            else
                log "ERROR: Timeout switching to ${tunnel2}"
                send_metric_errors 2
            fi
            sed -i -e "s/${tunnel1}/${tunnel2}/g" ${peer_state}

        elif grep -Fxq "${tunnel2}" ${peer_state}; then
            log "WARNING: VPN Healthcheck failed for ${tunnel2}, attempting failover to ${tunnel1}"
            send_metric_errors 1
            down_tunnel $tunnel2
            down_tunnel $tunnel1
            sleep 2
            up_tunnel $tunnel1
            if [[ $? -eq 0 ]]; then
                log "INFO: Successfully switched to ${tunnel1}, resuming healthchecks"
            else
                log "ERROR: Timeout switching to ${tunnel1}"
                send_metric_errors 2
            fi
            sed -i -e "s/${tunnel2}/${tunnel1}/g" ${peer_state}
        
        elif [[ ! -f ${peer_state} ]]; then
            echo "${tunnel1}" > ${peer_state}
        else
            log "CRITICAL: Something went wrong with peer state synchronization! Please use '/etc/init.d/vpn-health-checker restart'"
            send_metric_errors 5
            exit 1
        fi
    fi
}

function ping_target() {
    local  target=$1
    if [ -z ${source_ip+x} ]; then
        ping -c $probes -W $time_out $target
        result=$?
    else 
        ping -I $source_ip -c $probes -W $time_out $target
        result=$?
    fi
}


if valid_ip $target_ip_1 -eq 0; then
    my_pid=$$
    if [ -f ${pidfile} ]; then
        rm -f ${pidfile}
        echo $my_pid >> ${pidfile}
    else
        echo $my_pid >> ${pidfile}
    fi
    
    trap "exit" INT
    /etc/init.d/ipsec start
    clear_connection

    last_data_sync_minute=$(date '+%M')
    while :
    do
        ping_target $target_ip_1
        
        if [[ ${result} -eq 0 ]]; then
            echo "==== 1st target ping is good! Nothing more to do ===="
            time_since_last_sync=$(date --date="-${last_data_sync_minute} minute" '+%M')
            if [[ ! ${time_since_last_sync} -eq 0 ]]; then
                last_data_sync_minute=$(date '+%M')
                send_metric_errors 0
            fi
        else
            echo "==== 1st target ping failed! Trying second target ping if it's configured ====" 
            if [ ! -z ${target_ip_2+x} ]; then
                if valid_ip $target_ip_2 -eq 0 ; then
                    ping_target $target_ip_2
                    if [[ ${result} -eq 0 ]]; then
                        echo "==== Second ping is good! Nothing more to do ===="
                    else  
                        echo "==== Second ping failed! Attempting failover ===="
                        failover_decision $result
                    fi
                else
                    echo "==== The second target exists, but is not valid ===="
                    result=1
                    failover_decision $result
                fi
            else
                echo "==== No second target IP was configured! Attempting failover ===="
                result=1
                failover_decision $result
            fi  
        fi
    done
else
    send_metric_errors 5
    log "ERROR: The value for target_ip_1 is invalid. The VPN healthchecker has stopped! Hit <enter> to continue" 
    exit 1 
fi
EOF
    chmod 755 $SCRIPTFILE
  fi
  if [ ! -f $CONFIGFILE ]; then
    cat <<'EOF' > $CONFIGFILE
# The environment is used to identify logs when upload them to aws cloudwatch logs.
environment_identifier=VpnHA

# The first openswan connection name used in this failover configuration. This will be the first tunnel up if all VPN peers are healthy
tunnel1=TunnelA

# The second openswan connection name used in this failover configuration. This tunnel will remain down until the first tunnel needs to failover.
tunnel2=TunnelB

# The target IP to use for determing tunnel health. This should be a stable private IP in the remote network over the tunnel that that is responsive to ping at all times. If you are using this to establish
# a VPN with an AWS VPC, keep in mind the VPC gateway IP does not respond to ping so you will need to choose something else like an EC2 IP. Target is temporarily set to loopback to prevent automatic tunnel failure
# when you first start the service.
target_ip_1=127.0.0.1

# (Optional) Uncomment the second target IP is to help avoid target_ip_1 from being a single point of failure. This second IP should also be responsive to pings and be up at all times. Note that it will be checked before failover if the first IP becomes
# unresponsive
#target_ip_2=127.0.0.1

# (Optional) Uncomment the source_ip line if you have multiple interfaces on openswan and wish to source your healthcheck probes from a particular source IP. Note that this IP must be part of the
# local private network allowed across the VPN tunnel
#source_ip=eth0

# How many ping probes to run against the IP target before declaring the tunnel healthy or unhealthy. This number of probes will be run individually against both target 1 AND target 2 if you have uncommented it above (2 x probes). Use caution when changing this default value
probes=10

# The timeout value in seconds before each ping probe is declared "timed out". Therefore total failure detection time in seconds will be (probes x timeout). Use caution when changing this default value
time_out=3

# Where you want to store healthcheck logging events if a tunnel fails. Logging will not track successful checks
vpn_logger=/var/log/vpn_health

# The file that keeps track of which tunnel is up currently (ie. tunnel1 vs tunnel2). No need to modify unless for advanced troubleshooting.
peer_state=/etc/vpn-health-checker/.peer_state

# The healthcheck PID file. No need to modify unless for advanced troubleshooting
pidfile=/var/run/vpn-health-checker.pid
EOF
  fi
  $SCRIPTFILE
fi
}

is_process_running () {
  if [ -f $PIDFILE ]; then
    PID=$(cat $PIDFILE)
    ps -p "$PID" >> /dev/null
    if [ $? -eq 0 ]; then
      return 1
    else
      return 0
    fi
  else
    return 0
  fi
}

start_process () {
  echo "Starting VPN healthchecker..."
  setup_healthcheck > /dev/null 2>&1 & disown $!; exit 0
}

start_process_debug () {
  echo "Starting VPN healthchecker in debug mode..."
      setup_healthcheck
}

teardown_healthcheck () {
  echo "Stopping VPN healthchecker..."
   for pid in $(ps -ef | awk '/vpn-health-checker/ {print $2}'); do kill -9 $pid; done > /dev/null 2>&1
}

start() {
  is_process_running
  if [ $? -eq 0 ];then
    start_process
  else
    echo "Process appears to already be running."
  fi
  RETVAL=$?
  return $RETVAL
}

stop() {
  is_process_running
  if [ $? -eq 1 ];then
    teardown_healthcheck
  else
    echo "Process appears to have been stopped already. Cleaning up anyways"
    teardown_healthcheck
  fi
  RETVAL=0
  return $RETVAL
}

restart() {
    stop
    start
    RETVAL=$?
    return $RETVAL
}

debug() {
  is_process_running
  if [ $? -eq 0 ];then
    start_process_debug
  else
    echo "Process appears to already be running."
  fi
  RETVAL=$?
  return $RETVAL
}

version() {
  echo "VPN Healthchecker - Version 1.5"
  RETVAL=$?
  return $RETVAL
}

status() {
  is_process_running
  if [ $? -eq 0 ];then
    echo "Process is stopped"
  else
    echo "Process is running ( PID: $(cat $PIDFILE) )"
  fi
  RETVAL=$?
  return $RETVAL
}

# do it
case "$1" in
  start|--start)
    start
    ;;
  stop|--stop)
    stop
    ;;
  restart|--restart)
    restart
    ;;
  debug|--debug)
    debug
    ;;
  version|--version)
    version
    ;;
  status|--status)
    status
    ;;
  *)
    echo "Usage: $prog {start|stop|restart|debug|version|status}"
    RETVAL=2
esac

exit $RETVAL