#!/bin/bash
set -o nounset
config_file=/var/lib/haproxy/conf/haproxy.config
pid_file=/var/lib/haproxy/run/haproxy.pid
old_pid=""
haproxy_conf_dir=/var/lib/haproxy/conf
readonly max_wait_time=30
readonly timeout_opts="-m 1 --connect-timeout 1"
readonly numeric_re='^[0-9]+$'
function haproxyHealthCheck() {
local wait_time=${MAX_RELOAD_WAIT_TIME:-$max_wait_time}
local port=${ROUTER_SERVICE_HTTP_PORT:-"80"}
local url="http://localhost:${port}"
local retries=0
local start_ts=$(date +"%s")
local proxy_proto="${ROUTER_USE_PROXY_PROTOCOL-}"
if ! [[ $wait_time =~ $numeric_re ]]; then
echo " - Invalid max reload wait time, using default $max_wait_time ..."
wait_time=$max_wait_time
fi
local end_ts=$((start_ts + wait_time))
# test with proxy protocol on
if [[ "${proxy_proto}" == "TRUE" || "${proxy_proto}" == "true" ]]; then
echo " - Proxy protocol on, checking ${url} ..."
while true; do
local statusline=$(echo $'PROXY UNKNOWN\r\nGET / HTTP/1.1\r\n' | socat tcp-connect:localhost:${port} stdio | head -1)
if [[ "$statusline" == *" 503 "* ]]; then
echo " - Health check ok : $retries retry attempt(s)."
return 0
fi
if [ $(date +"%s") -ge $end_ts ]; then
echo " - Exceeded max wait time ($wait_time) in health check - $retries retry attempt(s)."
return 1
fi
sleep 0.5
retries=$((retries + 1))
done
return 0
fi
echo " - Checking ${url} ..."
while true; do
local httpcode=$(curl $timeout_opts -s -o /dev/null -I -H "Host: " -w "%{http_code}" ${url})
if [ "$httpcode" == "503" ]; then
echo " - Health check ok : $retries retry attempt(s)."
return 0
fi
if [ $(date +"%s") -ge $end_ts ]; then
echo " - Exceeded max wait time ($wait_time) in health check - $retries retry attempt(s)."
return 1
fi
sleep 0.5
retries=$((retries + 1))
done
}
# How many times to retry removal of the iptables rules (if requested at all)
# It will sleep for 1/2 a second between attempts, so the time is retries / 2 secs
retries=20
# sort the path based map files for the haproxy map_beg function
for mapfile in "$haproxy_conf_dir"/*.map; do
sort -r "$mapfile" -o "$mapfile"
done
old_pids=$(ps -A -opid,args | grep haproxy | egrep -v -e 'grep|reload-haproxy' | awk '{print $1}' | tr '\n' ' ')
reload_status=0
installed_iptables=0
if [ -n "$old_pids" ]; then
if $(set | grep DROP_SYN_DURING_RESTART= > /dev/null) && [[ "$DROP_SYN_DURING_RESTART" == 'true' || "$DROP_SYN_DURING_RESTART" == '1' ]]; then
# We install the syn eater so that connections that come in during the restart don't
# go onto the wrong socket, which is then closed.
ports=$(grep -E -o '^\s*bind\s+:[[:digit:]]+\w' "$config_file" | cut -f2 -d: | paste -d, -s)
if [ -n "$ports" ]; then
# If this doesn't insert, we don't care, we still want to reload
/usr/sbin/iptables -I INPUT -p tcp -m multiport --dports $ports --syn -j DROP \
-m comment --comment "Eat SYNs while reloading haproxy" || :
installed_iptables=1
# The sleep is needed to let the socket drain before the new
# haproxy starts and binds to the same port. The value was
# determined by trial and error: I stopped seeing failures at
# 0.01 under load, so I put in a 10x margin. At worst, we may
# leave a connection in the old process' listen buffer that
# won't get handled, and they'll get a reset. I didn't want to
# set it too long, because that affects the overall time a
# reload takes which means that incoming connections aren't
# handled while the SYN eater is in place.
sleep 0.1
fi
fi
/usr/sbin/haproxy -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids
reload_status=$?
if [[ "$installed_iptables" == 1 ]]; then
# We NEVER want to leave the syn eater in place after the reload or haproxy
# will never get new connections. So try to remove it twenty times, and if
# that fails, log the error and return failure so the pod logs a fatal error.
i=0
while (( i++ < retries )) ; do
/usr/sbin/iptables -D INPUT -p tcp -m multiport --dports $ports --syn -j DROP \
-m comment --comment "Eat SYNs while reloading haproxy" || :
# Test the condition and end the loop if the rule has been removed
/usr/sbin/iptables -L INPUT | grep -F '/* Eat SYNs while reloading haproxy */' || break
>&2 echo "Unable to remove SYN eating rule, attempt $i. Will retry..."
# But sleep for a bit before retrying
sleep 0.5
done
if (( i >= retries )); then
# We failed to remove the rule... log failure and exit to signal the caller
>&2 echo "Unable to remove the iptables SYN eating rule. Aborting after $retries retries"
exit 1
fi
fi
else
/usr/sbin/haproxy -f $config_file -p $pid_file
reload_status=$?
fi
[ $reload_status -ne 0 ] && exit $reload_status
haproxyHealthCheck