NGINX + HAProxy Load Balancing

build.sh

#!/bin/bash
set -e

mkdir -p build

cd build
wget https://github.com/haproxy/haproxy/archive/v1.7.0.tar.gz
tar -xvvzf v1.7.0.tar.gz
cd haproxy-1.7.0
make TARGET=linux26 -j 4
mv haproxy ../..

cd ..
wget https://nginx.org/download/nginx-1.11.10.tar.gz
tar -xvvzf nginx-1.11.10.tar.gz
cd nginx-1.11.10
./configure \
    --prefix=/etc/nginx \
    --sbin-path=/usr/sbin/nginx \
    --conf-path=/etc/nginx/nginx.conf \
    --error-log-path=/var/log/nginx/error.log \
    --http-log-path=/var/log/nginx/access.log \
    --pid-path=/var/run/nginx.pid \
    --lock-path=/var/run/nginx.lock \
    --user=nginx \
    --group=nginx \
    --with-http_ssl_module \
    --with-stream \
    --without-http_rewrite_module \
    --without-http_gzip_module
make -j 4
mv objs/nginx ../..

cleanup.sh

#!/bin/bash

kill $(cat /tmp/nginx.pid)
kill $(cat /tmp/haproxy.pid)

full_test.sh

#!/bin/bash
echo "building the proxies"
./build.sh

echo "Setting up the demo"
./setup.sh

# Now to run a test do something like
# ./restart_haproxy.sh
# ab ...
# ./cleanup.sh

haproxy.cfg

global
    daemon
    unix-bind mode 666

defaults
    timeout connect 200ms
    timeout client 1000ms
    timeout server 1000ms

frontend test.http
    mode http
    bind /tmp/sockets/test.http.sock
    default_backend test.http

backend test.http
    mode http
    option httpchk GET /
    server testhttp 0.0.0.0:80 observe layer7 check

frontend test.tcp
    mode tcp
    bind /tmp/sockets/test.tcp.sock
    default_backend test.tcp

backend test.tcp
    mode tcp
    option httpchk GET /
    server testtcp 0.0.0.0:80 observe layer4 check

nginx.conf

worker_processes 1;
pid /tmp/nginx.pid;
error_log /nail/home/jlynch/blog/error.log warn;
worker_rlimit_nofile 40000;

events {
        worker_connections 10000;
        use epoll;
        multi_accept on;
}

stream {
        access_log off;
        tcp_nodelay on;
        server {
                listen 0.0.0.0:20000;
                proxy_timeout 1010ms;
                proxy_pass test.http.nginx_listener;
        }
        upstream test.http.nginx_listener {
                server unix:/tmp/sockets/test.http.sock;
        }
        server {
                listen 0.0.0.0:20001;
                proxy_timeout 1010ms;
                proxy_pass test.tcp.nginx_listener;
        }
        upstream test.tcp.nginx_listener {
                server unix:/tmp/sockets/test.tcp.sock;
        }
}

restart_haproxy.sh

#!/bin/bash

# Restart haproxy every 100ms
while [ 1 ]; do
  ./haproxy -f /tmp/haproxy.cfg -p /tmp/haproxy.pid -sf $(cat /tmp/haproxy.pid)
  sleep 0.100
done

restart_nginx.sh

#!/bin/bash

# Reload nginx workers every 100ms
while [ 1 ]; do
  kill -HUP $(cat /tmp/nginx.pid)
done

setup.sh

#!/bin/bash
set -e

mkdir -p /tmp/nginx
mkdir -p /tmp/sockets

cp haproxy.cfg /tmp
cp nginx.cfg /tmp

# Check configs
./nginx -t -c /tmp/nginx.cfg -p /tmp/nginx
./haproxy -c -f /tmp/haproxy.cfg

# Reload / Restart
kill -HUP $(cat /tmp/nginx.pid) || ./nginx -c /tmp/nginx.cfg -p /tmp/nginx
./haproxy -f /tmp/haproxy.cfg -p /tmp/haproxy.pid -sf $(cat /tmp/haproxy.pid)

test.sh

ab -g test.dat -c 10 -n 1000000 http://localhost:20000/

upgrade_nginx.sh

#!/bin/bash

# Upgrade nginx every 200ms
while [ 1 ]; do
  kill -USR2 $(cat /tmp/nginx.pid) && sleep 0.200
  kill -WINCH $(cat /tmp/nginx.pid.oldbin)
  kill -QUIT $(cat /tmp/nginx.pid.oldbin)
  echo $(cat /tmp/nginx.pid)
done