HAProxy Layer 7 Retries and Chaos Engineering

blog20190712-01.cfg

backend be_servers
    balance roundrobin
    server s1 server1:80 maxconn 10
    server s2 server2:80 maxconn 10
    server s3 server3:80 maxconn 10

blog20190712-02.txt

fe_main be_servers/s1 0/0/-1/-1/12006 503 216 - - sC-- 1/1/0/0/3 0/0 "GET / HTTP/1.1"

blog20190712-03.cfg

backend be_servers
    balance roundrobin
    option redispatch
    server s1 server1:80 maxconn 10
    server s2 server2:80 maxconn 10
    server s3 server3:80 maxconn 10

blog20190712-04.txt

fe_main be_servers/s2 0/3000/0/6/3006 200 228 - - ---- 1/1/0/0/+1 0/0 "GET / HTTP/1.1"

blog20190712-05.txt

fe_main be_servers/s1 0/0/0/-1/1757 502 229 - - SH-- 1/1/0/0/0 0/0 "GET / HTTP/1.1"

blog20190712-06.cfg

backend be_servers
    balance roundrobin
    option redispatch
    retry-on empty-response
    server s1 server1:80 maxconn 10
    server s2 server2:80 maxconn 10
    server s3 server3:80 maxconn 10

blog20190712-07.sh

$ sudo pumba netem --tc-image gaiadocker/iproute2 --duration 5m delay --time 5000 server1 

blog20190712-08.cfg

defaults
    log global
    mode http
    option httplog
    timeout connect 3s
    timeout client 5s
    timeout server 5s
    timeout queue 30s

blog20190712-09.cfg

backend be_servers
    balance roundrobin
    option redispatch
    retry-on empty-response conn-failure
    server s1 server1:80 maxconn 10
    server s2 server2:80 maxconn 10
    server s3 server3:80 maxconn 10

blog20190712-10.yaml

proxy:
- name: http_proxy
 config:
   host: 0.0.0.0       # muxy IP
   port: 81            # muxy port
   proxy_host: server1 # proxied IP/host
   proxy_port: 80      # proxied port

middleware:
- name: delay
  config:
    response_delay: 5000

blog20190712-11.cfg

backend be_servers
    balance roundrobin
    option redispatch
    retry-on empty-response conn-failure
    server s1 muxy:81 maxconn 10
    server s2 server2:80 maxconn 10
    server s3 server3:80 maxconn 10

blog20190712-12.txt

fe_main be_servers/s1 0/0/1/-1/5001 504 218 - - sH-- 1/1/0/0/0 0/0 "GET / HTTP/1.1"

blog20190712-13.cfg

retry-on empty-response conn-failure response-timeout

blog20190712-14.yaml

proxy:
- name: http_proxy
 config:
   host: 0.0.0.0
   port: 81
   proxy_host: server1
   proxy_port: 80

middleware:
- name: http_tamperer
  config:
    response:
      status: 500

blog20190712-15.cfg

retry-on empty-response conn-failure response-timeout 500

blog20190712-16.txt

fe_main be_servers/s2 0/16/0/16/32 200 224 - - ---- 1/1/0/0/+1 0/0 "GET / HTTP/1.1"

blog20190712-17.cfg

retry-on all-retryable-errors

blog20190712-18.cfg

retry-on conn-failure empty-response junk-response response-timeout 0rtt-rejected 500 502 503 504

blog20190712-19.cfg

backend be_servers    
    balance roundrobin
    option redispatch 1
    retry-on all-retryable-errors
    retries 3
    http-request disable-l7-retry if METH_POST
    server s1 server1:80 maxconn 10
    server s2 server2:80 maxconn 10
    server s3 server3:80 maxconn 10