Dynamic Configuration with the HAProxy Runtime API

blog20181128-01.cfg

global
    stats socket ipv4@127.0.0.1:9999 level admin
    stats socket /var/run/hapee-lb.sock mode 666 level admin
    stats timeout 2m

blog20181128-02.sh

$ socat readline /var/run/hapee-lb.sock
$ socat readline tcp4-connect:127.0.0.1:9999

> help

blog20181128-03.sh

$ echo "help" | socat stdio /var/run/hapee-lb.sock
$ echo "help" | socat stdio tcp4-connect:127.0.0.1:9999

blog20181128-04.txt

> help
    help                     : this message
    prompt                   : toggle interactive mode with prompt
    quit                     : disconnect
    show errors              : report last request and response errors for each proxy
    clear counters           : clear max statistics counters (add 'all' for all counters)
    show info                : report information about the running process
    show stat                : report counters for each proxy and server
    show schema json         : report schema used for stats
    disable agent            : disable agent checks (use 'set server' instead)
    disable health           : disable health checks (use 'set server' instead)
    disable server           : disable a server for maintenance (use 'set server' instead)
    enable agent             : enable agent checks (use 'set server' instead)
    enable health            : enable health checks (use 'set server' instead)
    enable server            : enable a disabled server (use 'set server' instead)
    set maxconn server       : change a server's maxconn setting
    set server               : change a server's state, weight or address
    get weight               : report a server's current weight
    set weight               : change a server's weight (deprecated)
    show sess [id]           : report the list of current sessions or dump this session
    shutdown session         : kill a specific session
    shutdown sessions server : kill sessions on a server
    clear table              : remove an entry from a table
    set table [id]           : update or create a table entry's data
    show table [id]          : report table usage stats or dump this table's contents
    disable frontend         : temporarily disable specific frontend
    enable frontend          : re-enable specific frontend
    set maxconn frontend     : change a frontend's maxconn setting
    show servers state [id]  : dump volatile server information (for backend )
    show backend             : list backends in the current running config
    shutdown frontend        : stop a specific frontend
    set dynamic-cookie-key backend : change a backend secret key for dynamic cookies
    enable dynamic-cookie backend  : enable dynamic cookies on a specific backend
    disable dynamic-cookie backend : disable dynamic cookies on a specific backend
    show stat resolvers [id] : dumps counters from all resolvers section and associated name servers
    set maxconn global       : change the per-process maxconn setting
    set rate-limit           : change a rate limiting value
    set timeout              : change a timeout setting
    show env [var]           : dump environment variables known to the process
    show cli sockets         : dump list of cli sockets
    add acl                  : add acl entry
    clear acl                : clear the content of this acl
    del acl                  : delete acl entry
    get acl                  : report the patterns matching a sample for an ACL
    show acl [id]            : report available acls or dump an acl's contents
    add map                  : add map entry
    clear map                : clear the content of this map
    del map                  : delete map entry
    get map                  : report the keys and values matching a sample for a map
    set map                  : modify map entry
    show map [id]            : report available maps or dump a map's contents
    show pools               : report information about the memory pools usage

blog20181128-05.sh

> show stat

blog20181128-06.sh

$ watch 'echo "show stat" | socat stdio /var/run/hapee-lb.sock | cut -d "," -f 1-2,5-10,34-36 | column -s, -t'
  # pxname  svname    scur  smax  slim  stot  bin    bout   rate  rate_lim  rate_max
  fe_main   FRONTEND  10    10    7000  83    6347   15476  0     0         10
  be_stats  BACKEND   0     0     700   0     0      0      0     0
  be_app    websrv1   0     0     0     0     0      0      0
  be_app    websrv2   0     0     0     0     0      0      0
  be_app    BACKEND   0     0     700   0     0      0      0     0
  be_other  websrv1   0     10    322   6347  15476  14     20
  be_other BACKEND 10 10 700 83 6347 15476 0 10

blog20181128-07.sh

# Drain traffic
> set server be_app/webserv1 state drain

# Allow server to accept traffic again
> set server be_app/webserv1 state ready

blog20181128-08.sh

# Disable health checks
> disable health be_app/webserv1

# Enable health checks
> enable health be_app/webserv1

blog20181128-09.sh

# Change weight by percentage of its original value
> set server be_app/webserv1 weight 50%
 
# Change weight in proportion to other servers
> set server be_app/webserv1 weight 100

blog20181128-10.sh

# Disable a frontend
> disable frontend fe_main

# Enable frontend
> enable frontend fe_main

blog20181128-11.cfg

frontend ft_web
    bind 0.0.0.0:8080
    
    # Stick table definition  
    stick-table type ip size 1m expire 15m store gpt0,http_req_rate(10s)
    http-request allow if { src -f /etc/hapee/whitelist.lst }
    http-request track-sc0 src
    acl abuse src_http_req_rate ge 5
    http-request sc-set-gpt0(0) 1 if abuse 
    http-request deny if { sc0_get_gpt0 ge 1 }

blog20181128-12.sh

# List the contents of the stick table
> show table ft_web
  # table: ft_web, type: ip, size:1048576, used:2
  0x1c1efe0: key=127.0.0.1 use=0 exp=5600 gpt0=1 http_req_rate(10000)=50
  0x1c0d8e0: key=192.168.122.1 use=0 exp=7713 gpt0=0 http_req_rate(10000)=1

# Set an entry's GPT0 value to 0  
> set table ft_web key 127.0.0.1 data.gpt0 0

# Delete the entry
> clear table ft_web key 127.0.0.1
    
# Verify successful deletion
> show table ft_web
  # table: ft_web, type: ip, size:1048576, used:2
  0x1c0d8e0: key=192.168.122.1 use=0 exp=7700 gpt0=0 http_req_rate(10000)=0

# List remaining blocked client IPs (those with tag GPT0 value greater than 0)
> show table ft_web data.gpt0 gt 0

blog20181128-13.sh

> show acl
  # id (file) description               
  0 () acl 'path_beg' file '/etc/hapee/hapee-lb.cfg' line 60
  1 () acl 'nbsrv' file '/etc/hapee/hapee-lb.cfg' line 61
  2 (/etc/hapee/whitelist.lst) pattern loaded from file '/etc/hapee/whitelist.lst' used by acl at file '/etc/hapee/hapee-lb.cfg' line 67
  3 () acl 'src' file '/etc/hapee/hapee-lb.cfg' line 67
  4 () acl 'src_http_req_rate' file '/etc/hapee/hapee-lb.cfg' line 69
  5 () acl 'sc0_get_gpt0' file '/etc/hapee/hapee-lb.cfg' line 71
  6 () acl 'path_beg' file '/etc/hapee/hapee-lb.cfg' line 80
  7 () acl 'always_true' file '/etc/hapee/hapee-lb.cfg' line 106
    
# We are interested in the 5th ACL (ID #4)
# Display the ACL's current settings
> show acl #4
  0xc9a3e0 10
    
# Add a new rate of 60
> add acl #4 60
  
# Verify that both values are now present
> show acl #4
  0xc9a3e0 10
  0xce8f10 60
    
# Delete the old value in any of the following two ways - by value, or by using the reference number
> del acl #4 10
> del acl #4 #0xc9a3e0
    
# Verify that only one value now remains in effect
> show acl #4
0xce8f10 60 

blog20181128-14.cfg

http-request allow if { src -f /etc/hapee/whitelist.lst }

blog20181128-15.sh

> show acl
  # id (file) description               
  0 () acl 'path_beg' file '/etc/hapee/hapee-lb.cfg' line 60
  1 () acl 'nbsrv' file '/etc/hapee/hapee-lb.cfg' line 61
  2 (/etc/hapee/whitelist.lst) pattern loaded from file '/etc/hapee/whitelist.lst' used by acl at file '/etc/hapee/hapee-lb.cfg' line 67
  3 () acl 'src' file '/etc/hapee/hapee-lb.cfg' line 67
  4 () acl 'src_http_req_rate' file '/etc/hapee/hapee-lb.cfg' line 69
  5 () acl 'sc0_get_gpt0' file '/etc/hapee/hapee-lb.cfg' line 71
  6 () acl 'path_beg' file '/etc/hapee/hapee-lb.cfg' line 80
  7 () acl 'always_true' file '/etc/hapee/hapee-lb.cfg' line 106
    
# We are interested in the third entry (ID #2)
# Display its content
> show acl #2
  0x24de920 191.168.1.1                             
  0x24de960 191.168.1.2
  0x24de9a0 191.168.1.3
  0x24de9e0 191.168.1.4
  0x24dcf10 191.168.1.5
  0x24de300 191.168.1.6
  0x24de340 191.168.1.7
    
# Add the new entry
> add acl #2 192.168.1.9
   
# Delete the old entry
> del acl #2 #0x24de300
    
# Verify the result
> show acl #2
  0x24de920 191.168.1.1                  
  0x24de960 191.168.1.2
  0x24de9a0 191.168.1.3
  0x24de9e0 191.168.1.4
  0x24dcf10 191.168.1.5
  0x24de340 191.168.1.7
  0x24de300 192.168.1.9

blog20181128-16.sh

# List keys files
> show tls-keys
  # id (file)               
  0 (/dev/shm/hapee_ticket_keys.txt)
    
# List keys in a particular file
> show tls-keys #0
  # id secret               
  # 0 (/dev/shm/hapee_ticket_keys.txt)
  0.0 vaL65b3ns0mAJbiQRGiQZ84H4C9N1dZAyDYrHXVqG6KRDuXCo8mpwfk6+xPtlM1m
  0.1 fQxhSJT8sBKNb6JAFZT11UkzplfXEI1uUijPQUTBysZpNqzT26s2RVARxCoo5E52
  0.2 QtxrjBbPrX6z/PljdHIFqmHMH2/Rc5zZzIKklcfBPJa01G6PU9Dp9ixcibeisZxU
    
# Update key 
> set ssl tls-keys #0 CaF7HpWr0gUByzxDqlbvYXCFT2zqmhnKFAdbM4MyQHfty974QO31Pn1OLJIR92rk
  TLS ticket key updated!
    
# Verify successful update
> show tls-keys #0                                                                 
  # id secret
  # 0 (/dev/shm/hapee_ticket_keys.txt)
  0.0 fQxhSJT8sBKNb6JAFZT11UkzplfXEI1uUijPQUTBysZpNqzT26s2RVARxCoo5E52
  0.1 QtxrjBbPrX6z/PljdHIFqmHMH2/Rc5zZzIKklcfBPJa01G6PU9Dp9ixcibeisZxU
  0.2 CaF7HpWr0gUByzxDqlbvYXCFT2zqmhnKFAdbM4MyQHfty974QO31Pn1OLJIR92rk

blog20181128-17.sh

$ openssl s_client -servername hapee-mh -connect 127.0.0.1:443 -tlsextdebug -status
  CONNECTED(00000003)                                     
  TLS server extension "server name" (id=0), len=0
  TLS server extension "renegotiation info" (id=65281), len=1
  0001 - <SPACES/NULS>
  TLS server extension "EC point formats" (id=11), len=4
  0000 - 03 00 01 02                                       ....
  TLS server extension "session ticket" (id=35), len=0
  TLS server extension "status request" (id=5), len=0
  TLS server extension "heartbeat" (id=15), len=1
  0000 - 01                                                .
  depth=2 C = FR, ST = France, O = MMH-LAB, OU = SERVER, CN = MMH-CA
  verify error:num=19:self signed certificate in certificate chain
  verify return:0
  OCSP response: 
  ======================================
  OCSP Response Data:
      OCSP Response Status: successful (0x0)
      Response Type: Basic OCSP Response
      Version: 1 (0x0)
      Responder Id: C = FR, ST = France, O = MMH-LAB, OU = SERVER, CN = MMH-OCSP
      Produced At: Nov 20 16:00:00 2017 GMT
      Responses:
      Certificate ID:
          Hash Algorithm: sha1
          Issuer Name Hash: 1A7A927EC43135F4B6FF62065A09E1C7FD02557B
          Issuer Key Hash: A45FE1C0DDC03B42C34306A08DF14ED33B74B86C
          Serial Number: 1000
      Cert Status: good
          This Update: Nov 20 16:00:00 2017 GMT
Next Update: Nov 30 16:00:00 2017 GMT

blog20181128-18.sh

$ openssl ocsp -CAfile intermediate/certs/ca-chain.cert.pem -issuer intermediate/certs/intermediate.cert.pem -cert intermediate/certs/hapee-mmh.cert.pem -url http://localhost:8080 -respout  /tmp/ocsp-resp.der                                                                                                 
  Response verify OK
  intermediate/certs/hapee-mmh.cert.pem: good
      This Update: Nov 20 16:00:00 2017 GMT
      Next Update: Dec 10 16:00:00 2017 GMT
    
$ echo "set ssl ocsp-response $(base64 -w 10000 /tmp/ocsp-resp.der)" | socat stdio /var/run/hapee-lb.sock
OCSP Response updated!

blog20181128-19.sh

$ openssl s_client -servername hapee-mh -connect 127.0.0.1:443 -tlsextdebug -status
  CONNECTED(00000003)
   ...
   ...
  OCSP response: 
  ======================================
   ...
   ...
  Cert Status: good
  This Update: Nov 20 16:00:00 2017 GMT
Next Update: Dec 10 16:00:00 2017 GMT

blog20181128-20.sh

$ echo "This is invalid traffic\n\nMore invalid traffic" | nc -v hapee-lab 80
  Connection to hapee-lab 80 port [tcp/http] succeeded!
  HTTP/1.0 400 Bad request
  Cache-Control: no-cache
  Connection: close
  Content-Type: text/html
  400 Bad Request
  Your browser sent an invalid request.

blog20181128-21.txt

hapee-lab hapee-lb[17817]: 192.168.122.1:53918 [20/Nov/2017:13:29:11.305] fe_main fe_main/ -1/-1/-1/-1/+0 400 +187 - - PR-- 0/0/0/0/0 0/0 ""

blog20181128-22.sh

> show errors
  Total events captured on [10/Nov/2017:13:41:58.421] : 3
     
  [10/Nov/2017:13:41:43.328] frontend fe_main (#2): invalid request
  backend  (#-1), server  (#-1), event #2
  src 192.168.122.1:54154, session #15, session flags 0x00000080
  HTTP msg state MSG_RQVER(6), msg flags 0x00000000, tx flags 0x00000000
  HTTP chunk len 0 bytes, HTTP body len 0 bytes
  buffer flags 0x00808002, out 0 bytes, total 46 bytes
  pending 46 bytes, wrapping at 16384, error at position 8:
     
  00000  This is invalid traffic\n
  00024  \n
  00025 More invalid traffic\n

blog20181128-23.sh

> show sess
  0xd38210: proto=tcpv4 src=192.168.122.1:55234 fe=fe_main be= srv= ts=02 age=11s calls=1rq[f=400000h,i=0,an=34h,rx=,wx=,ax=] rp[f=80000000h,i=0,an=00h,rx=,wx=,ax=] s0=[7,8h,fd=1,ex=] s1=[0,10h,fd=-1,ex=] exp=
  0xd24dd0: proto=unix_stream src=unix:1 fe=GLOBAL be= srv= ts=02 age=0s calls=1 rq[f=c08202h,i=0,an=00h,rx=10m,wx=,ax=] rp[f=80008002h,i=0,an=00h,rx=,wx=,ax=] s0=[7,8h,fd=2,ex=] s1=[7,4018h,fd=-1,ex=] exp=10m

blog20181128-24.sh

> shutdown session 0xd38210

blog20181128-25.cfg

log-format "%ci:%cp_%[src,map_ip(/etc/hapee/ip-country.lst)] [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"

blog20181128-26.txt

192.168.122.1:51000_FR [20/Nov/2017:19:34:11.688] fe_main be_app/websrv1 0/0/0/0/0 200 5720 - - --NI 1/1/0/1/0 0/0 "GET / HTTP/1.1"

blog20181128-27.txt

2.0.0.0/12 FR
2.16.0.0/24 FR
2.16.2.0/23 FR
2.16.10.0/24 FR

blog20181128-28.sh

# Loop through the map entries on disk and add them to the running configuration
$ IFS=$'\n'
$ for ip_country in $(cat /tmp/more-ip-countries.lst); do
    echo "add map /tmp/ip-country.lst $ip_country"
done | socat stdio /var/run/hapee-lb.sock