haproxytechblog/blog20240626-01.cfg Secret

## blog20240626-01.cfg
frontend main
  bind :443  ssl crt-list /etc/haproxy/certs/crt-list.txt
  default_backend app

## blog20240626-02.txt
/etc/haproxy/certs/site1.pem [alpn h2 ocsp-update on]

## blog20240626-03.cfg
crt-store web
  crt-base /etc/haproxy/ssl/certs
  load crt "site1.pem" ocsp "site1.ocsp" alias "site1" ocsp-update on

## blog20240626-04.cfg
frontend  main
  bind *:443 ssl crt "@web/site1"
  default_backend app

## blog20240626-05.cfg
crt-store web
  crt-base /etc/ssl/certs/
  key-base /etc/ssl/private/
  load crt "site1.crt" alias "site1"
  load crt "site2.crt" key "site2.key"

## blog20240626-06.cfg
frontend main
  bind *:443 ssl crt "@web/site1" crt "@web/site2.crt"

## blog20240626-07.cfg
global
  ssl-security-level 5

## blog20240626-08.cfg
frontend  main
  guid 64aeebba-0911-483c-aa39-8f689cf3f657
  bind *:80
  default_backend app

backend app
  guid 9de9a5de-c3c2-47c4-b945-c4f61b7ece6a
  balance roundrobin
  server app1 127.0.0.1:8080 check guid ae9194cc-f473-43de-ac21-43142340cf9c

## blog20240626-09.sh
$ sudo systemctl reload haproxy

## blog20240626-10.sh
$ echo "dump stats-file" | \
  sudo socat stdio tcp4-connect:127.0.0.1:9999 > <path/to/stats/file>

## blog20240626-11.cfg
global
  stats-file <path/to/stats/file>

## blog20240626-12.cfg
bind :80 guid-prefix my-prefix

## blog20240626-13.cfg
backend app
  balance hash pathq
  hash-type consistent
  server app1 192.168.56.30:80 check hash-key addr
  server app2 192.168.56.31:80 check hash-key addr

## blog20240626-14.cfg
backend app
  server app1 /var/run/my-application-socket.sock check send-proxy namespace my-namespace

## blog20240626-15.cfg
log-format "$HAPROXY_HTTP_LOG_FMT fs-aborted: %[fs.aborted] fs-rst-code: %[fs.rst_code] bs-aborted: %[bs.aborted] bs-rst-code: %[bs.rst_code]"

## blog20240626-16.txt
fs-aborted: 1 fs-rst-code: 8 bs-aborted: 0 bs-rst-code: -

## blog20240626-17.cfg
frontend mysite
  bind :80
  log-format "%{+json}o %(client_ip)ci %(client_port)cp %(request_date)tr %(frontend_name)ft %(backend_name)b %(server_name)s %(time_to_receive)TR %(time_waiting)Tw %(time_to_connect)Tc %(time_server_response)Tr %(time_active)Ta %(status_code)ST %(bytes_read)B %(request_cookies)CC %(response_cookies)CS %(termination_state)tsc %(process_active_connections)ac %(frontend_active_connections)fc %(backend_active_connections)bc %(server_active_connections)sc %(retries)rc %(server_queue)sq %(backend_queue)bq %(request_headers)hr %(response_headers)hs %(request_line)r"
  use_backend webservers

## blog20240626-18.json
{"client_ip": "172.21.0.1", "client_port": 57526, "request_date": "05/Jun/2024:20:59:57.698", "frontend_name": "mysite", "backend_name": "webservers", "server_name": "web1", "time_to_receive": 0, "time_waiting": 0, "time_to_connect": 0, "time_server_response": 0, "time_active": 0, "status_code": 200, "bytes_read": 993, "request_cookies": "", "response_cookies": "", "termination_state": "----", "process_active_connections": 1, "frontend_active_connections": 1, "backend_active_connections": 0, "server_active_connections": 0, "retries": 0, "server_queue": 0, "backend_queue": 0, "request_headers": null, "response_headers": null, "request_line": "GET / HTTP/1.1"}

## blog20240626-19.cbor
BF69636C69656E745F69706A3137322E32312E302E316B636C69656E745F706F727419E0306C726571756573745F64617465781830352F4A756E2F323032343A32313A30323A33392E3334346D66726F6E74656E645F6E616D657F666D7973697465FF6C6261636B656E645F6E616D656A776562736572766572736B7365727665725F6E616D6564776562316F74696D655F746F5F72656365697665006C74696D655F77616974696E67006F74696D655F746F5F636F6E6E656374007474696D655F7365727665725F726573706F6E7365006B74696D655F616374697665006B7374617475735F636F646518C86A62797465735F726561641903E16F726571756573745F636F6F6B696573F670726573706F6E73655F636F6F6B696573F6717465726D696E6174696F6E5F7374617465642D2D2D2D781A70726F636573735F6163746976655F636F6E6E656374696F6E7301781B66726F6E74656E645F6163746976655F636F6E6E656374696F6E7301781A6261636B656E645F6163746976655F636F6E6E656374696F6E730078197365727665725F6163746976655F636F6E6E656374696F6E73006772657472696573006C7365727665725F7175657565006D6261636B656E645F7175657565006F726571756573745F68656164657273F670726573706F6E73655F68656164657273F66C726571756573745F6C696E657

## blog20240626-20.cfg
frontend mysite
  bind :80
  acl denylist src -f virt@denylist.acl
  http-request deny if denylist
  default_backend webservers

## blog20240626-21.sh
$ echo "add acl virt@denylist.acl 172.20.0.1" | sudo socat stdio tcp4-connect:127.0.0.1:9999

## blog20240626-22.cfg
frontend mysite
  bind :80

  # Store path in a variable
  http-request set-var(txn.path) path

  use_backend %[var(txn.path),map_beg(/etc/haproxy/paths.map,webservers)]

  # Custom log format using map_beg_key
  log-format "Matching key: %[var(txn.path),map_beg_key(/etc/haproxy/paths.map)]"

## blog20240626-23.txt
/api  apiservers
/cart cartservers

## blog20240626-24.txt
Matching key: /api
Matching key: /cart
Matching key: -

## blog20240626-25.cfg
frontend mysite
  bind :80
  bind :443 ssl default-crt /certs/default.pem.ecdsa /certs/default.pem.rsa crt /certs/

  # Redirects to HTTPS
  http-request redirect scheme https unless { ssl_fc }

## blog20240626-26.cfg
global
  http-err-codes 403

frontend mysite
  bind :80

  # Create storage for tracking client errors
  stick-table  type ip  size 1m  expire 24h  store http_err_cnt

  # Begin tracking requests
  http-request track-sc0 src

  default_backend webservers

## blog20240626-27.sh
$ echo "show table mysite" | socat stdio tcp4-connect:127.0.0.1:9999

# table: mysite, type: ip, size:1048576, used:1
0x7f99498020e8: key=172.19.0.10 use=0 exp=86396991 shard=0 http_err_cnt=20

## blog20240626-28.sh
$ echo "show table mysite" | socat stdio tcp4-connect:127.0.0.1:9999

# table: mysite, type: ip, size:1048576, used:1
0x7f7de4bb50d8: key=172.20.0.1 use=0 exp=86389115 shard=0 http_req_cnt=4

## blog20240626-29.sh
$ echo "set table mysite ptr 0x7f7de4bb50d8 data.http_req_cnt 0" | socat stdio tcp4-connect:127.0.0.1:9999

## blog20240626-30.sh
$ echo "clear table mysite ptr 0x7f7de4bb50d8" | socat stdio tcp4-connect:127.0.0.1:9999

## blog20240626-31.cfg
global
  setcap cap_net_bind_service,cap_net_admin

## blog20240626-32.sh
$ sudo setcap cap_net_bind_service,cap_net_admin=p /usr/local/sbin/haproxy

## blog20240626-33.cfg
global
  setcap cap_net_admin

## blog20240626-34.cfg
backend servers
  http-request set-bc-mark 2
  server server1 192.168.56.11:80

## blog20240626-35.sh
$ sudo ip rule add fwmark 2 table 2
$ sudo ip route add table 2 default via 192.168.56.1 dev eth2

## blog20240626-36.sh
$ sudo iptables -A OUTPUT --dst 192.168.56.11 -j LOG --log-prefix "output: "

## blog20240626-37.sh
$ sudo tail -f /var/log/kern.log

output: IN= OUT=eth2 SRC=192.168.56.5 DST=192.168.56.11 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=21392 DF PROTO=TCP SPT=54738 DPT=80 WINDOW=64240 RES=0x00 SYN URGP=0 MARK=0x2

## blog20240626-38.sh
$ sudo setcap 'cap_net_admin=ep' /usr/local/sbin/haproxy

## blog20240626-39.cfg
global
  setcap cap_net_admin

## blog20240626-40.cfg
backend servers
  http-request set-bc-tos 26
  server server1 192.168.56.11:80

## blog20240626-41.cfg
log-forward syslog
  bind :514
  dgram-bind :514
  log backend@mylog local1

backend mylog
  mode log
  balance roundrobin

  server log1 udp@192.168.56.42:514 weight 10
  server log2 udp@192.168.56.43:514 weight 20

## blog20240626-42.json
{
    "log-driver": "syslog",
    "log-opts": {
      "syslog-address": "udp://127.0.0.1:514",
      "syslog-facility": "local1"
    }
}

## blog20240626-43.sh
$ sudo docker run -p 80:80 -it --rm nginx:latest

## blog20240626-44.cfg
frontend mysite
  bind :80
  unique-id-format %[uuid(7)]
  log-format "%{+json}o %(id)[unique-id] %(client_ip)ci %(client_port)cp %(request_line)r"
  use_backend webservers

## blog20240626-45.json
{"id": "018fea59-8306-7d01-9a97-01332adf4905", "client_ip": "172.21.0.1", "client_port": 53280, "request_line": "GET /favicon.ico HTTP/1.1"}

## blog20240626-46.cfg
global
  ocsp-update.httpproxy 192.168.0.10:8000

## blog20240626-47.txt
scrape_configs:
- job_name: 'load-balancer-metrics'
  static_configs:
  - targets: ['haproxy:8405']
  params:
    extra-counters: ["on"]

## blog20240626-48.cfg
global
  tune.ssl.keylog on

## blog20240626-49.cfg
backend servers
  server s1 192.168.56.50:443 ssl verify required ca-file /etc/haproxy/certs/ca.crt ssl-min-ver TLSv1.3

## blog20240626-50.cfg
frontend mysite
  log-format "CLIENT_EARLY_TRAFFIC_SECRET %[ssl_bc_client_random,hex] %[ssl_bc_client_early_traffic_secret]\nCLIENT_HANDSHAKE_TRAFFIC_SECRET %[ssl_bc_client_random,hex] %[ssl_bc_client_handshake_traffic_secret]\nSERVER_HANDSHAKE_TRAFFIC_SECRET %[ssl_bc_client_random,hex] %[ssl_bc_server_handshake_traffic_secret]\nCLIENT_TRAFFIC_SECRET_0 %[ssl_bc_client_random,hex] %[ssl_bc_client_traffic_secret_0]\nSERVER_TRAFFIC_SECRET_0 %[ssl_bc_client_random,hex] %[ssl_bc_server_traffic_secret_0]\nEXPORTER_SECRET %[ssl_bc_client_random,hex] %[ssl_bc_exporter_secret]\nEARLY_EXPORTER_SECRET %[ssl_bc_client_random,hex] %[ssl_bc_early_exporter_secret]"

## blog20240626-51.txt
CLIENT_EARLY_TRAFFIC_SECRET C030AF8EAEE688F1F3A360E5D53E260DEAB346F93CE594153D95E33E4BFD5F80 -
CLIENT_HANDSHAKE_TRAFFIC_SECRET C030AF8EAEE688F1F3A360E5D53E260DEAB346F93CE594153D95E33E4BFD5F80 15ab9abf57145fe49c73d9a617eca9b918d5c4dd455c4bb923c04a936475241facbac21f66bca7c459f5179f753f4afa
SERVER_HANDSHAKE_TRAFFIC_SECRET C030AF8EAEE688F1F3A360E5D53E260DEAB346F93CE594153D95E33E4BFD5F80 09bded135c6b85959d0c2eaf09d177cc4fb9e2d9777cbda5a234d0894ef84b64bbd346cc331a16111d4273d639090d5b
CLIENT_TRAFFIC_SECRET_0 C030AF8EAEE688F1F3A360E5D53E260DEAB346F93CE594153D95E33E4BFD5F80 155b07c8fcef945cbad456f6b11e216fde42f9ac1cdc8c6eff4bed845caf520a2a490ccba3ae06ffe3d9091904674c41
SERVER_TRAFFIC_SECRET_0 C030AF8EAEE688F1F3A360E5D53E260DEAB346F93CE594153D95E33E4BFD5F80 18ed3dc1188b7ed1085cbdf41b0f0388b80904f6f21b8962f57cdf460d5694f2b2d99f7055ac44f0e6afefc9e790626b
EXPORTER_SECRET C030AF8EAEE688F1F3A360E5D53E260DEAB346F93CE594153D95E33E4BFD5F80 9479651fd91e38d549b284ecae7c6430743ae56cc4e8fb899eaf0a4016891d3991b01691c1c4c787d95a10c

## blog20240626-52.sh
$ tcpdump -s 0 port 443 -i eth0 -w mycap.pcap

## blog20240626-53.sh
$ echo "show quic" | socat stdio tcp4-connect:127.0.0.1:9999
# conn/frontend                     state   in_flight infl_p lost_p         Local Address           Foreign Address      local & remote CIDs
0x7f3501354e80[04]/mysite           ESTAB           0      0      0        172.22.0.5:443          172.22.0.1:40761 2beffb69742599f2 fd1050

$ echo "show quic tp" | socat stdio tcp4-connect:127.0.0.1:9999
* 0x7f44d3b356a0[00]: scid=afa23221abf0a878........................ dcid=d473e9..................................
  loc. TPs: odcid=0917fd0b378fc3ae5b761af255 iscid=afa23221abf0a878
    midle_timeout=30000ms mudp_payload_sz=2048 ack_delay_exp=3 mack_delay=25ms act_cid_limit=8
    md=1687140 msd_bidi_l=16380 msd_bidi_r=16380 msd_uni=16380 ms_bidi=100 ms_uni=3
    (no_act_migr,stless_rst_tok)
  rem. TPs: iscid=d473e9
    midle_timeout=30000ms mudp_payload_sz=65527 ack_delay_exp=3 mack_delay=20ms act_cid_limit=8
    md=25165824 msd_bidi_l=12582912 msd_bidi_r=1048576 msd_uni=1048576 ms_bidi=16 ms_uni=16
    (no_act_migr)    versions:chosen=0x00000001,negotiated=0x00000001
  st=opened           mux=ready                                     expire=04s

$ echo "show quic sock" | socat stdio tcp4-connect:127.0.0.1:9999
* 0x7f44d3b36670[01]: scid=36416af6fda6c5d2........................ dcid=47e1d4..................................
  st=opened           mux=ready                                     expire=18s
  fd=71               local_addr=172.22.0.5:443 foreign_addr=172.22.0.1:40203

$ echo "show quic pktns" | socat stdio tcp4-connect:127.0.0.1:9999
* 0x7f44d3b36670[01]: scid=36416af6fda6c5d2........................ dcid=47e1d4..................................
  st=opened           mux=ready                                     expire=07s
  [01rtt] rx.ackrng=1      tx.inflight=0

$ echo "show quic cc" | socat stdio tcp4-connect:127.0.0.1:9999
* 0x7f44d3b356a0[02]: scid=6efe15fc4097ac7d........................ dcid=5ed3d6..................................
  st=opened           mux=ready                                     expire=27s
  srtt=1    rttvar=0    rttmin=1    ptoc=0    cwnd=17093  mcwnd=17093  sentpkts=13     lostpkts=0      reorderedpkts=0

$ echo "show quic mux" | socat stdio tcp4-connect:127.0.0.1:9999
* 0x7f44d3b356a0[02]: scid=6efe15fc4097ac7d........................ dcid=5ed3d6..................................
  st=opened           mux=ready                                     expire=16s
  qcc=0x0x7f44cde23090 flags=0x0 sc=0 hreq=0
    qcs=0x0x7f44cde36c60 id=2 flags=0x0 st=OPN rxoff=29
    qcs=0x0x7f44cde36930 id=3 flags=0x0 st=OPN txoff=17
    qcs=0x0x7f44cde36df0 id=6 flags=0x0 st=OPN rxoff=1
    qcs=0x0x7f44cde0c050 id=10 flags=0x0 st=OPN rxoff=1

## blog20240626-54.sh
$ echo "add server app/app2 172.16.0.12:8080 cookie app2" | socat stdio tcp4-connect:127.0.0.1:31232

## blog20240626-55.txt
New server registered.

## blog20240626-56.sh
$ echo "show activity; wait 10s; show activity" | socat -t 20 stdio tcp4-connect:127.0.0.1:9999

## blog20240626-57.txt
thread_id: 1 (1..2)
date_now: 1717443564.121845
uptime_now: 659.199676
[...]


Done.

thread_id: 1 (1..2)
date_now: 1717443574.123014
uptime_now: 669.200845

[...]

## blog20240626-58.sh
$ echo "disable server app/app1; shutdown sessions server app/app1; wait 2s srv-removable app/app1; del server app/app1" | socat stdio tcp4-connect:127.0.0.1:31232

## blog20240626-59.txt
Done.

Server deleted.

## blog20240626-60.sh
$ echo "add server app/app2 127.0.0.1:5571 enabled check cookie app2" | socat stdio tcp4-connect:127.0.0.1:9999

## blog20240626-61.txt
'enabled' option is not accepted for dynamic server
	frontend main
	bind :443 ssl crt-list /etc/haproxy/certs/crt-list.txt
	default_backend app
	crt-store web
	crt-base /etc/haproxy/ssl/certs
	load crt "site1.pem" ocsp "site1.ocsp" alias "site1" ocsp-update on
	frontend main
	bind *:443 ssl crt "@web/site1"
	default_backend app
	crt-store web
	crt-base /etc/ssl/certs/
	key-base /etc/ssl/private/
	load crt "site1.crt" alias "site1"
	load crt "site2.crt" key "site2.key"
	frontend main
	bind *:443 ssl crt "@web/site1" crt "@web/site2.crt"
	frontend main
	guid 64aeebba-0911-483c-aa39-8f689cf3f657
	bind *:80
	default_backend app

	backend app
	guid 9de9a5de-c3c2-47c4-b945-c4f61b7ece6a
	balance roundrobin
	server app1 127.0.0.1:8080 check guid ae9194cc-f473-43de-ac21-43142340cf9c
	$ echo "dump stats-file" \| \
	sudo socat stdio tcp4-connect:127.0.0.1:9999 > <path/to/stats/file>
	backend app
	balance hash pathq
	hash-type consistent
	server app1 192.168.56.30:80 check hash-key addr
	server app2 192.168.56.31:80 check hash-key addr
	backend app
	server app1 /var/run/my-application-socket.sock check send-proxy namespace my-namespace
	frontend mysite
	bind :80
	log-format "%{+json}o %(client_ip)ci %(client_port)cp %(request_date)tr %(frontend_name)ft %(backend_name)b %(server_name)s %(time_to_receive)TR %(time_waiting)Tw %(time_to_connect)Tc %(time_server_response)Tr %(time_active)Ta %(status_code)ST %(bytes_read)B %(request_cookies)CC %(response_cookies)CS %(termination_state)tsc %(process_active_connections)ac %(frontend_active_connections)fc %(backend_active_connections)bc %(server_active_connections)sc %(retries)rc %(server_queue)sq %(backend_queue)bq %(request_headers)hr %(response_headers)hs %(request_line)r"
	use_backend webservers