Circuit Breaking in HAProxy

blog20201222-01.cfg

backend serviceA
   default-server maxconn 30  check  observe layer7  error-limit 50  on-error mark-down  inter 1s  rise 30  slowstart 20s
   server s1 192.168.0.10:80
   server s2 192.168.0.11:80

blog20201222-02.cfg

backend serviceA
   # Create storage for tracking client info
   stick-table  type string  size 1  expire 30s  store http_req_rate(10s),gpc0,gpc0_rate(10s),gpc1

   # Is the circuit broken?
   acl circuit_open be_name,table_gpc1 gt 0

   # Reject request if circuit is broken
   http-request return status 503 content-type "application/json" string "{ \"message\": \"Circuit Breaker tripped\" }" if circuit_open

   # Begin tracking requests
   http-request track-sc0 be_name

   # Count HTTP 5xx server errors
   http-response sc-inc-gpc0(0) if { status ge 500 }

   # Store the HTTP request rate and error rate in variables
   http-response set-var(res.req_rate) sc_http_req_rate(0)
   http-response set-var(res.err_rate) sc_gpc0_rate(0)

   # Check if error rate is greater than 50% using some math
   http-response sc-inc-gpc1(0) if { int(100),mul(res.err_rate),div(res.req_rate) gt 50 }
  
   server s1 192.168.0.10:80 check
   server s2 192.168.0.11:80 check

blog20201222-03.cfg

stick-table  type string  size 1  expire 30s  store http_req_rate(10s),gpc0,gpc0_rate(10s),gpc1

blog20201222-04.cfg

acl circuit_open be_name,table_gpc1 gt 0

blog20201222-05.cfg

http-request return status 503 content-type "application/json" string "{ \"message\": \"Circuit Breaker tripped\" }" if circuit_open

blog20201222-06.cfg

http-request track-sc0 be_name

blog20201222-07.cfg

http-response sc-inc-gpc0(0) if { status ge 500 }

blog20201222-08.cfg

http-response set-var(res.req_rate) sc_http_req_rate(0)
http-response set-var(res.err_rate) sc_gpc0_rate(0)

blog20201222-09.cfg

http-response sc-inc-gpc1(0) if { int(100),mul(res.err_rate),div(res.req_rate) gt 50 }

blog20201222-10.cfg

http-response sc-inc-gpc1(0) if { var(res.req_rate) gt 100 } { int(100),mul(res.err_rate),div(res.req_rate) gt 50 }

blog20201222-11.cfg

backend serviceA
   default-server maxconn 30  check  observe layer7  error-limit 50  on-error mark-down  inter 1s  rise 30  slowstart 20s

   # Circuit break the whole backend if the number 
   # of servers becomes less than or equal to 4
   http-request return status 503 content-type "application/json" string "{ \"message\": \"Circuit Breaker tripped\" }" if { nbsrv() le 4 }

   server s1 192.168.0.10:80
   server s2 192.168.0.11:80
   server s3 192.168.0.12:80
   server s4 192.168.0.13:80
   server s5 192.168.0.14:80
   server s6 192.168.0.15:80
   server s7 192.168.0.16:80
   server s8 192.168.0.17:80
   server s9 192.168.0.18:80
   server s10 192.168.0.19:80