Creating an HAProxy AI Gateway to Control LLM Costs, Security, and Privacy

blog20240711-01.cfg

http-request set-var(txn.openai_key_hash) 
log-format-sd "%{+Q,+E}o[request@58750 host=%[var(txn.host)] referer=%[var(txn.referer)] user_agent=%[var(txn.user_agent)]][custom@58750 openai_key_hash=%[var(txn.openai_key_hash)]]"
http_auth_bearer(Authorization),sha2(256),hex

blog20240711-02.txt

# example denylist.acl
5fd924625a10e0baacdb8

blog20240711-03.txt

acl blocked_key var(txn.openai_key_hash) -m -i -f /denylist.acl
Http-request deny deny_status 403 if blocked_key

blog20240711-04.txt

<key hash> <per minute prompt limit>:<per day prompt limit>:<per minute completion limit>:<per day completion limit>

blog20240711-05.txt

5fd924625a10e0baacdb8 100:200:1000:50000
813490e4ba67813490e4  300:600:2000:30000

blog20240711-06.cfg

backend rates
   stick-table rates_prompt_minute.local type string len 128 size 50k expire 1m store gpc(1),gpc_rate(1,60s) peers "$peers_section_name"
    stick-table rates_prompt_minute.aggregate type string len 128 size 50k expire 1m store gpc(1),gpc_rate(1,60s) peers "$peers_section_name"
    stick-table rates_prompt_day.local type string len 128 size 50k expire 24h store gpc(1),gpc_rate(1,1d) peers "$peers_section_name"
    stick-table rates_prompt_day.aggregate type string len 128 size 50k expire 24h store gpc(1),gpc_rate(1,1d) peers "$peers_section_name"
   stick-table rates_completion_minute.local type string len 128 size 50k expire 1m store gpc(1),gpc_rate(1,60s) peers "$peers_section_name"
    stick-table rates_completion_minute.aggregate type string len 128 size 50k expire 1m store gpc(1),gpc_rate(1,60s) peers "$peers_section_name"
    stick-table rates_completion_day.local type string len 128 size 50k expire 24h store gpc(1),gpc_rate(1,1d) peers "$peers_section_name"
    stick-table rates_completion_day.aggregate type string len 128 size 50k expire 24h store gpc(1),gpc_rate(1,1d) peers "$peers_section_name"

blog20240711-07.cfg

frontend mysite
  http-request set-var(txn.maxrate_min_prompt) var(txn.openai_key_hash),map(/rate-limits.map,0),field(1,:)
  http-request set-var(txn.maxrate_day_prompt) var(txn.openai_key_hash),map(rate-limits.map,0),field(2,:)
  http-request set-var(txn.maxrate_min_completion) var(txn.openai_key_hash),map(rate-limits.map,0),field(3,:)
  http-request set-var(txn.maxrate_day_completion) var(txn.openai_key_hash),map(rate-limits.map,0),field(4,:)

blog20240711-08.cfg

http-request track-sc0 var(txn.openai_key_hash) table rates/rates_prompt_minute.local
http-request track-sc1 var(txn.openai_key_hash) table rates/rates_prompt_minute.aggregate
http-request track-sc2 var(txn.openai_key_hash) table rates/rates_prompt_day.local
http-request track-sc3 var(txn.openai_key_hash) table rates/rates_prompt_day.aggregate

http-request track-sc4 var(txn.openai_key_hash) table rates/rates_completion_minute.local
http-request track-sc5 var(txn.openai_key_hash) table rates/rates_completion_minute.aggregate
http-request track-sc6 var(txn.openai_key_hash) table rates/rates_completion_day.local
http-request track-sc7 var(txn.openai_key_hash) table rates/rates_completion_day.aggregate

blog20240711-09.cfg

http-request set-var(txn.rate_prompt_minute) sc_gpc_rate(0,1) 
http-request set-var(txn.rate_prompt_day) sc_gpc_rate(0,3) 
http-request set-var(txn.rate_completion_minute) sc_gpc_rate(0,5) 
http-request set-var(txn.rate_completion_day) sc_gpc_rate(0,7)

blog20240711-10.txt

If (Current rate - Maximum rate <= 0) then
   Over the limit

blog20240711-11.cfg

http-request deny status 429  if { var(txn.rate_prompt_minute),sub(txn.maxrate_min_prompt) gt 0 }
http-request deny status 429  if { var(txn.rate_prompt_day),sub(txn.maxrate_day_prompt) gt 0 }
http-request deny status 429  if { var(txn.rate_completion_minute),sub(txn.maxrate_min_completion) gt 0 }
http-request deny status 429  if { var(txn.rate_completion_day),sub(txn.maxrate_day_completion) gt 0 }

blog20240711-12.cfg

http-response set-var(txn.prompt_tokens) res.body,json_query('$.usage.prompt_tokens','int')
http-response set-var(txn.completion_tokens) res.body,json_query('$.usage.completion_tokens','int')

blog20240711-13.cfg

http-request sc-add-gpc(0,0) var(txn.smtp.prompt_tokens) if { var(txn.rate_prompt_minute),sub(txn.maxrate_min_prompt) le 0 }
http-request sc-add-gpc(0,2) var(txn.smtp.prompt_tokens) if { var(txn.rate_prompt_day),sub(txn.maxrate_day_prompt) le 0 }
http-request sc-add-gpc(0,4) var(txn.smtp.completion_tokens) if { var(txn.rate_prompt_minute),sub(txn.maxrate_min_completion) le 0 }
http-request sc-add-gpc(0,6) var(txn.smtp.completion_tokens) if { var(txn.rate_prompt_day),sub(txn.maxrate_day_comp.etion) le 0 }