Skip to content

Instantly share code, notes, and snippets.

@haproxytechblog
Last active January 24, 2024 13:23
Show Gist options
  • Save haproxytechblog/87da0cc17e1711c4825ecd7e5a24441c to your computer and use it in GitHub Desktop.
Save haproxytechblog/87da0cc17e1711c4825ecd7e5a24441c to your computer and use it in GitHub Desktop.
Bot Protection with HAProxy
backend per_ip_and_url_rates
stick-table type binary len 8 size 1m expire 24h store http_req_rate(24h)
backend per_ip_rates
stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s)
frontend fe_main
bind :80
# track client's source IP in per_ip_rates stick table
http-request track-sc0 src table per_ip_rates
# track client's source IP + URL accessed in
# per_ip_and_url_rates stick table
http-request track-sc1 url32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif }
# Increment general-purpose counter in per_ip_rates if client
# is visiting page for the first time
http-request sc-inc-gpc0(0) if { sc_http_req_rate(1) eq 1 }
default_backend web_servers
frontend fe_main
bind :80
http-request track-sc0 src table per_ip_rates
http-request track-sc1 url32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif }
# Set the threshold to 15 within the time period
acl exceeds_limit sc_gpc0_rate(0) gt 15
# Increase the new-page count if this is the first time
# they've accessed this page, unless they've already
# exceeded the limit
http-request sc-inc-gpc0(0) if { sc_http_req_rate(1) eq 1 } !exceeds_limit
# Deny requests if over the limit
http-request deny if exceeds_limit
default_backend web_servers
backend per_ip_rates
stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s),gpt0
http-request sc-set-gpt0(0) 1 if exceeds_limit
http-request deny if { sc_get_gpt0(0) eq 1 }
http-request sc-set-gpt0(0) 1 if exceeds_limit
use_backend be_bot_jail if { sc_get_gpt0(0) eq 1 }
$ echo "show table per_ip_and_url_rates" | socat stdio /var/run/hapee-1.8/hapee-lb.sock
# table: per_ip_and_url_rates, type: binary, size:1048576, used:2
0x10ab92c: key=203E97AA7F000001000000000000000000000000 use=0 exp=557590 http_req_rate(86400000)=1
0x10afd7c: key=3CBC49B17F000001000000000000000000000000 use=0 exp=596584 http_req_rate(86400000)=5
# table: per_ip_rates, type: ip, size:1048576, used:1
0x10ab878: key=127.0.0.1 use=0 exp=594039 gpc0=2 gpc0_rate(30000)=2
http-request use-service lua.request_recaptcha unless { lua.verify_solved_captcha "ok" } { sc_get_gpt0(0) eq 1 }
http-request track-sc1 base32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif }
http-request sc-inc-gpc0(0) if { sc_http_req_rate(1) eq 1 } !exceeds_limit
backend per_ip_and_url_bruteforce
stick-table type binary len 8 size 1m expire 10m store http_req_rate(3m)
http-request track-sc2 base32+src table per_ip_and_url_bruteforce if METH_POST { path /login }
http-request deny if { sc_http_req_rate(2) gt 10 }
backend per_ip_rates
stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s),http_err_rate(5m)
http-request deny if { sc_http_err_rate(0) gt 10 }
http-request sc-inc-gpc0(0) if { sc_http_err_rate(0) eq 1 } !exceeds_limit
use_backend be_honeypot if { sc_http_err_rate(0) gt 10 }
http-request track-sc1 url32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif } || { src -f /etc/hapee-1.8/whitelist.acl }
unless { src -f /etc/hapee-1.8/whitelist.acl -f /etc/hapee-1.8/admins.acl }
module-load hapee-lb-maxmind.so
maxmind-load COUNTRY /etc/hapee-1.8/geolocation/GeoLite2-Country.mmdb
maxmind-cache-size 10000
http-request set-header x-geoip-country %[src,maxmind-lookup(COUNTRY,country,iso_code)]
x-geoip-country: US
module-load hapee-lb-netacuity.so
netacuity-load 26 /etc/hapee-1.8/geolocation/netacuity/
netacuity-cache-size 10000
http-request set-header x-geoip-country %[src,netacuity-lookup-ipv4(“pulse-two-letter-country”)]
x-geoip-country: US
import sys
ip_blocks_file = sys.argv[1]
city_locations_file = sys.argv[2]
#First load the city locations into memory, as we will be using them a lot
city_locations = {}
city_locations_handle = open(city_locations_file,'r')
for city_location_line in city_locations_handle.readlines():
city_location_parts = city_location_line.split(",")
if len(city_location_parts) < 13:
continue
if not city_location_parts[0].isdigit():
continue
location_id=city_location_parts[0]
locale_code=city_location_parts[1]
continent_code=city_location_parts[2]
continent_name=city_location_parts[3]
country_iso_code=city_location_parts[4]
country_name=city_location_parts[5]
subdivision_1_iso_code=city_location_parts[6]
subdivision_1_name=city_location_parts[7]
subdivision_2_iso_code=city_location_parts[8]
subdivision_2_name=city_location_parts[9]
city_name=city_location_parts[10]
metro_code=city_location_parts[11]
time_zone=city_location_parts[12]
#print "Found country code '" + str(country_iso_code) + "' for id '" + str(location_id) + "'"
city_locations[location_id] = [country_iso_code, city_name]
#print "Country code for 10471023: " + str(city_locations[str(10471023)][0])
#Next build the country_iso_code and city_name files with this data
#Open map file handles
country_iso_code_file= open('country_iso_code.map', 'w')
city_name_file = open('city_name.map', 'w')
gps_map_file = open('gps.map', 'w')
#Process the lines of the ip block file
ip_blocks_handle = open(ip_blocks_file,'r')
for ip_block_line in ip_blocks_handle.readlines():
ip_block_line_parts = ip_block_line.split(',')
if len(ip_block_line_parts) < 9:
continue
network=ip_block_line_parts[0]
geoname_id=ip_block_line_parts[1]
#Per docs "registered" is where the IP is registered, rather then used
registered_country_geoname_id=ip_block_line_parts[2]
#"represented" only applies to military bases/etc and is their country
represented_country_geoname_id=ip_block_line_parts[3]
is_anonymous_proxy=ip_block_line_parts[4]
is_satellite_provider=ip_block_line_parts[5]
postal_code=ip_block_line_parts[6]
latitude=ip_block_line_parts[7]
longitude=ip_block_line_parts[8].rstrip() #Last column gets a newline appended to it
if not geoname_id in city_locations:
continue
#Write the country map line
country_iso_code_file.write(network + ' ' + city_locations[geoname_id][0] + '\n')
#Write the city map line
city_name_file.write(network + ' ' + city_locations[geoname_id][1].strip('"') + '\n')
#Write the GPS map line
gps_map_file.write(network + ' ' + longitude + ", " + latitude + '\n')
country_iso_code_file.close()
city_name_file.close()
gps_map_file.close()
python read_city_map.py GeoLite2-City-CSV_20181127/GeoLite2-City-Blocks-IPv4.csv GeoLite2-City-CSV_20181127/GeoLite2-City-Locations-en.csv
http-request set-header x-geoip-country %[src,map(/etc/hapee-1.8/country_iso_code.map)]
x-geoip-country: US
use_backend be_honeypot if { sc_http_err_rate(0) gt 5 } { req.hdr(x-geoip-country) CN }
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment