Bot Protection with HAProxy

blog20181127-01.cfg

backend per_ip_and_url_rates
    stick-table type binary len 8 size 1m expire 24h store http_req_rate(24h)

backend per_ip_rates
    stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s)

blog20181127-02.cfg

frontend fe_main
    bind :80

    # track client's source IP in per_ip_rates stick table
    http-request track-sc0 src table per_ip_rates
    
    # track client's source IP + URL accessed in 
    # per_ip_and_url_rates stick table
    http-request track-sc1 url32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif }
    
    # Increment general-purpose counter in per_ip_rates if client
    # is visiting page for the first time
    http-request sc-inc-gpc0(0) if { sc_http_req_rate(1) eq 1 }
    default_backend web_servers

blog20181127-03.cfg

frontend fe_main
    bind :80
    http-request track-sc0 src table per_ip_rates
    http-request track-sc1 url32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif }
    
    # Set the threshold to 15 within the time period
    acl exceeds_limit sc_gpc0_rate(0) gt 15

    # Increase the new-page count if this is the first time 
    # they've accessed this page, unless they've already
    # exceeded the limit
    http-request sc-inc-gpc0(0) if { sc_http_req_rate(1) eq 1 } !exceeds_limit

    # Deny requests if over the limit
    http-request deny if exceeds_limit
        
    default_backend web_servers

blog20181127-04.cfg

backend per_ip_rates
    stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s),gpt0

blog20181127-05.cfg

http-request sc-set-gpt0(0) 1 if exceeds_limit
http-request deny if { sc_get_gpt0(0) eq 1 }

blog20181127-06.cfg

http-request sc-set-gpt0(0) 1 if exceeds_limit
use_backend be_bot_jail if { sc_get_gpt0(0) eq 1 }

blog20181127-07.sh

$ echo "show table per_ip_and_url_rates" | socat stdio /var/run/hapee-1.8/hapee-lb.sock
# table: per_ip_and_url_rates, type: binary, size:1048576, used:2
0x10ab92c: key=203E97AA7F000001000000000000000000000000 use=0 exp=557590 http_req_rate(86400000)=1
0x10afd7c: key=3CBC49B17F000001000000000000000000000000 use=0 exp=596584 http_req_rate(86400000)=5

blog20181127-08.sh

# table: per_ip_rates, type: ip, size:1048576, used:1
0x10ab878: key=127.0.0.1 use=0 exp=594039 gpc0=2 gpc0_rate(30000)=2

blog20181127-09.cfg

http-request use-service lua.request_recaptcha unless { lua.verify_solved_captcha "ok" } { sc_get_gpt0(0) eq 1 }

blog20181127-10.cfg

http-request track-sc1 base32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif }

blog20181127-11.cfg

http-request sc-inc-gpc0(0) if { sc_http_req_rate(1) eq 1 } !exceeds_limit

blog20181127-12.cfg

backend per_ip_and_url_bruteforce
    stick-table type binary len 8 size 1m expire 10m store http_req_rate(3m)

blog20181127-13.cfg

http-request track-sc2 base32+src table per_ip_and_url_bruteforce if METH_POST { path /login }
http-request deny if { sc_http_req_rate(2) gt 10 }

blog20181127-14.cfg

backend per_ip_rates
    stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s),http_err_rate(5m)

blog20181127-15.cfg

http-request deny if { sc_http_err_rate(0) gt 10 }

blog20181127-16.cfg

http-request sc-inc-gpc0(0) if { sc_http_err_rate(0) eq 1 } !exceeds_limit

blog20181127-17.cfg

use_backend be_honeypot if { sc_http_err_rate(0) gt 10 }

blog20181127-18.cfg

http-request track-sc1 url32+src table per_ip_and_url_rates unless { path_end .css .js .png .jpeg .gif } || { src -f /etc/hapee-1.8/whitelist.acl }

blog20181127-19.cfg

unless { src -f /etc/hapee-1.8/whitelist.acl -f /etc/hapee-1.8/admins.acl }

blog20181127-20.cfg

module-load hapee-lb-maxmind.so
maxmind-load COUNTRY /etc/hapee-1.8/geolocation/GeoLite2-Country.mmdb
maxmind-cache-size 10000

blog20181127-21.cfg

http-request set-header x-geoip-country %[src,maxmind-lookup(COUNTRY,country,iso_code)]

blog20181127-22.cfg

x-geoip-country: US

blog20181127-23.cfg

module-load hapee-lb-netacuity.so
netacuity-load 26 /etc/hapee-1.8/geolocation/netacuity/
netacuity-cache-size 10000

blog20181127-24.cfg

http-request set-header x-geoip-country %[src,netacuity-lookup-ipv4(“pulse-two-letter-country”)]

blog20181127-25.cfg

x-geoip-country: US

blog20181127-26.py

import sys
ip_blocks_file = sys.argv[1]
city_locations_file = sys.argv[2]
#First load the city locations into memory, as we will be using them a lot
city_locations = {}
city_locations_handle = open(city_locations_file,'r')
for city_location_line in city_locations_handle.readlines():
    city_location_parts = city_location_line.split(",")
    if len(city_location_parts) < 13:
        continue
    if not city_location_parts[0].isdigit():
        continue
    location_id=city_location_parts[0]
    locale_code=city_location_parts[1]
    continent_code=city_location_parts[2]
    continent_name=city_location_parts[3]
    country_iso_code=city_location_parts[4]
    country_name=city_location_parts[5]
    subdivision_1_iso_code=city_location_parts[6]
    subdivision_1_name=city_location_parts[7]
    subdivision_2_iso_code=city_location_parts[8]
    subdivision_2_name=city_location_parts[9]
    city_name=city_location_parts[10]
    metro_code=city_location_parts[11]
    time_zone=city_location_parts[12]
    #print "Found country code '" + str(country_iso_code) + "' for id '" + str(location_id) + "'"
    city_locations[location_id] = [country_iso_code, city_name]
#print "Country code for 10471023: " + str(city_locations[str(10471023)][0])
#Next build the country_iso_code and city_name files with this data
#Open map file handles
country_iso_code_file= open('country_iso_code.map', 'w')
city_name_file = open('city_name.map', 'w')
gps_map_file = open('gps.map', 'w')
#Process the lines of the ip block file
ip_blocks_handle = open(ip_blocks_file,'r')
for ip_block_line in ip_blocks_handle.readlines():
    ip_block_line_parts = ip_block_line.split(',')
    if len(ip_block_line_parts) < 9:
        continue
    network=ip_block_line_parts[0]
    geoname_id=ip_block_line_parts[1]
    #Per docs "registered" is where the IP is registered, rather then used
    registered_country_geoname_id=ip_block_line_parts[2]
    #"represented" only applies to military bases/etc and is their country
    represented_country_geoname_id=ip_block_line_parts[3]
    is_anonymous_proxy=ip_block_line_parts[4]
    is_satellite_provider=ip_block_line_parts[5]
    postal_code=ip_block_line_parts[6]
    latitude=ip_block_line_parts[7]
    longitude=ip_block_line_parts[8].rstrip() #Last column gets a newline appended to it
    if not geoname_id in city_locations:
        continue
    #Write the country map line
    country_iso_code_file.write(network + ' ' + city_locations[geoname_id][0] + '\n')
    #Write the city map line
    city_name_file.write(network + ' ' + city_locations[geoname_id][1].strip('"') + '\n')
    #Write the GPS map line
    gps_map_file.write(network + ' ' + longitude + ", " + latitude + '\n')
country_iso_code_file.close()
city_name_file.close()
gps_map_file.close()

blog20181127-27.sh

python read_city_map.py GeoLite2-City-CSV_20181127/GeoLite2-City-Blocks-IPv4.csv GeoLite2-City-CSV_20181127/GeoLite2-City-Locations-en.csv

blog20181127-28.cfg

http-request set-header x-geoip-country %[src,map(/etc/hapee-1.8/country_iso_code.map)]

blog20181127-29.cfg

x-geoip-country: US

blog20181127-30.cfg

use_backend be_honeypot if { sc_http_err_rate(0) gt 5 } { req.hdr(x-geoip-country) CN }