Skip to content

Instantly share code, notes, and snippets.

@sbeam
Forked from anonymous/test_whitelist.rb
Last active August 29, 2015 14:07
Show Gist options
  • Save sbeam/c12eba45df78fef2cc95 to your computer and use it in GitHub Desktop.
Save sbeam/c12eba45df78fef2cc95 to your computer and use it in GitHub Desktop.
# this tests a critical regex used in the 'Whitelist Query Params' config in
# Fastly, to transform request URLs into a normalized version that strips
# 'traffic partner' tracking params and other noise, but leaves any valid keys
# that match the params we actually use. A lot hinges on the proper operation
# of this regex to keep hit ratio high but leave needed params intact.
#
# This version leaves trailing ? and &, which is unavoidable now but are pulled
# out by a simple second sweep.
#
# http://regex101.com/r/oJ2gI6/10
# map of original urls to transformed version. If value is nil it should be unchanged.
URLS = {
'http://ovens.reviewed.com/?utm_source=VSW&utm_medium=PPC' =>
'http://ovens.reviewed.com/',
'/blah/foo?1234-1234_6&brand=foo&utm_source=1234&type=camera&page=5&utm_medium=blah&foo&1234-1234_0' =>
'/blah/foo?brand=foo&type=camera&page=5',
'/blah/foo?nonsense=42&fooparam' =>
'/blah/foo',
'/blah/foo?&2938120-12923_3&utm_source=GatesOfHell&1=2' =>
'/blah/foo',
'/blah/foo?tags=Ice+Dispenser&page=42&fooparam=19&fifi' =>
'/blah/foo?tags=Ice+Dispenser&page=42',
'http://staging.espressomakerinfo.com/how_we_test?&11163888-449553_6&brand=123&n23=12&xxxxxxxxxxxxxxx&page=23' =>
'http://staging.espressomakerinfo.com/how_we_test?&brand=123&page=23',
'http://dishwashers.reviewed.com/features/goodbye-granite-the-6-hottest-countertop-finishes?utm_source=taboola&utm_medium=USAT%20Recirc&' =>
'http://dishwashers.reviewed.com/features/goodbye-granite-the-6-hottest-countertop-finishes',
'http://refrigerators.reviewed.com/features/seriously-stop-refrigerating-these-foods?utm_source=TB_paid&utm_medium=cpc&' =>
'http://refrigerators.reviewed.com/features/seriously-stop-refrigerating-these-foods',
'http://ovens.reviewed.com/news/with-one-chip-ge-creates-diy-smart-appliances?utm_source=usat&utm_medium=referral&utm_campaign=collab&' =>
'http://ovens.reviewed.com/news/with-one-chip-ge-creates-diy-smart-appliances',
'/content/lg-65ec9700-4k-oled-tv-first-impressions-review?utm_medium=referral&utm_source=feedblitz&utm_campaign=usatoday-techtopstories' =>
'/content/lg-65ec9700-4k-oled-tv-first-impressions-review',
'http://televisions.reviewed.com/features/the-ultimate-movie-player-starts-at-4000?utm_medium=referral&utm_source=dlvr.it.test&dlvrit=384245' =>
'http://televisions.reviewed.com/features/the-ultimate-movie-player-starts-at-4000',
'http://laundry.reviewed.com/?utm_source=Adwords&utm_medium=PPC-intoff&gclid=CjwKEAjwkf-gBRCd-b2m2aOo0EQSJABMeQDkU6Wmd-QCSogFgFbcYZzVrk9c99AtrRdj43rZzJ67pRoCtqnw_wcB' =>
'http://laundry.reviewed.com/',
'/features/bone-skulls-and-harmony-check-out-these-french-speakers?amp;utm_medium=referral&utm_campaign=collab' =>
'/features/bone-skulls-and-harmony-check-out-these-french-speakers',
'/features/bone-skulls-and-harmony-check-out-these-french-speakers?amp;utm_medium' =>
'/features/bone-skulls-and-harmony-check-out-these-french-speakers',
'/seriously-stop-refrigerating-these-foods?utm_source=TB_paid&utm_medium=cpc&' =>
'/seriously-stop-refrigerating-these-foods',
'http://laundry.reviewed.com/types/front-loading?page=1&_escaped_fragment_=' => nil,
'/ads/interstitial_1?adid=stv_01' => nil,
'http://www.reviewed.com/' => nil,
'http://vacuums.reviewed.com//d/reviews/hoover-duros.htm?order_by=msrp%252Cdesc' => nil,
'/ratings.htm?keywords=Daewoo+Steam+4.5CuFt+Washer%2C+7.3CuFt+GAS+Dryer+1%2C300+Spin+Speed%2C+Smart+Detergent+System&order_by=rating%2Cdesc' => nil,
'/something.com/articles/12875/attachments.json?tags=Front+Photo%2CHandle+Photo%2CFingerprints+Photo%2CWater%2FIce+Dispenser+Photo%2CControls+Photo%2CWater%2FIce+Dispenser+Controls+Photo%2CInterior+Photo%2CRefrigerator+Main+1+Image%2CRefrigerator+Main+2+Image%2CRefrigerator+Main+3+Image%2CVegetable+Drawer+Photo%2CRefrigerator+Door+1-1+Image%2CIce+Maker+Photo%2CWater+Filter+Photo%2CRefrigerator+Door+2-1+Image%2CFreezer+Main+1+Image%2CFreezer+Door+1+Image%2CSides+Photo%2CBack+Photo&page=1&per_page=6' => nil,
'/brands/hifiman?brand=hifiman' => nil,
}
tests, expectations = ["",""]
URLS.each do |source, res|
tests << "\ntxreq -url \"#{source}\""
expectations << "\nrxreq\nexpect req.url == \"#{res || source}\"\n"
end
vcltest = <<EOt
varnishtest "#1030"
server s1 {
#{expectations}
} -start
varnish v1 -vcl+backend {
sub vcl_recv {
if( req.url ) {
set req.url = regsuball(req.url, "(^|&|\\?)(?!_escaped_fragment_|brand|branch|type|min_msrp|max_msrp|_type|keywords|page|per_page|tags|name|order_by|adid|return_url|event|classifications|zip|website_ids)([^&\\/=]+)(=[^&]+)?(?:(&|$))", "&");
set req.url = regsuball(req.url, "&{2,}", "");
if (req.url ~ "&" && !(req.url ~ "\\?")) {
set req.url = regsub(req.url, "&", "?");
}
}
}
} -start
client c1 {
#{tests}
} -run
EOt
filename = 'test-whitelist.vcl'
File.write("/tmp/#{filename}", vcltest)
system("sudo docker run --rm -v /tmp/#{filename}:/opt/#{filename}:ro zenedith/varnish /bin/sh -c 'ldconfig; varnishtest /opt/#{filename}'")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment