Skip to content

Instantly share code, notes, and snippets.

@ilokano
Last active December 15, 2016 17:43
Show Gist options
  • Save ilokano/d26ecc2e99c4ebe7f0b3 to your computer and use it in GitHub Desktop.
Save ilokano/d26ecc2e99c4ebe7f0b3 to your computer and use it in GitHub Desktop.
I place this inside /etc/nginx/conf.d
map $http_user_agent $bad_agent {
default 0;
"~*^[Ww]eb[Bb]andit" 1;
"~*^Acunetix" 1;
"~*^binlar" 1;
"~*^BlackWidow" 1;
"~^Bolt" 1;
"~*^casper" 1;
"~*^checkprivacy" 1;
"~*^ChinaClaw" 1;
"~*^clshttp" 1;
"~*^cmsworldmap" 1;
"~*^comodo" 1;
"~*^Custo" 1;
"~^Default" 1;
"~*^diavol" 1;
"~*^DIIbot" 1;
"~*^DISCo" 1;
"~*^dotbot" 1;
"~*^eCatch" 1;
"~*^EirGrabber" 1;
"~*^EmailCollector" 1;
"~*^EmailSiphon" 1;
"~*^EmailWolf" 1;
"~*^extract" 1;
"~*^ExtractorPro" 1;
"~*^EyeNetIE" 1;
"~*^feedfinder" 1;
"~*^FHscan" 1;
"~*^FlashGet" 1;
"~*^flicky" 1;
"~*^GetRight" 1;
"~*^GetWeb!" 1;
"~*^Go-Ahead-Got-It" 1;
"~*^g00g1e" 1;
"~*^Go!Zilla" 1;
"~*^grab" 1;
"~*^GrabNet" 1;
"~*^Grafula" 1;
"~*^harvest" 1;
"~*^HMView" 1;
"~*^ia_archiver" 1;
"~*^InterGET" 1;
"~*^InternetSeer.com" 1;
"~*^jakarta" 1;
"~*^Java" 1;
"~*^JetCar" 1;
"~*^kanagawa" 1;
"~*^kmccrew" 1;
"~*^larbin" 1;
"~*^LeechFTP" 1;
"~*^libwww" 1;
"~*^Maxthon$" 1;
"~*^microsoft.url" 1;
"~*^miner" 1;
"~*^Mozilla.*Indy" 1;
"~*^Mozilla.*NEWT" 1;
"~*^MSFrontPage" 1;
"~*^Navroad" 1;
"~*^NearSite" 1;
"~*^NetAnts" 1;
"~*^NetSpider" 1;
"~*^NetZIP" 1;
"~*^nutch" 1;
"~*^Octopus" 1;
"~*^PageGrabber" 1;
"~*^pavuk" 1;
"~*^pcBrowser" 1;
"~*^PeoplePal" 1;
"~*^planetwork" 1;
"~*^psbot" 1;
"~*^purebot" 1;
"~*^pycurl" 1;
"~*^RealDownload" 1;
"~*^ReGet" 1;
"~^Rippers" 1;
"~*^SeaMonkey$" 1;
"~*^sitecheck.internetseer.com" 1;
"~*^SiteSnagger" 1;
"~*^skygrid" 1;
"~*^SmartDownload" 1;
"~*^sucker" 1;
"~*^SuperBot" 1;
"~*^SuperHTTP" 1;
"~*^Surfbot" 1;
"~*^tAkeOut" 1;
"~^Teleport" 1;
"~^Toata" 1;
"~*^turnit" 1;
"~*^vikspider" 1;
"~*^VoidEYE" 1;
"~*^WebAuto" 1;
"~*^WebCopier" 1;
"~*^WebFetch" 1;
"~*^WebLeacher" 1;
"~*^WebReaper" 1;
"~*^WebSauger" 1;
"~*^WPScan" 1;
"~*^WebStripper" 1;
"~*^WebWhacker" 1;
"~*^WebZIP" 1;
"~*^Wget" 1;
"~*^Widow" 1;
"~*^WWW-Mechanize" 1;
"~*^WWWOFFLE" 1;
"~*^Yandex" 1;
"~*^Zeus" 1;
"~*^zmeu" 1;
"~*AhrefsBot" 1;
"~*CazoodleBot" 1;
"~*discobot" 1;
"~*ecxi" 1;
"~*GT::WWW" 1;
"~*heritrix" 1;
"~*HTTP::Lite" 1;
"~*HTTrack" 1;
"~*ia_archiver" 1;
"~*id-search" 1;
"~*id-search.org" 1;
"~*IDBot" 1;
"~*IRLbot" 1;
"~*LinksManager.com_bot" 1;
"~*linkwalker" 1;
"~*lwp-trivial" 1;
"~*MFC_Tear_Sample" 1;
"~Microsoft" 1;
"~*MJ12bot" 1;
"~*panscient.com" 1;
"~*PECL::HTTP" 1;
"~*PHPCrawl" 1;
"~*PleaseCrawl" 1;
"~*SBIder" 1;
"~*Snoopy" 1;
"~*Steeler" 1;
"~*URI::Fetch" 1;
"~*urllib" 1;
"~*webalta" 1;
"~*WebCollage" 1;
"~*zermelo" 1;
"~*ZyBorg" 1;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment