Skip to content

Instantly share code, notes, and snippets.

@edrohler

edrohler/web.BADBOT.config

Last active July 17, 2019 14:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save edrohler/5138433e9fd8fe9358df to your computer and use it in GitHub Desktop.
Save edrohler/5138433e9fd8fe9358df to your computer and use it in GitHub Desktop.
Download ZIP
Web config section for bad bot blocking in asp.net
Raw
web.BADBOT.config
<system.webServer>
<staticContent>
<clientCache httpExpires="Thur, 31 Mar 2020 00:00:00 GMT" cacheControlMode="UseExpires" />
</staticContent>
<httpProtocol>
<customHeaders>
<!-- Remove Excessive Header -->
<remove name="X-Powered-By"/>
<!-- Removes Azure non-HttpOnly Cookie -->
<add name="Arr-Disable-Session-Affinity" value="True" />
<!-- Deny X-Frame Options -->
<add name="X-Frame-Options" value="DENY"/>
</customHeaders>
</httpProtocol>
<rewrite>
<rules>
<!-- Bad Bot Blocking Rules -->
<rule name="BadBotBlocking" stopProcessing="true">
<match url="^.*" ignoreCase="false" />
<conditions logicalGrouping="MatchAny">
<add input="{HTTP_USER_AGENT}" pattern="^BlackWidow" />
<add input="{HTTP_USER_AGENT}" pattern="^Bolt\ 0" />
<add input="{HTTP_USER_AGENT}" pattern="^Bot\ mailto:craftbot\@yahoo\.com" />
<add input="{HTTP_USER_AGENT}" pattern="CazoodleBot" />
<add input="{HTTP_USER_AGENT}" pattern="^ChinaClaw" />
<add input="{HTTP_USER_AGENT}" pattern="^Custo" />
<add input="{HTTP_USER_AGENT}" pattern="^Default\ Browser\ 0" />
<add input="{HTTP_USER_AGENT}" pattern="^DIIbot" />
<add input="{HTTP_USER_AGENT}" pattern="^DISCo" />
<add input="{HTTP_USER_AGENT}" pattern="discobot" />
<add input="{HTTP_USER_AGENT}" pattern="^Download\ Demon" />
<add input="{HTTP_USER_AGENT}" pattern="^eCatch" />
<add input="{HTTP_USER_AGENT}" pattern="ecxi" />
<add input="{HTTP_USER_AGENT}" pattern="^EirGrabber" />
<add input="{HTTP_USER_AGENT}" pattern="^EmailCollector" />
<add input="{HTTP_USER_AGENT}" pattern="^EmailSiphon" />
<add input="{HTTP_USER_AGENT}" pattern="^EmailWolf" />
<add input="{HTTP_USER_AGENT}" pattern="^Express\ WebPictures" />
<add input="{HTTP_USER_AGENT}" pattern="^ExtractorPro" />
<add input="{HTTP_USER_AGENT}" pattern="^EyeNetIE" />
<add input="{HTTP_USER_AGENT}" pattern="^FlashGet" />
<add input="{HTTP_USER_AGENT}" pattern="^GetRight" />
<add input="{HTTP_USER_AGENT}" pattern="^GetWeb!" />
<add input="{HTTP_USER_AGENT}" pattern="^Go!Zilla" />
<add input="{HTTP_USER_AGENT}" pattern="^Go-Ahead-Got-It" />
<add input="{HTTP_USER_AGENT}" pattern="^GrabNet" />
<add input="{HTTP_USER_AGENT}" pattern="^Grafula" />
<add input="{HTTP_USER_AGENT}" pattern="GT::WWW" />
<add input="{HTTP_USER_AGENT}" pattern="heritrix" />
<add input="{HTTP_USER_AGENT}" pattern="^HMView" />
<add input="{HTTP_USER_AGENT}" pattern="HTTP::Lite" />
<add input="{HTTP_USER_AGENT}" pattern="HTTrack" />
<add input="{HTTP_USER_AGENT}" pattern="ia_archiver" />
<add input="{HTTP_USER_AGENT}" pattern="IDBot" />
<add input="{HTTP_USER_AGENT}" pattern="id-search" />
<add input="{HTTP_USER_AGENT}" pattern="id-search\.org" />
<add input="{HTTP_USER_AGENT}" pattern="^Image\ Stripper" />
<add input="{HTTP_USER_AGENT}" pattern="^Image\ Sucker" />
<add input="{HTTP_USER_AGENT}" pattern="Indy\ Library" />
<add input="{HTTP_USER_AGENT}" pattern="^InterGET" />
<add input="{HTTP_USER_AGENT}" pattern="^Internet\ Ninja" />
<add input="{HTTP_USER_AGENT}" pattern="^InternetSeer\.com" />
<add input="{HTTP_USER_AGENT}" pattern="IRLbot" />
<add input="{HTTP_USER_AGENT}" pattern="ISC\ Systems\ iRc\ Search\ 2\.1" />
<add input="{HTTP_USER_AGENT}" pattern="^Java" />
<add input="{HTTP_USER_AGENT}" pattern="^JetCar" />
<add input="{HTTP_USER_AGENT}" pattern="^JOC\ Web\ Spider" />
<add input="{HTTP_USER_AGENT}" pattern="^larbin" />
<add input="{HTTP_USER_AGENT}" pattern="^LeechFTP" />
<add input="{HTTP_USER_AGENT}" pattern="libwww" />
<add input="{HTTP_USER_AGENT}" pattern="libwww-perl" />
<add input="{HTTP_USER_AGENT}" pattern="^Link" />
<add input="{HTTP_USER_AGENT}" pattern="LinksManager.com_bot" />
<add input="{HTTP_USER_AGENT}" pattern="linkwalker" />
<add input="{HTTP_USER_AGENT}" pattern="lwp-trivial" />
<add input="{HTTP_USER_AGENT}" pattern="^Mass\ Downloader" />
<add input="{HTTP_USER_AGENT}" pattern="^Maxthon$" />
<add input="{HTTP_USER_AGENT}" pattern="MFC_Tear_Sample" />
<add input="{HTTP_USER_AGENT}" pattern="^microsoft\.url" />
<add input="{HTTP_USER_AGENT}" pattern="Microsoft\ URL\ Control" />
<add input="{HTTP_USER_AGENT}" pattern="^MIDown\ tool" />
<add input="{HTTP_USER_AGENT}" pattern="^Mister\ PiX" />
<add input="{HTTP_USER_AGENT}" pattern="Missigua\ Locator" />
<add input="{HTTP_USER_AGENT}" pattern="^Mozilla\.*Indy" />
<add input="{HTTP_USER_AGENT}" pattern="^Mozilla\.*NEWT" />
<add input="{HTTP_USER_AGENT}" pattern="^MSFrontPage" />
<add input="{HTTP_USER_AGENT}" pattern="^Navroad" />
<add input="{HTTP_USER_AGENT}" pattern="^NearSite" />
<add input="{HTTP_USER_AGENT}" pattern="^NetAnts" />
<add input="{HTTP_USER_AGENT}" pattern="^NetSpider" />
<add input="{HTTP_USER_AGENT}" pattern="^Net\ Vampire" />
<add input="{HTTP_USER_AGENT}" pattern="^NetZIP" />
<add input="{HTTP_USER_AGENT}" pattern="^Nutch" />
<add input="{HTTP_USER_AGENT}" pattern="^Octopus" />
<add input="{HTTP_USER_AGENT}" pattern="^Offline\ Explorer" />
<add input="{HTTP_USER_AGENT}" pattern="^Offline\ Navigator" />
<add input="{HTTP_USER_AGENT}" pattern="^PageGrabber" />
<add input="{HTTP_USER_AGENT}" pattern="panscient.com" />
<add input="{HTTP_USER_AGENT}" pattern="^Papa\ Foto" />
<add input="{HTTP_USER_AGENT}" pattern="^pavuk" />
<add input="{HTTP_USER_AGENT}" pattern="PECL::HTTP" />
<add input="{HTTP_USER_AGENT}" pattern="^PeoplePal" />
<add input="{HTTP_USER_AGENT}" pattern="^pcBrowser" />
<add input="{HTTP_USER_AGENT}" pattern="PHPCrawl" />
<add input="{HTTP_USER_AGENT}" pattern="PleaseCrawl" />
<add input="{HTTP_USER_AGENT}" pattern="^psbot" />
<add input="{HTTP_USER_AGENT}" pattern="^RealDownload" />
<add input="{HTTP_USER_AGENT}" pattern="^ReGet" />
<add input="{HTTP_USER_AGENT}" pattern="^Rippers\ 0" />
<add input="{HTTP_USER_AGENT}" pattern="SBIder" />
<add input="{HTTP_USER_AGENT}" pattern="^SeaMonkey$" />
<add input="{HTTP_USER_AGENT}" pattern="^sitecheck\.internetseer\.com" />
<add input="{HTTP_USER_AGENT}" pattern="^SiteSnagger" />
<add input="{HTTP_USER_AGENT}" pattern="^SmartDownload" />
<add input="{HTTP_USER_AGENT}" pattern="Snoopy" />
<add input="{HTTP_USER_AGENT}" pattern="Steeler" />
<add input="{HTTP_USER_AGENT}" pattern="^SuperBot" />
<add input="{HTTP_USER_AGENT}" pattern="^SuperHTTP" />
<add input="{HTTP_USER_AGENT}" pattern="^Surfbot" />
<add input="{HTTP_USER_AGENT}" pattern="^tAkeOut" />
<add input="{HTTP_USER_AGENT}" pattern="^Teleport\ Pro" />
<add input="{HTTP_USER_AGENT}" pattern="^Toata\ dragostea\ mea\ pentru\ diavola" />
<add input="{HTTP_USER_AGENT}" pattern="URI::Fetch" />
<add input="{HTTP_USER_AGENT}" pattern="urllib" />
<add input="{HTTP_USER_AGENT}" pattern="User-Agent" />
<add input="{HTTP_USER_AGENT}" pattern="^VoidEYE" />
<add input="{HTTP_USER_AGENT}" pattern="^Web\ Image\ Collector" />
<add input="{HTTP_USER_AGENT}" pattern="^Web\ Sucker" />
<add input="{HTTP_USER_AGENT}" pattern="Web\ Sucker" />
<add input="{HTTP_USER_AGENT}" pattern="webalta" />
<add input="{HTTP_USER_AGENT}" pattern="^WebAuto" />
<add input="{HTTP_USER_AGENT}" pattern="^[Ww]eb[Bb]andit" />
<add input="{HTTP_USER_AGENT}" pattern="WebCollage" />
<add input="{HTTP_USER_AGENT}" pattern="^WebCopier" />
<add input="{HTTP_USER_AGENT}" pattern="^WebFetch" />
<add input="{HTTP_USER_AGENT}" pattern="^WebGo\ IS" />
<add input="{HTTP_USER_AGENT}" pattern="^WebLeacher" />
<add input="{HTTP_USER_AGENT}" pattern="^WebReaper" />
<add input="{HTTP_USER_AGENT}" pattern="^WebSauger" />
<add input="{HTTP_USER_AGENT}" pattern="^Website\ eXtractor" />
<add input="{HTTP_USER_AGENT}" pattern="^Website\ Quester" />
<add input="{HTTP_USER_AGENT}" pattern="^WebStripper" />
<add input="{HTTP_USER_AGENT}" pattern="^WebWhacker" />
<add input="{HTTP_USER_AGENT}" pattern="^WebZIP" />
<add input="{HTTP_USER_AGENT}" pattern="Wells\ Search\ II" />
<add input="{HTTP_USER_AGENT}" pattern="WEP\ Search" />
<add input="{HTTP_USER_AGENT}" pattern="^Wget" />
<add input="{HTTP_USER_AGENT}" pattern="^Widow" />
<add input="{HTTP_USER_AGENT}" pattern="^WWW-Mechanize" />
<add input="{HTTP_USER_AGENT}" pattern="^WWWOFFLE" />
<add input="{HTTP_USER_AGENT}" pattern="^Xaldon\ WebSpider" />
<add input="{HTTP_USER_AGENT}" pattern="zermelo" />
<add input="{HTTP_USER_AGENT}" pattern="^Zeus" />
<add input="{HTTP_USER_AGENT}" pattern="^Zeus\.*Webster" />
<add input="{HTTP_USER_AGENT}" pattern="ZyBorg" />
</conditions>
<action type="CustomResponse" statusCode="403" statusReason="Forbidden" statusDescription="Forbidden"/>
</rule>
</rules>
</rewrite>
</system.webServer>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment