Nginx 把所有爬蟲都擋下來，在 nginx server block 中放置

Nginx check useragent 阻擋爬蟲機器人.sh

set $limit_bots 0;

if ($http_user_agent ~ ^$|FCKFK|no-store|WordPress\/)
{
    set $limit_bots 1;
}


if ($http_accept ~ ms-powerpoint)
{
    set $limit_bots 1;
}

if ($http_user_agent ~ ^Mozilla...0..compatible..MSIE...0..Windows.NT...1..Trident...0.$|^Mozilla...0..compatible..MSIE...0..Windows.NT...1.$|^Mozilla...0..compatible..MSIE...0..Windows.NT........$)
{
    set $limit_bots 1;
}

if ($http_user_agent ~* "python|curl|java|wget|httpclient|okhttp") {
    set $limit_bots 1;
}

if ($http_user_agent ~* "google|bing|yandex|msnbot|AltaVista|Googlebot|Slurp|BlackWidow|Bot|ChinaClaw|Custo|DISCo|Download|Demon|eCatch|EirGrabber|EmailSiphon|EmailWolf|SuperHTTP|Surfbot|WebWhacker|Express|WebPictures|ExtractorPro|EyeNetIE|FlashGet|GetRight|GetWeb!|Go!Zilla|Go-Ahead-Got-It|GrabNet|Grafula|HMView|Go!Zilla|Go-Ahead-Got-It|rafula|HMView|HTTrack|Stripper|Sucker|Indy|InterGET|Ninja|JetCar|Spider|larbin|LeechFTP|Downloader|tool|Navroad|NearSite|NetAnts|tAkeOut|WWWOFFLE|GrabNet|NetSpider|Vampire|NetZIP|Octopus|Offline|PageGrabber|Foto|pavuk|pcBrowser|RealDownload|ReGet|SiteSnagger|SmartDownload|SuperBot|WebSpider|Teleport|VoidEYE|Collector|WebAuto|WebCopier|WebFetch|WebGo|WebLeacher|WebReaper|WebSauger|eXtractor|Quester|WebStripper|WebZIP|Wget|Widow|Zeus|Twengabot|htmlparser|libwww|Python|perl|urllib|scan|Curl|email|PycURL|Pyth|PyQ|WebCollector|WebCopy|webcraw") 
{ 
    set $limit_bots 1;
} 

if ($limit_bots = 1) {
  return 403;
}