Skip to content

Instantly share code, notes, and snippets.

@mascot6699
Created April 26, 2016 11:36
Show Gist options
  • Save mascot6699/f6bb18600bead31dbd39ea29e86e9f5e to your computer and use it in GitHub Desktop.
Save mascot6699/f6bb18600bead31dbd39ea29e86e9f5e to your computer and use it in GitHub Desktop.
map $http_user_agent $limit_bots {
default 0;
~*(google|bing|yandex|msnbot) 1;
~*(AltaVista|Googlebot|Slurp|BlackWidow|Bot|ChinaClaw|Custo|DISCo|Download|Demon|eCatch|EirGrabber|EmailSiphon|EmailWolf|SuperHTTP|Surfbot|WebWhacker) 1;
~*(Express|WebPictures|ExtractorPro|EyeNetIE|FlashGet|GetRight|GetWeb!|Go!Zilla|Go-Ahead-Got-It|GrabNet|Grafula|HMView|Go!Zilla|Go-Ahead-Got-It) 1;
~*(rafula|HMView|HTTrack|Stripper|Sucker|Indy|InterGET|Ninja|JetCar|Spider|larbin|LeechFTP|Downloader|tool|Navroad|NearSite|NetAnts|tAkeOut|WWWOFFLE) 1;
~*(GrabNet|NetSpider|Vampire|NetZIP|Octopus|Offline|PageGrabber|Foto|pavuk|pcBrowser|RealDownload|ReGet|SiteSnagger|SmartDownload|SuperBot|WebSpider) 1;
~*(Teleport|VoidEYE|Collector|WebAuto|WebCopier|WebFetch|WebGo|WebLeacher|WebReaper|WebSauger|eXtractor|Quester|WebStripper|WebZIP|Wget|Widow|Zeus) 1;
~*(Twengabot|htmlparser|libwww|Python|perl|urllib|scan|email|PycURL|Pyth|PyQ|WebCollector|WebCopy|webcraw) 1;
}
server {
listen 80;
listen 443;
server_name domain.name server_url;
ssl on;
ssl_certificate /var/www/devly.co/cert/ssl-bundle.crt;
ssl_certificate_key /var/www/devly.co/cert/devly_co.key;
root /path/to/project/root/;
# force https-redirects
if ($scheme = http) {
return 301 https://$server_name$request_uri;
}
# set client body size to 10M #
client_max_body_size 10M;
location = /favicon.ico { access_log off; log_not_found off; }
location /static/ {
root /path/to/static/folder/;
}
location / {
#if ($http_user_agent ~ (libwww|Wget|LWP|damnBot|BBBike|java|spider|crawl) ) {
# return 403;
#}
# if ($limit_bots = 1) {
# return 403;
#}
include proxy_params;
proxy_pass http://localhost:8000;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment