Skip to content

Instantly share code, notes, and snippets.

@wvengen
Last active May 31, 2019 07:27
Show Gist options
  • Save wvengen/ec5c795b0f2960d7ea07 to your computer and use it in GitHub Desktop.
Save wvengen/ec5c795b0f2960d7ea07 to your computer and use it in GitHub Desktop.
Installation script for scrapyd
#
# Installation script for scrapyd on Debian/Ubuntu
# http://scrapyd.readthedocs.org/
#
# - Latest scrapyd from Scrapy package repository
# - Password protection with http basic auth
# - HTTPS with self-signed certificate
# - Works on Amazon EC2
#
FQDN=scrapy.example.com
PASSWORD=secretpass
# Install latest scrapyd
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
echo 'deb http://archive.scrapy.org/ubuntu scrapy main' >/etc/apt/sources.list.d/scrapy.list
apt-get update
apt-get install -y scrapyd
# Configure scrapyd
cat >/etc/scrapyd/conf.d/local <<EOF
[scrapyd]
bind_address = 127.0.0.1
EOF
# Create self-signed certificate for https
SSL=/etc/ssl/private
openssl req -nodes -newkey rsa:4096 -keyout $SSL/cert.key -out $SSL/cert.csr -subj "/CN=$FQDN"
openssl x509 -req -days 3650 -in $SSL/cert.csr -signkey $SSL/cert.key -out $SSL/cert.crt
# Setup webserver front-end
apt-get install -y nginx-light apache2-utils
cat >/etc/nginx/sites-enabled/default <<EOF
server {
listen 6843 default_server;
ssl on;
ssl_certificate /etc/ssl/private/cert.crt;
ssl_certificate_key /etc/ssl/private/cert.key;
ssl_session_timeout 5m;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
ssl_prefer_server_ciphers on;
location / {
proxy_pass http://localhost:6800/;
proxy_redirect off;
proxy_set_header Authorization "";
auth_basic "Restricted";
auth_basic_user_file /etc/nginx/htpasswd;
}
}
EOF
htpasswd -b -c /etc/nginx/htpasswd deploy "$PASSWORD"
# Restart and get ready
/etc/init.d/scrapyd restart
/etc/init.d/nginx restart
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment