Shellbye

#!/bin/sh

sudo apt-get update
sudo apt-get install -y squid3
sudo cp /etc/squid3/squid.conf /etc/squid3/squid.conf.original
sudo chmod a-w /etc/squid3/squid.conf.original
echo "use vim /etc/squid3/squid.conf to change port(default 3128) and http_access allow"
# change port: http_port 8888
# allow access: http_access allow all <==this should at top of all http_access
# ref: https://help.ubuntu.com/lts/serverguide/squid.html

Shellbye

# ref http://stackoverflow.com/questions/3297196/how-to-set-up-a-squid-proxy-with-basic-username-and-password-authentication
# first install apache:
# $ sudo apt-get install -y apache2-utils
# add these five lines to /etc/squid3/squid.conf
auth_param basic program /usr/lib/squid3/basic_ncsa_auth /etc/squid3/passwords
auth_param basic realm proxy
acl authenticated proxy_auth REQUIRED
http_access allow authenticated
http_port 3128 # or whatever you like

Shellbye

#!/usr/bin/env bash

mkdir spider
cd spider/
yum update -y
# install pip on centos6
rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
yum install -y python-pip
# install wget
yum install -y wget

Shellbye

# https://forum.directadmin.com/showthread.php?t=15878
cat /etc/*release*
cat /etc/centos-release # http://www.liquidweb.com/kb/how-to-check-your-centos-version/
cat /etc/redhat-release


# https://linuxconfig.org/how-to-check-centos-version
# the later two may need some package to install
rpm --query centos-release

Shellbye

wget https://pypi.python.org/packages/source/T/Twisted/Twisted-16.1.1.tar.bz2
tar -jvxf Twisted-16.1.1.tar.bz2
cd Twisted-16.1.1
python setup.py install

Shellbye

# http://www.liquidweb.com/kb/how-to-install-pip-on-centos-7/
curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
sudo python get-pip.py

Shellbye

#!/usr/bin/env bash

apt-get update
apt-get install -y python-pip
pip install django==1.8.6 # or use install_django_without_pip.sh

Shellbye

#!/usr/bin/env bash

wget https://www.djangoproject.com/m/releases/1.8/Django-1.8.6.tar.gz
tar -xvzf Django-1.8.6.tar.gz
cd Django-1.8.6
python setup.py install

Shellbye

#!/usr/bin/env bash

wget https://pypi.python.org/packages/e2/af/0a3981fffc5cd43078eb8b1057702e0dd2d5771e5aaa36cbd140e32f8473/Pillow-3.2.0.tar.gz#md5=7cfd093c11205d9e2ebe3c51dfcad510
tar -xzvf Pillow-3.2.0.tar.gz
cd Pillow-3.2.0

apt-get update
apt-get install -y libjpeg-dev # ValueError: jpeg is required unless explicitly disabled using --disable-jpeg, aborting
apt-get install -y zlib1g-dev # ValueError: zlib is required unless explicitly disabled using --disable-zlib, aborting
apt-get install -y python-dev # error: command 'x86_64-linux-gnu-gcc' failed with exit status 1

Shellbye

#!/usr/bin/env bash

# install nginx & uwsgi
sudo apt-get install nginx
pip install uwsgi

# connect ngxin to django site
ln -s /path/to/your/mysite/mysite_nginx.conf /etc/nginx/sites-enabled/

# restart or start whatever needed