pyspider.sh

#!/bin/bash
apt-get -y remove apache2
apt-get -y remove apache2.2-common
apt-get -y autoremove apache2

apt-get update -y
apt-get install -y curl wget tmux unzip python python-dev python-distribute python-pip libcurl4-openssl-dev libxml2-dev libxslt1-dev python-lxml git libssl-dev redis-server libpcre3 libpcre3-dev zlib1g-dev build-essential tcl8.5

#安装pre库
wget ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.37.tar.gz
tar -zxvf pcre-8.37.tar.gz
cd pcre-8.37
./configure
make
make install
cd /root
#wget http://labs.frickle.com/files/ngx_cache_purge-1.5.tar.gz
#安装nginx
apt-get install -y libpcre3 libpcre3-dev libpcrecpp0 libssl-dev zlib1g-dev
apt-get install -y nginx
#chmod +x /etc/init.d/nginx
#upstart添加开机启动
#update-rc.d -f nginx defaults
#/usr/sbin/update-rc.d -f nginx defaults
#取消开机启动
#update-rc.d -f nginx remove
service nginx restart

#install mongodb
#!/bin/bash
set -e

#remove the lock
set +e
dpkg --configure -a
rm /var/lib/dpkg/lock > /dev/null
rm /var/cache/apt/archives/lock > /dev/null
set -e

#fixing locale issue
apt-get update -y
apt-get install language-pack-en-base -y
dpkg-reconfigure locales

cat >> ~/.profile << EOF
# Fix for locale issues when connecting to AWS Ubuntu instances over SSH
export LANG="en_GB.utf-8"
export LANGUAGE="en_GB.utf-8"
export LC_ALL="en_GB.utf-8"
EOF

#install mongodb 3.2
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
#on ubuntu 14
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
#reload local package

apt-get update
apt-get install -y mongodb-org
service mongod start
#install pymongo
pip install pymongo

pip install flask
pip install requests
#pip install "elasticsearch-dsl<1.0.0"
pip install gunicorn
pip install python-slugify
pip install flask-mongoengine
#pip install uwsgi
pip install supervisor
pip install redis

#配置supervisor
echo_supervisord_conf > /etc/supervisord.conf
#git clone https://github.com/Supervisor/initscripts.git
#cp initscripts/ubuntu /etc/init.d/supervisord
curl https://gist.github.com/howthebodyworks/176149/raw/88d0d68c4af22a7474ad1d011659ea2d27e35b8d/supervisord.sh > /etc/init.d/supervisord
chmod +x /etc/init.d/supervisord
update-rc.d supervisord defaults
supervisord

mkdir -p /etc/supervisor
cat >> /etc/supervisord.conf << EOF
[include]
files = /etc/supervisor/*.conf
EOF

supervisorctl reload
#service supervisord restart
#查看supervisor状态
#ps aux | grep supervisord
#设置redis密码
#vi /etc/redis/redis.conf
#设置redis密码
#vi /etc/redis/redis.conf
#取消注释# requirepass foobared
#用/requirepass进行查找,然后用n选择下一个
#requirepass jrtf8h4ewhrfe4hk
sed -i 's/# requirepass foobared/requirepass rediswico/g' /etc/redis/redis.conf
#修改后，重启Redis服务器。
/etc/init.d/redis-server restart

#instal phantomjs
#apt-get install -y phantomjs=1.9.0-1

#install pyspider
pip install pyspider
pip install six --upgrade
#remove phantomjs
rm -f /usr/bin/phantomjs

cat > /root/config.json << EOF
{
  "taskdb": "mongodb+taskdb://127.0.0.1:27017/pyspider_taskdb",
  "projectdb": "mongodb+projectdb://127.0.0.1:27017/pyspider_projectdb",
  "resultdb": "mongodb+resultdb://127.0.0.1:27017/pyspider_resultdb",
  "message_queue": "redis://root:rediswico@127.0.0.1:6379/db",
  "webui": {
    "username": "xxx",
    "password": "xxxxxx",
    "need-auth": true
  }
}
EOF

#设置pyspider用supervisor启动
cat > /etc/supervisor/pyspider.conf << EOF
[program:pyspider]
command=/usr/local/bin/pyspider -c /root/config.json
autostart=true
autorestart=true
user=root
directory=/root
EOF
#cat $/etc/supervisor/pyspider.conf >> $/etc/supervisor/supervisord.conf
supervisorctl reload