Skip to content

Instantly share code, notes, and snippets.

@chen206

chen206/tornado-server.md

Last active February 19, 2017 12:31
Show Gist options
  • Save chen206/4117467 to your computer and use it in GitHub Desktop.
Save chen206/4117467 to your computer and use it in GitHub Desktop.
Download ZIP
tornado server, ubuntu 12.04.2, mysql5.5, python2.7, tornado2.4.1, virtualenv, celery3.0.19, redis2.7.4, mongo
Raw
tornado-server.md

AptGet source.list

/etc/apt/source.list

deb http://cn.archive.ubuntu.com/ubuntu/ precise main restricted
deb-src http://cn.archive.ubuntu.com/ubuntu/ precise main restricted

deb http://cn.archive.ubuntu.com/ubuntu/ precise-updates main restricted
deb-src http://cn.archive.ubuntu.com/ubuntu/ precise-updates main restricted

deb http://cn.archive.ubuntu.com/ubuntu/ precise multiverse
deb-src http://cn.archive.ubuntu.com/ubuntu/ precise multiverse
deb http://cn.archive.ubuntu.com/ubuntu/ precise-updates multiverse
deb-src http://cn.archive.ubuntu.com/ubuntu/ precise-updates multiverse


deb http://cn.archive.ubuntu.com/ubuntu/ precise-backports main restricted universe multiverse
deb-src http://cn.archive.ubuntu.com/ubuntu/ precise-backports main restricted universe multiverse

deb http://security.ubuntu.com/ubuntu precise-security main restricted
deb-src http://security.ubuntu.com/ubuntu precise-security main restricted
deb http://security.ubuntu.com/ubuntu precise-security universe
deb-src http://security.ubuntu.com/ubuntu precise-security universe
deb http://security.ubuntu.com/ubuntu precise-security multiverse
deb-src http://security.ubuntu.com/ubuntu precise-security multiverse

Tornado Web Server

Python, virtualenv, pip

sudo apt-get update
sudo apt-get install -y python-setuptools python-dev python-virtualenv libmysqlclient-dev unzip git
sudo easy_install pip
sudo easy_install -U distribute

sudo chown -R lbadvisor:lbadvisor /opt
export WEBROOT=/opt/www
sudo mkdir -p $WEBROOT
cd $WEBROOT
touch $WEBROOT/requirements.txt

requirements

$WEBROOT/requirements.txt

tornado==2.4.1
celery==3.0.19
redis==2.7.4
MySQL-python==1.2.4
python-memcached==1.48
requests==1.2.0
supervisor==3.0b1
openpyxl==1.6.1
xlrd==0.9.2
xlwt==0.7.5
pymongo==2.5.2

virtualenv

cd $WEBROOT
virtualenv --distribute $WEBROOT
##virtualenv --distribute --python=/usr/bin/python2.7 myenv-py27
source $WEBROOT/bin/activate

pip install -r $WEBROOT/requirements.txt

##deactivate

UTF-8

sudo vi /etc/python2.7/sitecustomize.py

#(windows) C:/Python27/Lib/site-packages/sitecustomize.py
#(mac) /Library/Python/2.7/site-packages/sitecustomize.py

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

try:
  import apport_python_hook
except ImportError:
  pass
else:
  apport_python_hook.install()

Nginx

sudo apt-get install nginx

/etc/nginx/nginx.conf

daemon off;
worker_processes 2; #depends on cpu, max<=8

error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
    use epoll;
}

http {
    charset utf-8;

    # Enumerate all the Tornado servers here
    upstream frontends {
        server 127.0.0.1:8000;
        server 127.0.0.1:8001;
        server 127.0.0.1:8002;
        server 127.0.0.1:8003;
        server 127.0.0.1:8004;
        server 127.0.0.1:8005;
        server 127.0.0.1:8006;
        server 127.0.0.1:8007;
    }

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    access_log /var/log/nginx/access.log;

    keepalive_timeout 65;
    proxy_read_timeout 200;
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    gzip on;
    gzip_min_length 1000;
    gzip_proxied any;
    gzip_types text/plain text/css text/xml
               application/x-javascript application/xml
               application/atom+xml text/javascript;

    # Only retry if there was a communication error, not a timeout
    # on the Tornado server (to avoid propagating "queries of death"
    # to all frontends)
    proxy_next_upstream error;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

/etc/nginx/sites-available/atweiquan

server {
    listen 80;

    location ^~ /static/ {
        root /opt/;
        if ($query_string) {
            expires max;
        }
    }

    location / {
        proxy_pass_header Server;
        proxy_set_header Host $http_host;
        proxy_redirect off;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Scheme $scheme;
        proxy_pass http://frontends;
    }
}

sudo ln -s /etc/nginx/sites-available/atweiquan /etc/nginx/sites-enabled/atweiquan

MySQL

https://help.ubuntu.com/12.04/serverguide/mysql.html

sudo vi /etc/apt/source.list
sudo apt-get update
sudo apt-get install mysql-server mysql-client
sudo netstat -tap | grep mysql

#re-init db
sudo rm -rf /var/lib/mysql
sudo mkdir -p /var/lib/mysql
sudo mysql_install_db
sudo chown -R mysql: /var/lib/mysql

#change the MySQL root password
sudo dpkg-reconfigure mysql-server-5.5

mysql -u root -p

mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY 'mypass' WITH GRANT OPTION;
mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'192.168.1.%' IDENTIFIED BY 'lbs123' WITH GRANT OPTION;
mysql> quit;

sudo service mysql restart

Mongodb

sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10
echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' | sudo tee /etc/apt/sources.list.d/10gen.list
sudo apt-get update
sudo apt-get install mongodb-10gen

/etc/mongodb.conf

dbpath=/home/lbadvisor/thu
logpath=/var/log/mongodb/mongodb.log
slowms = 1500 #100
quiet = true
logappend=false
bind_ip = 192.168.1.24
#nojournal = true

/etc/logrotate.d/mongodb

# Put this in /etc/logrotate.d/mongodb
# http://stackoverflow.com/questions/5004626/mongodb-log-file-growth
 
/var/log/mongodb/*.log {
    daily
    size 50M
    rotate 10
    compress
    dateext
    missingok
    notifempty
    sharedscripts
    postrotate
        /bin/kill -SIGUSR1 `cat /var/lib/mongodb/mongod.lock 2> /dev/null` 2> /dev/null || true
    endscript
}

sudo chown -R mongodb:mongodb /home/lbadvisor/thu
sudo service mongodb start | stop | restart

sudo update-rc.d mongodb defaults

#sudo rm -rf /home/lbadvisor/thu/mongo.lock

# sudo mongod --fork -f /etc/mongodb.conf
# sudo mongod --shutdown -f /etc/mongodb.conf

Redis

su -
wget http://redis.googlecode.com/files/redis-2.6.11.tar.gz
tar xvfz redis-2.6.11.tar.gz 
cd redis-2.6.11/
mkdir -p /opt/redis
make PREFIX=/opt/redis install
cp redis.conf /opt/redis/redis.conf
adduser redis --no-create-home --disabled-password --disabled-login
#  useradd -rMU -d /opt/redis -s /bin/false -c redis-server redis
chown -R redis:redis /opt/redis
touch /var/log/redis.log
chown redis:redis /var/log/redis.log

Supervisord

supervisord.conf

/etc/supervisord.conf

[unix_http_server]
file=/tmp/supervisor.sock   ; path to your socket file
chmod=0700                       ; sockef file mode (default 0700)

[supervisord]
logfile=/var/log/supervisor/supervisord.log ; supervisord log file
logfile_maxbytes=50MB       ; maximum size of logfile before rotation
logfile_backups=10          ; number of backed up logfiles
loglevel=info               ; info, debug, warn, trace
pidfile=/var/run/supervisord.pid ; pidfile location
nodaemon=false              ; run supervisord as a daemon
minfds=1024                 ; number of startup file descriptors
minprocs=200                ; number of process descriptors
user=root                   ; default user
childlogdir=/var/log/supervisor/            ; where child log files will live

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use unix:// schem for a unix sockets.

[include]
files = /etc/supervisor/conf.d/*.conf

/etc/supervisord/conf.d/nginx.conf

[program:nginx]
priority=2
command=/usr/local/bin/pidproxy /var/run/nginx.pid /usr/sbin/nginx
stopsignal=INT
stdout_logfile=NONE
stderr_logfile=NONE

/etc/supervisord/conf.d/redis.conf

[program:redis]
priority=10
directory=/opt/redis/bin
command=/opt/redis/bin/redis-server /opt/redis/redis.conf
user=redis
redirect_stderr=true

/etc/supervisord/conf.d/atweiquan.conf

[program:atweiquan]
command=python /opt/atweiquan/atweiquan/app.py --port=80%(process_num)02d
process_name=%(program_name)s-80%(process_num)02d
directory=/opt/atweiquan/atweiquan/
autorestart=true
redirect_stderr=true
stdout_logfile=/var/log/atweiquan/%(program_name)s-80%(process_num)02d.log
stdout_logfile_maxbytes=500MB
stdout_logfile_backups=50
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
loglevel=info
numprocs=8
numprocs_start=0

/etc/supervisord/conf.d/celeryd.conf

; ============================
;  celeryd supervisor example
; ============================

; NOTE: If you're using Django, you shouldn't use this file.
; Use
; http://github.com/celery/django-celery/tree/master/extra/supervisord/celeryd.conf
; instead!

[program:celery]
command=celeryd --loglevel=INFO

; Set PYTHONPATH to the directory containing celeryconfig.py
environment=PYTHONPATH=/opt/atweiquan/atweiquan/tasks

directory=/opt/atweiquan/atweiquan/tasks
user=nobody
numprocs=1
stdout_logfile=/var/log/celeryd.log
stderr_logfile=/var/log/celeryd.log
autostart=true
autorestart=true
startsecs=10

; Need to wait for currently executing tasks to finish at shutdown.
; Increase this if you have very long running tasks.
stopwaitsecs = 600

; if rabbitmq is supervised, set its priority higher
; so it starts first
priority=998

/etc/supervisord/conf.d/celerybeat.conf

[program:celerybeat]
command=celerybeat --loglevel=INFO

environment=PYTHONPATH=/opt/atweiquan/atweiquan/tasks

directory=/opt/atweiquan/atweiquan/tasks
user=nobody
numprocs=1
stdout_logfile=/var/log/celerybeat.log
stderr_logfile=/var/log/celerybeat.log
autostart=true
autorestart=true
startsecs=10

priority=999
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment