Setting up the GIS stack on AWS

map_daq.sh

# load in OSM data to DB.

osm2pgsql -c -G -U scientraffic -d scientraffic_db los-angeles.osm.bz2 --style=/usr/share/osm2pgsql/default.style

nginx_site_scientraffic

server {
    listen 80;
    server_name scientraffic.com;
    server_name www.scientraffic.com;
    root /var/www/scientraffic;

    location /static/ {
        alias /var/www/scientraffic/traffic/static/;
        #include /etc/nginx/gzip.conf;
        expires 30d;
        access_log off;
    }

    location / {
        include uwsgi_params;
        uwsgi_pass unix:/tmp/scientraffic.sock;
    }
}

nginx_site_tiles

server {
    listen 80;
    server_name tiles.scientraffic.com;
    server_name tiles1.scientraffic.com;
    server_name tiles2.scientraffic.com;
    server_name tiles3.scientraffic.com;
    root /var/www/tilestache;

    location / {
        include uwsgi_params;
        uwsgi_pass unix:/tmp/tiles.sock;
    }
}

postgis_tables.sh

# Some good references are:
# http://russbrooks.com/2010/11/25/install-postgresql-9-on-os-x
# http://www.paolocorti.net/2008/01/30/installing-postgis-on-ubuntu/
# http://postgis.refractions.net/documentation/manual-1.5/ch02.html#id2630392
 
#1. Install PostgreSQL postgis and postgres
 
sudo apt-get install postgis
 
#2. Create a template to be used on creating GIS-enabled databases
 
sudo su postgres
 
createdb postgis_template
createlang plpgsql postgis_template
 
#Import Postgis Data
psql -d postgis_template -f /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql
psql -d postgis_template -f /usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql
 
#Test if works
psql -d postgis_template -c "SELECT postgis_full_version();"
 
#3. Set template permissions to gisgroup
createuser -R -S -L -D -I gisgroup;
 
psql -d postgis_template
ALTER DATABASE postgis_template OWNER TO gisgroup;
ALTER TABLE geometry_columns OWNER TO gisgroup;
ALTER TABLE spatial_ref_sys OWNER TO gisgroup;
CREATE SCHEMA gis_schema AUTHORIZATION gisgroup;
\q
 
#4. Adds an app to your application
createuser -i -l -S -R -d scientraffic
 
psql -d postgres
GRANT gisgroup TO scientraffic;
\q
 
#5. Create your app database
createdb -T postgis_template -O scientraffic scientraffic_db;
 
# edit the permissions file to add the scientraffic user and restart
sudo vim /etc/postgresql/9.1/main/pg_hba.conf
# NB that the line
# local   all             scientraffic                            trust
# must be above 
# local   all             all                                     peer
/etc/init.d/postgresql reload

Recipe.md

Setting up the GIS stack on AWS

General

sudo apt-get install fail2ban
sudo apt-get install htop
sudo apt-get install git

GCC and friends

sudo apt-get install build-essential 
sudo apt-get install libboost-all-dev
sudo apt-get install cmake

Basic Python

Python.h, etc

sudo apt-get install python-dev

For easy install:

sudo apt-get install python-setuptools

Now pip:

sudo easy_install pip
sudo pip install virtualenv

Important C libararies need for lxml & friends

# needed for lxml
sudo apt-get install libxml2-dev libxslt-dev
sudo apt-get install mysql-client libmysqlclient-dev

Numerical analysis:

sudo pip install numpy matplotlib ipython

Mapnik

From https://github.com/mapnik/mapnik/wiki/UbuntuInstallation

sudo apt-get install osm2pgsql
sudo apt-get install -y python-software-properties
sudo apt-get install software-properties-common
sudo add-apt-repository ppa:mapnik/v2.1.0
sudo apt-get update
sudo apt-get install libmapnik mapnik-utils python-mapnik
sudo pip install nik2img

TileStache

Out of the box, we have problems with some PIL zip decoder. See here: https://milkator.wordpress.com/2012/10/26/install-and-setup-tilestache/

sudo apt-get install libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev
sudo pip uninstall PIL
sudo ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib
sudo ln -s /usr/lib/x86_64-linux-gnu/libfreetype.so /usr/lib
sudo ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib
sudo pip install -U PIL

Dennis-OSRM

From https://github.com/DennisOSRM/Project-OSRM/wiki/Building%20OSRM

sudo apt-get install build-essential git scons pkg-config libprotoc-dev libprotobuf7 \
protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev \
libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev \
libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev \
libboost-filesystem-dev lua5.1 liblua5.1-0-dev libluabind-dev

for running the tests (this is 279MB!):

sudo apt-get install rubygems osmosis
sudo gem install bundler

set up a swap file, the micro memory is too small to compile all of it.

From http://inprvt.com/index.php/blogs/entry/how-to-add-swap-space-on-a-linux-based-ec2-server

dd if=/dev/zero of=/swapfile1 bs=1024 count=524288
mkswap /swapfile1
swapon /swapfile1

(I'm not going to automount it)

Now compile the package:

git clone https://github.com/DennisOSRM/Project-OSRM.git
cd Project-OSRM && scons

Graphing Analysis

See: http://igraph.wikidot.com/installing-python-igraph-on-linux

sudo add-apt-repository ppa:igraph/ppa
sudo apt-get update
sudo apt-get install python-igraph

# For dionysus, for alpha shapes.
# http://www.mrzv.org/software/dionysus/index.html
# needs CGAL and cmake and boost

hg clone http://hg.mrzv.org/Dionysus/
cd Dionysus
hg up tip
mkdir build
cd build
cmake ..
make

Webserving

Nginx & uWSGI

Following http://tghw.com/blog/multiple-django-and-flask-sites-with-nginx-and-uwsgi-emperor

sudo apt-get install nginx uwsgi

Configure uwsgi

curl "https://gist.github.com/ekfriis/5075470/raw/uwsgi.conf" | sudo tee /etc/init/uwsgi.conf
sudo mkdir -p /var/log/uwsgi
sudo mkdir -p /etc/uwsgi/apps-available
sudo mkdir -p /etc/uwsgi/apps-enabled

Importing OSM data

curl -O http://osm-metro-extracts.s3.amazonaws.com/los-angeles.osm.pbf 
osm2pgsql -c -G -U scientraffic -d scientraffic_db los-angeles.osm.bz2 --style=/usr/local/share/osm2pgsql/default.style

This doesn't work - not enough memory (I think). Let's do the heavy lifting on the laptop.
To tunnel, I had to change md5->trust for host in the line

#/etc/postgresql/9.1/main/pg_hba.conf 
host    all             all             127.0.0.1/32            trust

Then restart with sudo /etc/init.d/postgresql restart

ssh -L 9999:127.0.0.1:5432 friis@www.scientraffic.com
osm2pgsql -c -G -U scientraffic -d scientraffic_db los-angeles.osm.bz2 --style=/usr/local/share/osm2pgsql/default.style --port 9999 --host localhost 

Rendering Maps

From https://github.com/joyent/node/wiki/Installing-Node.js-via-package-manager

sudo apt-get install python-software-properties python g++ make
sudo add-apt-repository ppa:chris-lea/node.js
sudo apt-get update
sudo apt-get install nodejs npm
sudo npm install -g carto millstone optimist

uwsgi.conf

description "uWSGI"
start on runlevel [2345]
stop on runlevel [06]
respawn

env UWSGI=/usr/bin/uwsgi
env LOGTO=/var/log/uwsgi/emperor.log

exec $UWSGI --master --emperor /etc/uwsgi/apps-enabled --die-on-term --uid www-data --gid www-data --logto $LOGTO

uwsgi_app_scientraffic.ini

# Lives in /etc/uwsgi/apps-available/scientraffic.ini
[uwsgi]
# Variables
base = /var/www/scientraffic
callable = application
# Generic Config
#plugins = python
env = HOME=/home/friis
home = %(base)/venv
pythonpath = %(base)
chdir = %(base)
module = app
logto = /var/log/uwsgi/app/scientraffic.log
socket = /tmp/scientraffic.sock