cartodb install notes 12.04.md

Cartodb install on Digital Ocean ubuntu 12.04 64-bit

based on https://github.com/CartoDB/cartodb with additions as necessary
11/2013 to 02/2014

Install git

sudo apt-get install git-core

Clone project

git clone --recursive https://github.com/CartoDB/cartodb.git

Some dependencies

apt-get update
sudo apt-get install python-software-properties

More dependencies

sudo apt-get install unp
sudo apt-get install zip

Add Cartodb PPAs

sudo add-apt-repository ppa:cartodb/gis
sudo add-apt-repository ppa:cartodb/mapnik
sudo add-apt-repository ppa:cartodb/nodejs
sudo add-apt-repository ppa:cartodb/redis

apt-get update

GDAL

sudo apt-get install gdal-bin libgdal1-dev

GEOS

# note: seems all ok from above; 0 upgraded, 0 newly installed, 0 to remove and 20 not upgraded.
sudo apt-get install libgeos-c1 libgeos-dev

JSON-C

sudo apt-get install libjson0 python-simplejson libjson0-dev

PROJ-4

sudo apt-get install proj-bin proj-data libproj-dev

POSTGRESQL

# Note: 12.04 so not using carto ppa here: sudo add-apt-repository ppa:cartodb/postgresql 
sudo apt-get install postgresql-9.1 postgresql-client-9.1 postgresql-contrib-9.1 postgresql-server-dev-9.1

PLP-PYTHON

sudo apt-get install postgresql-plpython-9.1

POST-GIS

cd /usr/local/src
wget http://download.osgeo.org/postgis/source/postgis-2.0.2.tar.gz
tar xzf postgis-2.0.2.tar.gz
cd postgis-2.0.2
./configure --with-raster --with-topology

# necessary in my 12.04
sudo apt-get install make

make
make install

Configure spatial-db tempate

# create and run setup script
touch /var/lib/postgresql/config_db.sh

nano /var/lib/postgresql/config_db.sh

#!/usr/bin/env bash
POSTGIS_SQL_PATH=/usr/share/postgresql/9.1/contrib/postgis-2.0
createdb -E UTF8 template_postgis
createlang -d template_postgis plpgsql
psql -d postgres -c \ "UPDATE pg_database SET datistemplate='true' WHERE datname='template_postgis'"
psql -d template_postgis -f $POSTGIS_SQL_PATH/postgis.sql
psql -d template_postgis -f $POSTGIS_SQL_PATH/spatial_ref_sys.sql
psql -d template_postgis -f $POSTGIS_SQL_PATH/legacy.sql
psql -d template_postgis -f $POSTGIS_SQL_PATH/rtpostgis.sql
psql -d template_postgis -f $POSTGIS_SQL_PATH/topology.sql
psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"
psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"

sudo su - postgres 
bash config_db.sh

# confirm template exists with
psql --command="\l"
exit

Install Ruby 1.9.2 (using RVM)

\curl -L https://get.rvm.io | bash
source /etc/profile.d/rvm.sh
rvm install 1.9.2

Node

sudo apt-get install nodejs npm

Redis

sudo apt-get install redis-server

ImageMagik

# something to do with windshaft and comparing images
sudo apt-get install imagemagick

Install Python dependencies

sudo apt-get install python-setuptools
easy_install pip
pip install -r cartodb20/python_requirements.txt

# when python-gdal bindings fail to install using above
sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
sudo apt-get update
sudo apt-get install python-gdal

sudo apt-add-repository --remove ppa:ubuntugis/ubuntugis-unstable
apt-get update

Varnish

# RealGeeks version recommended in Cartodb docs
sudo pip install -e git+https://github.com/RealGeeks/python-varnish.git@0971d6024fbb2614350853a5e0f8736ba3fb1f0d#egg=python-varnish

Mapnik

# following not working, note these seem to be v 2.1.1 
## sudo apt-get install libmapnik-dev python-mapnik mapnik-utils

# so remove the cartodb ppa
sudo apt-add-repository --remove ppa:cartodb/mapnik

# install mapnik ppa
sudo add-apt-repository ppa:mapnik/nightly-2.1 
apt-get update

# install boost (q: with suggests? --install-suggests, a: so far seems ok without)
sudo apt-get install libboost1.49-dev

# then install mapnik
sudo apt-get install libmapnik-dev python-mapnik mapnik-utils

# can test with
python
import mapnik

CartoDB SQL API

git clone git://github.com/CartoDB/CartoDB-SQL-API.git
cd CartoDB-SQL-API
git checkout master 
npm install

mv config/environments/development.js.example config/environments/development.js
# set: module.exports.node_host    = ''; # so it listens all hostsy

# run test
export PGUSER=postgres # note see config below
make check # 143 tests pass

Windshaft-cartodb

# note about g++ memory error (only 512 MB RAM for me: http://stackoverflow.com/questions/14800609/phusion-passenger-nginx-module-installation-error-on-ubuntu-server-12-04-2-64)
git clone git://github.com/CartoDB/Windshaft-cartodb.git
cd Windshaft-cartodb
git checkout master
npm install

mv config/environments/development.js.example config/environments/development.js
# set mapnik version "2.1.1"
# set ,host in line 4 to: '' so it listens on any port
# create millstone directory writable
mkdir -p /tmp/cdb-tiler-dev/millstone-dev

# testing
# 1. echo PGUSER=postgres
# 2. make check

Additional pg config

## pg
#In /etc/postgresql/9.1/main/pg_hba.conf, set auth method to "trust":
local   all             postgres                                trust
host    all             all             127.0.0.1/32            trust

# restart psql
sudo service postgresql restart

# set pw (also add to database.yml -- optional/redundant given above??
sudo -u postgres psql
\password
(Set password) 
\q

Setup (see https://github.com/CartoDB/cartodb for details)

export SUBDOMAIN=carto # unset SUBDOMAIN to undo
cd cartodb

# bundle
rvm use 1.9.2@cartodb --create && bundle install # from cartodb/
# sudo bundle install # not needed? anyway bundle not found for sudo

mv config/app_config.yml.sample config/app_config.yml
nano config/app_config.yml
# only change so far: developers_host:    'http://carto.localhost.lan:3000'
# also quote sql_api_ports '8080' (private and public)

mv config/database.yml.sample config/database.yml
nano config/database.yml # no changes since no pw for me

echo "127.0.0.1 ${SUBDOMAIN}.localhost.lan" | sudo tee -a /etc/hosts
# add similar entry to local /etc/hosts
xxx.xxx.xxx.xxx carto.localhost.lan
xxx.xxx.xxx.xxx admin.localhost.lan

Create user script and alter limits

# start redis-server with redis-server
# create user script (edit to avoid error): sh script/create_dev_user ${SUBDOMAIN}
bundle exec rake cartodb:db:set_user_quota["myuser",10240] 
bundle exec rake cartodb:db:set_unlimited_table_quota["myuser"]
bundle exec rake cartodb:db:set_user_private_tables_enabled["myuser",'true']
bundle exec rake cartodb:db:set_user_account_type["myuser",'[DEDICATED]'] 

Start all servers

bundle exec foreman start -p 3000

What to do if compass errors on startup/login:

enter your /cartodb directory
$ git submodule init
$ git submodule update

see: http://gis.stackexchange.com/questions/81218/compass-error-cant-find-proper-directories

Useful miscellany

# redis clean save and shutdown
redis-cli save
redis-cli shutdown

apt-cache policy mapnik-utils

# drop db
sudo su - postgres
dropdb carto_db_development

# drop roles
sudo -u postgres psql
DROP ROLE publicuser
DROP ROLE tileuser

# node tests
export PGUSER = "postgres"
make check # 143 tests complete

# restart pg
sudo service postgresql restart

# references
ref to 12.04 setup: https://gist.github.com/arjendk/63a0603d9dd278503edb

# redis 
redis-cli -n 5 monitor
echo "hget rails:users:carto database_name" | redis-cli -n 5
echo "keys rails:users*" | redis-cli -n 5
sh script/restore_redis
redis-cli save; redis-cli stop

config files: app_config.yml

defaults: &defaults
  debug_assets: true
  mandatory_keys:     [layer_opts, sql_api, varnish_management, redis, session_domain]
  session_domain:     '.localhost.lan'
  secret_token:       '71c2b25921b84a1cb21c71503ab8fb23'
  account_host:       'localhost:3000'
  account_path:       '/account'
  tiler:
    private:
      protocol:      'http'
      domain:        'localhost.lan'
      port:          '8181'
    public:
      protocol:      'http'
      domain:        'localhost.lan'
      port:          '8181'
  sql_api:
    private:
      protocol:   'http'
      domain:     'localhost.lan'
      endpoint:   '/api/v1/sql'
      port:       '8080'
    public:
      protocol:   'http'
      domain:     'localhost.lan'
      endpoint:   '/api/v1/sql'
      port:       '8080'
  api_requests_service_url: ''
  developers_host:    'http://carto.localhost.lan:3000'
  google_analytics:
    primary:          ''
    embeds:           ''
    domain:           ''
  varnish_management:
    critical: false
    host: '127.0.0.1'
    port: 6082
    purge_command: 'purge'
    retries: 5
    timeout: 5
  redis:
    host: '127.0.0.1'
    port: 6379
    databases:
      tables_metadata:     0
      api_credentials:     3
      users_metadata:      5
      redis_migrator_logs: 6
#  mixpanel:
#    api_key: xxx
#    api_secret: xxx
#    token: xxx
#  github:
#    repo: xxx
#    org: xxx
#    auth: xxx:xxx
  superadmin:
    username: "superadmin"
    password: "monkey"
  geocoder:
    app_id: ''
    token:  ''
    mailto: ''
  importer:
    s3:
      access_key_id:
      secret_access_key:
      bucket_name:
      url_ttl:
  error_track:
    url: 'https://viz2.cartodb.com/api/v1/sql'
    percent_users: 10












  layer_opts:
    public_opts: ["type", "active", "query", "opacity", "auto_bound", 
                  "interactivity", "debug", "visible", "tiler_domain", 
                  "tiler_port", "tiler_protocol", "sql_domain", "sql_port", 
                  "sql_protocol", "extra_params", "cdn_url", "table_name", 
                  "user_name", "style_version", "tile_style", "query_wrapper"]
    default_tile_styles:
      point: "{\n  marker-fill: #FF6600;\n  marker-opacity: 0.9;\n  marker-width: 12;\n  marker-line-color: white;\n  marker-line-width:
 3;\n  marker-line-opacity: 0.9;\n  marker-placement: point;\n  marker-type: ellipse;\n  marker-allow-overlap: true;\n}"
      geometry: "{\n // points\n [mapnik-geometry-type=point] {\n    marker-fill: #FF6600;\n    marker-opacity: 1;\n    marker-width: 12
;\n    marker-line-color: white;\n    marker-line-width: 3;\n    marker-line-opacity: 0.9;\n    marker-placement: point;\n    marker-typ
e: ellipse;marker-allow-overlap: true;\n  }\n\n //lines\n [mapnik-geometry-type=linestring] {\n    line-color: #FF6600; \n    line-width
: 2; \n    line-opacity: 0.7;\n  }\n\n //polygons\n [mapnik-geometry-type=polygon] {\n    polygon-fill:#FF6600;\n    polygon-opacity: 0.
7;\n    line-opacity:1;\n    line-color: #FFFFFF;\n   }\n }"
      polygon: "{\n  polygon-fill:#FF6600;\n  polygon-opacity: 0.7;\n  line-opacity:1;\n  line-color: #FFFFFF;\n}"
      multipolygon: "{\n  polygon-fill:#FF6600;\n  polygon-opacity: 0.7;\n  line-opacity:1;\n  line-color: #FFFFFF;\n}"
      multilinestring: "{\n  line-color:#FF6600;\n  line-width:1;\n  line-opacity: 0.7;\n}"
    data:
      kind: "carto"
      options:
        query:              ""
        opacity:            0.99
        auto_bound:         false
        interactivity:      "cartodb_id"
        debug:              false
        visible:            true
        tiler_domain:       "localhost.lan"
        tiler_port:         "80"
        tiler_protocol:     "http"
        sql_domain:         "localhost.lan"
        sql_port:           "80"
        sql_protocol:       "http"
        extra_params:       { cache_policy: 'persist' }
        cdn_url:            ""
        tile_style_history: []
        style_version:      "2.1.1"
      infowindow:
        template_name:      "table/views/infowindow_light"
    background:
      kind: "background"
      options: { color: '#ffffff' }
    base:
      kind: "tiled"
      options:
        visible:      true
        type:         "Tiled"
        urlTemplate:  "https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_
zghvKt9Q&app_id=KuYppsdXZznpffJsKT24"
        name:         'Nokia Day'
        className:    "nokia_day"
        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
    gmaps:
      kind: "gmapsbase"
      options:
        base_type: "roadmap"
        style: ""
  cartodb_com_hosted: true
  cartodb_central_domain_name: 'cartodb.com'
  aws:
    s3:
      access_key_id: "test"
      secret_access_key: "test"
  assets:
    s3_bucket_name: "tests"
    max_file_size: 5242880 # 5.megabytes
  dropbox_api_key: ""
  gdrive: 
    api_key: ""
    app_id: ""

development:
  <<: *defaults
  varnish_management:
    critical: false
    host: '127.0.0.1'
    port: 6082
    purge_command: 'purge'
    url_purge_command: 'url.purge'
    retries: 5
    timeout: 5

test:
  <<: *defaults
  redis:
    host: '127.0.0.1'
    port: 6335

staging:
  <<: *defaults

production:
  <<: *defaults

config files: Windshaft development.js

Windshaft-cartodb/config/environments/development.js 
var config = {
     environment: 'development'
    ,port: 8181
    ,host: ''
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
    ,user_from_host: '^(.*)\\.localhost'
    // Maximum number of connections for one process
    // 128 is a good value with a limit of 1024 open file descriptors
    ,maxConnections:128
    // idle socket timeout, in miliseconds 
    ,socket_timeout: 600000
    ,enable_cors: true
    ,cache_enabled: false
    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]
)'
    // Templated database username for authorized user
    // Supported labels: 'user_id' (read from redis)
    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
    // Templated database password for authorized user
    // Supported labels: 'user_id', 'user_password' (both read from redis)
    ,postgres_auth_pass: '<%= user_password %>'
    ,postgres: {
        // Parameters to pass to datasource plugin of mapnik
        // See http://github.com/mapnik/mapnik/wiki/PostGIS
        type: "postgis",
        user: "publicuser",
        password: "public",
        host: '127.0.0.1',
        port: 5432,
        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
        /* experimental
        geometry_field: "the_geom",
        extent: "-180,-90,180,90",
        srid: 4326,
        */
        row_limit: 65535,
        simplify_geometries: true,
        max_size: 500
    }
    ,mapnik_version: '2.1.1'
    ,renderer: {
      // Milliseconds since last access before renderer cache item expires
      cache_ttl: 60000,
      metatile: 4,
      bufferSize: 64
    }
    ,millstone: {
        // Needs to be writable by server user
        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
    }
    ,redis: {
        host: '127.0.0.1',
        port: 6379,
        // Max number of connections in each pool.
        // Users will be put on a queue when the limit is hit.
        // Set to maxConnection to have no possible queues.
        // There are currently 3 pools involved in serving
        // windshaft-cartodb requests so multiply this number
        // by 3 to know how many possible connections will be
        // kept open by the server. The default is 50.
        max: 50,
        idleTimeoutMillis: 1, // idle time before dropping connection
        reapIntervalMillis: 1 // time between cleanups
    }
    ,sqlapi: {
        protocol: 'http',
        domain: 'localhost.lan',
        port: 8080,
        version: 'v1'
    }
    ,varnish: {
        host: 'localhost',
        port: 6082,
        ttl: 86400
    }
    // If useProfiler is true every response will be served with an
    // X-Tiler-Profile header containing elapsed timing for various 
    // steps taken for producing the response.
    ,useProfiler:true
};

module.exports = config;

config files:CartoDB-SQL-API development.js

more ../CartoDB-SQL-API/config/environments/development.js 
module.exports.base_url     = '/api/:version';
module.exports.node_port    = 8080;
module.exports.node_host    = '';
// idle socket timeout, in miliseconds
module.exports.node_socket_timeout    = 600000;
module.exports.environment  = 'development';
module.exports.db_base_name = 'cartodb_dev_user_<%= user_id %>_db';
module.exports.db_user      = 'development_cartodb_user_<%= user_id %>';
module.exports.db_host      = 'localhost';
module.exports.db_port      = '5432';
// Max database connections in the pool
// Subsequent connections will wait for a free slot.
// NOTE: not used by OGR-mediated accesses 
module.exports.db_pool_size = 500;
// Milliseconds before a connection is removed from pool
module.exports.db_pool_idleTimeout = 30000;
// Milliseconds between idle client checking
module.exports.db_pool_reapInterval = 1000;
module.exports.redis_host   = '127.0.0.1';
module.exports.redis_port   = 6379;
module.exports.redisPool    = 50;
module.exports.redisIdleTimeoutMillis   = 100;
module.exports.redisReapIntervalMillis  = 10;
module.exports.redisLog     = false;
// Max number of entries in the query tables cache
module.exports.tableCacheMax = 8192;
// Max age of query table cache items, in milliseconds
module.exports.tableCacheMaxAge = 1000*60*10;
// Temporary directory, make sure it is writable by server user
module.exports.tmpDir = '/tmp';