cartodb install

gistfile1.sh

#!/bin/sh

# hostname = cartodb.e-ecology.cloudlet.sara.nl
# user = verhoes = verhoes.cartodb.e-ecology.cloudlet.sara.nl

apt-get update
apt-get upgrade -y
echo 'cartodb.e-ecology.cloudlet.sara.nl' > /etc/hostname

# reboot

# snapshot upgrade

apt-get install git -y
cd /opt
git clone --recursive https://github.com/CartoDB/cartodb.git
cd cartodb
git checkout v2.11.0

# snapshot clone cartodb

apt-get install python-software-properties -y

# gdal 1.9.2, don't use 1.10.0 because it conflicts with postgis and only needed from cartodb>2.2
add-apt-repository ppa:ubuntugis/ppa -y
# os node is 0.6.12, use repo for  0.10.27
add-apt-repository ppa:chris-lea/node.js -y
# os mapnik is 2.0.0, need 2.1.1 use repo
add-apt-repository ppa:mapnik/v2.2.0 -y
echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
apt-get update
apt-get upgrade

apt-get install unp zip -y
apt-get install libgeos-c1 libgeos-dev -y

apt-get install gdal-bin libgdal1-dev -y
apt-get install libjson0 python-simplejson libjson0-dev -y
apt-get install proj-bin proj-data libproj-dev -y

# uninstalled gdal to be able to install pg9.3
apt-get install postgresql-client-9.3 postgresql-contrib-9.3 postgresql-plpython-9.3 postgresql-9.3-postgis-2.1 -y

apt-get install nodejs -y

apt-get install redis-server -y
apt-get install python2.7-dev build-essential -y
# compile varnish 2 instaed of installing 3
#apt-get install varnish -y

apt-get install libmapnik-dev python-mapnik mapnik-utils -y
apt-get install imagemagick -y
apt-get install curl -y
apt-get install ruby1.9.1-dev -y
apt-get install python-pip python-gdal -y

# snapshot apt-get

-u postgres createdb -E UTF8 --locale=en_US.utf8 -T template0 template_postgis
-u postgres createlang -d template_postgis plpgsql
-u postgres psql -d postgres -c "UPDATE pg_database SET datistemplate='true' WHERE datname='template_postgis'"
-u postgres psql -d template_postgis -c "CREATE EXTENSION postgis;"
-u postgres psql -d template_postgis -c "CREATE EXTENSION postgis_topology;"
-u postgres psql -d template_postgis -c "CREATE EXTENSION fuzzystrmatch;"
-u postgres psql -d template_postgis -c "CREATE EXTENSION postgis_tiger_geocoder;"
-u postgres psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"
-u postgres psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"

# snapshot template_postgis



-i
cd /opt/cartodb
export CPLUS_INCLUDE_PATH=/usr/include/gdal
export C_INCLUDE_PATH=/usr/include/gdal
# edit python_requirements.txt change, as postgis-2.1 is compiled against 1.9.2
<gdal==1.10.0
>gdal==1.9.2

pip install -r python_requirements.txt

cd /opt
git clone git://github.com/CartoDB/CartoDB-SQL-API.git
cd CartoDB-SQL-API
npm install

cd ..
git clone git://github.com/CartoDB/Windshaft-cartodb.git
cd Windshaft-cartodb
npm install

exit

# Snapshot reqsdone

-u postgres psql -d postgres -c "ALTER USER postgres WITH PASSWORD 'postgres'"
# In config file add postgres pw

gem install bundler
cd /opt/cartodb
bundle install
cp config/app_config.yml.sample config/app_config.yml
# edit it => Bind to 0.0.0.0, domains, pw
# change varnish.purge_command from purge to ban, see https://github.com/CartoDB/cartodb/issues/333
diff config/app_config.yml config/app_config.yml.sample
4c4
<   session_domain:     '.cartodb.e-ecology.cloudlet.sara.nl'
---
>   session_domain:     '.localhost.lan'
11c11
<       domain:        'cartodb.e-ecology.cloudlet.sara.nl'
---
>       domain:        'localhost.lan'
17c17
<       domain:        'cartodb.e-ecology.cloudlet.sara.nl'
---
>       domain:        'localhost.lan'
22c22
<       domain:        'cartodb.e-ecology.cloudlet.sara.nl'
---
>       domain:        'localhost.lan'
28c28
<       domain:     'cartodb.e-ecology.cloudlet.sara.nl'
---
>       domain:     'localhost.lan'
33c33
<       domain:     'cartodb.e-ecology.cloudlet.sara.nl'
---
>       domain:     'localhost.lan'
37c37
<   developers_host:    'http://verhoes.cartodb.e-ecology.cloudlet.sara.nl:3000'
---
>   developers_host:    'http://developers.localhost.lan:3000'
48c48
<     purge_command: 'ban'
---
>     purge_command: 'purge'
129c129
<         tiler_domain:       "cartodb.e-ecology.cloudlet.sara.nl"
---
>         tiler_domain:       "localhost.lan"
132c132
<         sql_domain:         "cartodb.e-ecology.cloudlet.sara.nl"
---
>         sql_domain:         "localhost.lan"
158c158
<   cartodb_com_hosted: false
---
>   cartodb_com_hosted: true
179c179
<     purge_command: 'ban'
---
>     purge_command: 'purge'

cp config/database.yml.sample config/database.yml
diff config/database.yml config/database.yml.sample
8c8
<   password: postgres
---
>   password:
17c17
<   password: postgres
---
>   password:
26c26
<   password: postgres
---
>   password:
35c35
<   password: postgres
---
>   password:

cd ../CartoDB-SQL-API
cp config/environments/development.js.example config/environments/development.js
# edit it => Bind to 0.0.0.0, domains, pw
diff config/environments/development.js config/environments/development.js.example
5c5
< module.exports.user_from_host = '^(.*)\\.cartodb.e-ecology.cloudlet.sara.nl';
---
> module.exports.user_from_host = '^(.*)\\.localhost';
7c7
< module.exports.node_host    = '0.0.0.0';
---
> module.exports.node_host    = '127.0.0.1';
19c19
< module.exports.db_pubuser_pass   = 'publicuser';
---
> module.exports.db_pubuser_pass   = 'public';

cd ../Windshaft-cartodb
cp config/environments/development.js.example config/environments/development.js
# edit it => Bind to 0.0.0.0, domains, pw
diff config/environments/development.js config/environments/development.js.example
4c4
<     ,host: '0.0.0.0'
---
>     ,host: '127.0.0.1'
7c7
<     ,user_from_host: '^(.*)\\.cartodb.e-ecology.cloudlet.sara.nl'
---
>     ,user_from_host: '^(.*)\\.localhost'
50c50
<         password: "publicuser",
---
>         password: "public",
105c105
<         domain: 'cartodb.e-ecology.cloudlet.sara.nl',
---
>         domain: 'localhost.lan',

cd /opt/cartodb
git diff
diff --git a/Procfile b/Procfile
index 63dc37c..397ff62 100644
--- a/Procfile
+++ b/Procfile
@@ -1,5 +1,4 @@
 web:    bundle exec rails server -p $PORT
 sql:    node ../CartoDB-SQL-API/app.js development
 tiler:  node ../Windshaft-cartodb/app.js development
-redis:  redis-server
 resque: script/resque


# on own machine

echo '145.100.58.109 cartodb.e-ecology.cloudlet.sara.nl verhoes.cartodb.e-ecology.cloudlet.sara.nl' >> /etc/hosts

# on carto db machine

dpkg-reconfigure locales
-u postgresql psql
update pg_database set datistemplate=false where datname='template1';
drop database Template1;
create database template1 with owner=postgres encoding='UTF-8' lc_collate='en_US.utf8' lc_ctype='en_US.utf8' template template0;
update pg_database set datistemplate=true where datname='template1';

# snapshot createuser

sh script/create_dev_user verhoes
Enter a password (cleartext!): ********
Enter an admin password (cleartext!): ********
Enter an email: verhoes@cartodb.e-ecology.cloudlet.sara.nl
--- Creating databases
[sequel] Created database 'carto_db_development'
[sequel] Created database 'carto_db_test'
Creating user publicuser
Creating user tileuser
rake aborted!
PG::Error: FATAL:  password authentication failed for user "publicuser"
FATAL:  password authentication failed for user "publicuser"

-u postgresql psql
alter user publicuser password 'publicuser';

sh script/create_dev_user verhoes
Enter a password (cleartext!): ********
Enter an admin password (cleartext!): ********
Enter an email: verhoes@cartodb.e-ecology.cloudlet.sara.nl
--- Creating databases
createdb: database creation failed: ERROR:  database "carto_db_development" already exists
[sequel] Created database 'carto_db_development'
createdb: database creation failed: ERROR:  database "carto_db_test" already exists
[sequel] Created database 'carto_db_test'
Creating user publicuser
Creating user tileuser
User verhoes created successfully


bundle pack -all
bundle install --path vendor/cache

cd ../Windshaft-cartodb
npm install
node app.js development
# ctrl-c
cd ../CartoDB-SQL-API
npm install
node app.js development
# ctrl-c

bundle exec foreman start -p 3000

# GOTO http://verhoes.cartodb.e-ecology.cloudlet.sara.nl:3000 login with verhoes:********

# snapshot online

# upload 669.csv
#keeps importing

# start extra importer 
VERBOSE=true QUEUE=* rake environment resque:work
# ctrl-c

# Table done, but page errors
-u postgres psql cartodb_dev_user_7710524a-d0a2-4951-83a8-1a44220cffb9_db
select * from pg_stat_activity;
# insert stuck on cdb_tablemetadata table

nano /etc/varnish/default
backend default {
    .host = "127.0.0.1";
    .port = "3000";
}
sub vcl_recv {     
    return (lookup);
}

/etc/init.d/varnish restart

# Hanging query solved, but table page still erors

# Goto visualizations, same error

# Delete tables

# Add new table

# Restarting varnish trigger import unstucking
/etc/init.d/varnish stop
# also works


# Downgrade cartodb
mv Gemfile.lock Gemfile.lock.mine
git checkout v2.11.0
bundle install

# Error ogr2ogr output:    -nlt PROMOTE_TO_MULTI: type not recognised.

diff --git a/services/importer/lib/importer/ogr2ogr.rb b/services/importer/lib/importer/ogr2ogr.rb
index 4e2e5b3..a2edfbd 100644
--- a/services/importer/lib/importer/ogr2ogr.rb
+++ b/services/importer/lib/importer/ogr2ogr.rb
@@ -93,7 +93,7 @@ module CartoDB
       end #precision_option
 
       def new_layer_type_option
-        "-nlt PROMOTE_TO_MULTI"
+        ""
       end #new_layer_type_option


# works

# give users more rights
bundle exec rake cartodb:db:set_user_quota["verhoes",10240]
bundle exec rake cartodb:db:set_unlimited_table_quota["verhoes"]
bundle exec rake cartodb:db:set_user_private_tables_enabled["verhoes",'true']
bundle exec rake cartodb:db:set_user_account_type["verhoes",'[DEDICATED]']