Install graphite with statsd and login/password auth

auth.wsgi

import md5
import os, sys

sys.path.append('/opt/graphite/webapp')
os.environ['DJANGO_SETTINGS_MODULE'] = 'graphite.settings'

from django.contrib.auth.models import User
from django import db

def check_password(environ, user, password):
    db.reset_queries()
    kwargs = {'username': user, 'is_active': True}
    try:
        try:
            user = User.objects.get(**kwargs)
        except User.DoesNotExist:
            return None

        if user.check_password(password):
            return True
        else:
            return False
    finally:
        db.connection.close()

def get_realm_hash(environ, user, realm):
    db.reset_queries()
    kwargs = {'username': user, 'is_active': True}
    try:
        try:
            user = User.objects.get(**kwargs)
        except User.DoesNotExist:
            return None

        if user:
            value = md5.new()
            # user:realm:password
            value.update('%s:%s:%s' % (user, realm, user.password))
            hash = value.hexdigest()
            return hash
        else:
            return None
    finally:
        db.connection.close()

daily.sh

#!/bin/bash
LOGS_DIR="logs"
REPORTS_DIR="reports"
for i in $(ls $LOGS_DIR);do
    if [ -d $LOGS_DIR/$i ];then
	mkdir -p $REPORTS_DIR/$i
	LAST_LOG=$(ls -rt1 $LOGS_DIR/$i/*.log 2>/dev/null| tail -n 1)
	if [ ! -z $LAST_LOG ];then
	    perl ./pgbadger $LAST_LOG -o $REPORTS_DIR/$i/`basename $LAST_LOG`.html
	    mv $LAST_LOG $LOGS_DIR/$i/`basename $LAST_LOG`.done
	fi
    fi
done

install-graphite-ubuntu-12.04.sh

####################################
# BASIC REQUIREMENTS
# http://graphite.wikidot.com/installation
# http://geek.michaelgrace.org/2011/09/how-to-install-graphite-on-ubuntu/
# Last tested & updated 10/13/2011
####################################

cd
sudo apt-get update
sudo apt-get upgrade

wget https://launchpad.net/graphite/0.9/0.9.10/+download/graphite-web-0.9.10.tar.gz
wget https://launchpad.net/graphite/0.9/0.9.10/+download/carbon-0.9.10.tar.gz
wget https://launchpad.net/graphite/0.9/0.9.10/+download/whisper-0.9.10.tar.gz
tar -zxvf graphite-web-0.9.10.tar.gz
tar -zxvf carbon-0.9.10.tar.gz
tar -zxvf whisper-0.9.10.tar.gz
mv graphite-web-0.9.10 graphite
mv carbon-0.9.10 carbon
mv whisper-0.9.10 whisper
rm graphite-web-0.9.10.tar.gz
rm carbon-0.9.10.tar.gz
rm whisper-0.9.10.tar.gz
sudo apt-get install --assume-yes apache2 apache2-mpm-worker apache2-utils apache2.2-bin apache2.2-common libapr1 libaprutil1 libaprutil1-dbd-sqlite3 build-essential python3.2 python-dev libpython3.2 python3-minimal libapache2-mod-wsgi libaprutil1-ldap memcached python-cairo-dev python-django python-ldap python-memcache python-pysqlite2 sqlite3 erlang-os-mon erlang-snmp rabbitmq-server bzr expect ssh libapache2-mod-python python-setuptools
sudo easy_install django-tagging

sudo easy_install zope.interface

sudo easy_install twisted

sudo easy_install txamqp

####################################
# INSTALL WHISPER
####################################

cd ~/whisper
sudo python setup.py install

####################################
# INSTALL CARBON
####################################

cd ~/carbon
sudo python setup.py install

cd /opt/graphite/conf
sudo cp carbon.conf.example carbon.conf
sudo cp storage-schemas.conf.example storage-schemas.conf
sudo gedit storage-schemas.conf
### Replace contents of storage-schemas.conf to be the following
[stats]
priority = 110
pattern = .*
retentions = 10:2160,60:10080,600:262974
###


####################################
# CONFIGURE GRAPHITE (webapp)
####################################

cd ~/graphite
sudo python check-dependencies.py
sudo python setup.py install

# CONFIGURE APACHE
###################
cd ~/graphite/examples
sudo cp example-graphite-vhost.conf /etc/apache2/sites-available/default
sudo cp /opt/graphite/conf/graphite.wsgi.example /opt/graphite/conf/graphite.wsgi
sudo mkdir /etc/httpd
sudo mkdir /etc/httpd/wsgi

sudo gedit /etc/apache2/sites-available/default
#####
# Change the line: WSGISocketPrefix run/wsgi
# To: WSGISocketPrefix /etc/httpd/wsgi
#####

sudo /etc/init.d/apache2 reload


####################################
# INITIAL DATABASE CREATION
####################################
cd /opt/graphite/webapp/graphite/
sudo python manage.py syncdb
# follow prompts to setup django admin user
sudo chown -R www-data:www-data /opt/graphite/storage/
sudo /etc/init.d/apache2 restart
cd /opt/graphite/webapp/graphite
sudo cp local_settings.py.example local_settings.py

####################################
# START CARBON
####################################
cd /opt/graphite/
sudo ./bin/carbon-cache.py start

####################################
# INSTALL STATSD
####################################
sudo aptitude install npm nodejs git
cd
git clone git://github.com/etsy/statsd.git
cp exampleConfig.js config.js
## Configure config.js
node stats.js config.js
   
####################################
# INSTALL STATSD CLIENT 
####################################
cd
git clone https://github.com/jsocol/pystatsd
cd pystatsd
sudo python setup.py install

####################################
# BASIC AUTH FOR APACHE 
# https://github.com/colinbjohnson/graphite-apache-auth
# working with the two files in this gist (auth.wsgi and sites-enabled-default)
####################################
sudo cp auth.wsgi /opt/graphite/conf/
sudo chmod +x /opt/graphite/conf/auth.wsgi
sudo cp sites-enabled-default /etc/apache2/sites-available/default
sudo /etc/init.d/apache2 restart

sites-enabled-default

# This needs to be in your server's config somewhere, probably
# the main httpd.conf
# NameVirtualHost *:80

# This line also needs to be in your server's config.
# LoadModule wsgi_module modules/mod_wsgi.so

# You need to manually edit this file to fit your needs.
# This configuration assumes the default installation prefix
# of /opt/graphite/, if you installed graphite somewhere else
# you will need to change all the occurances of /opt/graphite/
# in this file to your chosen install location.

<IfModule !wsgi_module.c>
    LoadModule wsgi_module modules/mod_wsgi.so
</IfModule>

# XXX You need to set this up!
# Read http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives#WSGISocketPrefix
WSGISocketPrefix /etc/httpd/wsgi

<VirtualHost *:80>
        ServerName graphite
        DocumentRoot "/opt/graphite/webapp"
        ErrorLog /opt/graphite/storage/log/webapp/error.log
        CustomLog /opt/graphite/storage/log/webapp/access.log common

        # I've found that an equal number of processes & threads tends
        # to show the best performance for Graphite (ymmv).
        WSGIDaemonProcess graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120
        WSGIProcessGroup graphite
        WSGIApplicationGroup %{GLOBAL}
        WSGIImportScript /opt/graphite/conf/graphite.wsgi process-group=graphite application-group=%{GLOBAL}

        # XXX You will need to create this file! There is a graphite.wsgi.example
        # file in this directory that you can safely use, just copy it to graphite.wgsi
        WSGIScriptAlias / /opt/graphite/conf/graphite.wsgi 

        Alias /content/ /opt/graphite/webapp/content/
        <Location "/content/">
                SetHandler None
        </Location>

        # XXX In order for the django admin site media to work you
        # must change @DJANGO_ROOT@ to be the path to your django
        # installation, which is probably something like:
        # /usr/lib/python2.6/site-packages/django
        Alias /media/ "@DJANGO_ROOT@/contrib/admin/media/"
        <Location "/media/">
                SetHandler None
        </Location>

        # The graphite.wsgi file has to be accessible by apache. It won't
        # be visible to clients because of the DocumentRoot though.
        <Directory /opt/graphite/conf/>
                Order deny,allow
                Allow from all
        </Directory>

	WSGIPassAuthorization On
	<Location />
		#AuthType Digest
		AuthType Basic
		AuthName "Graphite"
		AuthBasicProvider wsgi
		#AuthDigestProvider wsgi
		WSGIAuthUserScript /opt/graphite/conf/auth.wsgi
		Require valid-user
	</Location>

</VirtualHost>

sync_log.sh

#!/bin/bash
ssh metrics@metrics.internal.cartodb.com mkdir -p /home/metrics/pgbadger/logs/`hostname -s`/
rsync -av --progress /var/log/postgresql/postgresql-9.1-main.log metrics@metrics.internal.cartodb.com:/home/metrics/pgbadger/logs/`hostname -s`/`date +%Y%m%d%H`.log