Scrapy Production/Worker/Dev Setup Fabfile

gistfile1.py

"""

Git Repo:
    branches
        dev
        worker
        production
        master

dev
    Platform: Mac Os X

worker
    Platform: Debian

production
    Platfolor: Debian

"""
from __future__ import with_statement
import os, platform, hashlib, ConfigParser
from fabric.api import local, run, sudo, env, require, put, get
from fabric.contrib.files import exists
from fabric.context_managers import cd
from fabric.contrib.console import confirm
from datetime import datetime

# globals

env.project = 'vacation'
env.packager = {
    'Debian': {
        'template':'apt-get install -y %s',
        'easy_install':('python-setuptools', 'easy_install'),
        'virtualenv':('python-virtualenv', 'virtualenv'),
        'pip':('python-pip', 'pip'),
        'libxml2':('python-libxml2', None),
        'pyopenssl':('python-openssl', None),
        'twisted':('python-twisted', None),
        'mercurial':('mercurial', 'hg')
        },
    'Darwin': {
        'template':'port install %s',
        'easy_install':('py25-setuptools', 'easy_install-2.5'),
        'virtualenv':('py25-virtualenv', 'virtualenv-2.5'),
        'pip':('py25-pip', 'pip-2.5'),
        'libxml2':('py25-libxml2', None),
        'pyopenssl':('py25-openssl', None),
        'twisted':('py25-twisted', None),
        'mercurial':('mercurial', 'hg')
    }
        
}
# environments

# scrapy 0.7 is still under development so we'll use a revision that we trust
env.scrapy_revision = 1627
env.scrapy = 'hg+http://hg.scrapy.org/scrapy/@%s#egg=scrapy' % env.scrapy_revision

def dev():
    "Configuration for development environment"
    env.role = 'dev'
    env.hosts = ['127.0.0.1']
    env.user = os.getlogin()
    env.path = os.path.join('Users', env.user, 'Sites')
    env.admin_host = '127.0.0.1:9010'
    env.platform = 'Darwin'
    
def worker():
    "Configuration for worker"
    env.role = 'worker'
    env.hosts = ['hstd.org']
    env.path = '/var/sites'
    env.admin_host = '127.0.0.1:9010'
    env.platform = 'Debian'

def production():
    "Configuration for production"
    env.role = 'production'
    env.hosts = ['beecoop.com']
    env.path = '/var/sites'
    env.admin_host = '127.0.0.1:9010'
    env.platform = 'Debian'

# tasks

def setup():
    """
    Setup a fresh virtualenv as well as a few useful directories, then run
    a full deployment
    """
    
    def install(app):
        "Return install command for current platform"
        require('platform')
        require('packager')
        return env.packager[env.platform]['template'] % env.packager[env.platform][app][0]
    
    def executable(app):
        "Return name of executable for current platform"
        require('platform')
        require('packager')
        return env.packager[env.platform][app][1]
    
    def get_hgrc():
        "Downloads copy of hgrc and returns path to it"
        random = hashlib.sha224(str(datetime.now())).hexdigest()
        tmp = '/tmp/%s'%random
        get('~/.hgrc', tmp)
        return tmp
    
    def hgfetch_enabled():
        "Checks if hg fetch is enabled"
        if exists('~/.hgrc'):
            tmp = get_hgrc()
            hgrc = ConfigParser.ConfigParser()
            hgrc.read(tmp)
            if hgrc.has_option('extensions', 'fetch'):
                return hgrc.has_option('extensions', 'fetch')
        return False
    
    def hgfetch_enable():
        "Enable hg fetch"
        hgrc = ConfigParser.ConfigParser()
        if exists('~/.hgrc'):
            tmp = get_hgrc()
            hgrc.read(tmp)
            if not hgrc.has_section('extensions'):
                hgrc.add_section('extensions')
        else:
            random = hashlib.sha224(str(datetime.now())).hexdigest()
            tmp = '/tmp/%s'%random
            hgrc.add_section('extensions')
        hgrc.set('extensions', 'fetch', '')
        file = open(tmp, 'w+')
        hgrc.write(file)
        file.close()
        put(tmp, '~/.hgrc')
    
    def test():
        "Test to make sure that the modules installed correctly."
        run("python -c 'import libxml2'")
        run("python -c 'import twisted'")        
        run("python -c 'import scrapy'")
    
    require('project')
    require('path')
    require('packager')
    require('user')
    require('role')
    require('platform')
    require('scrapy')
    
    if env.role == 'dev':
        # project rezides in user's directory so we dont' need sudo
        cmd = local
    else:
        # project rezides in system files, so we need sudo
        cmd = sudo
    
    # create project directory container if it doesn't exist
    if not exists(env.path):
        cmd('mkdir -p %s'%env.path)

    sudo(install('easy_install'))
    sudo(install('virtualenv'))
    sudo(install('pip'))
    
    # create virtual environment
    with cd(env.path):
        cmd('%s %s' % (executable('virtualenv'), env.project))
    
    projectdir = os.path.join(env.path, env.project)
    
    # install django
    cmd('%s -E %s install -U %s' % (executable('pip'), projectdir, 'django'))
        
    sudo(install('twisted'))
    sudo(install('libxml2'))
    sudo(install('pyopenssl'))
    
    sudo(install('mercurial'))

    enabled = hgfetch_enabled()
    if not enabled:
        print "To use this script you must enable hg fetch."
        if confirm("Would you like me to enable hg fetch for you?", default=True):
            hgfetch_enable()
            enabled = True
        else:
            print 'Setup is NOT complete, because hg fetch must be enabled.'
            print 'Pease, enable hg fetch and run setup again.'
    
    if enabled:
        # install scrapy from hg trunk
        cmd('%s -E %s install -e %s' % (executable('pip'), projectdir, env.scrapy))
    
    test()