My fabfile with shipper lib

fabfile.py

# -*- coding: utf-8 -*-

import os, sys
from fabric.api import *
from fabric.colors import *

sys.path.insert( 0, os.path.abspath( os.path.join( os.path.realpath( os.path.curdir ), '../' ) ) )
from lib.server_ip_map import ServerIpMap, ServerIpMapException
from lib.modenv_parser import ModEnvParser

sim = ServerIpMap('/srv/sys_restore/etc/server_ip_mappings.txt')
env.user = 'deployer'
env.roledefs = {'staging': sim.get_group('www-staging', 'external_ip'), 
                'live': sim.get_group('www-live', 'external_ip')}

env.roledefs['staging'].extend(env.roledefs['assets-staging'])
env.roledefs['live'].extend(env.roledefs['assets-live'])

env.roledefs['staging'].reverse()
env.roledefs['live'].reverse()

env.parallel = 'True'
#env.forward_agent = 'True'

def sshagent_run(cmd):
    """ Helper function.
    Runs a command with SSH agent forwarding enabled.

    Note: Fabric (and paramiko) currently can't forward your SSH agent.
    This helper uses your system's ssh to do so.
    """

    local('ssh -A %s@%s "%s"' % (env.user, env.host, cmd))

def get_env():
    """ This is really lame but Fabric doesn't tell you what role
        it's currently running on. We must also compare with env.roles
        because assets is actually in both staging and live.
    """

    if env.host_string in env.roledefs['staging'] and 'staging' in env.roles:
        return 'staging'
    elif env.host_string in env.roledefs['live'] and 'live' in env.roles:
        return 'live'
    else:
        return false

def clean_modules(mods):
    """ converts and cleans modules list based on which instance we're on. """
        
    modules = [int(i) for i in mods.split(';')]
    modenv = ModEnvParser(get_env())

    if 0 in modules:
        # do all modules
        modules = modenv.get_all_modids()
        modules = modules.keys()

    if env.host_string in env.roledefs['assets']:
        # we handle assets (id 5) uniquely
        modules = [x for x in modules if x == 5]
    elif env.host_string not in env.roledefs['assets']:
        modules = [x for x in modules if x != 5]

    return modules

@task
def host_type():
    run('uname -a')
    print "\n"

@task
def ship(mods='0', branch=''):
    """ ships code for defined modules.
        does not deploy though.
    """

    if branch != '':
        branch = ' -b '+branch

    modules = clean_modules(mods)
    if len(modules) < 1:
        # don't run anything (wrong instance)
        print blue("no modules to ship on this instance")
        return

    modenv = ModEnvParser(get_env())
    cmds = []
    for m in modules:
        mod_script = modenv.get_ship_script(m)
        cmds.append('./' + mod_script + ' -e ' + get_env() + branch)

    full_cmd = ' && '.join(cmds)
    cd_path = 'cd /srv/sys_restore/deployment/local/modules'
    full_cmd = cd_path + ' && ' + full_cmd
    print white(full_cmd)

    # shipping needs key forwarding, so it can pull from github
    sshagent_run(full_cmd)
#    run(full_cmd)


@task
def deploy(mods='0'):
    """ deploys latest code for defined modules. 
        order matters here (assets should deploy first), thought it seems
        order is determined by the host list
    """

    modules = clean_modules(mods)

    if len(modules) < 1:
        # don't run anything (wrong instance)
        print blue("no modules to deploy on this instance")
        return

    modules = (',').join(map(str, modules)) # modules are ints; map to str first

    cmd = './deploy-latest-release.py -e ' + get_env() + ' -m ' + modules
    cd_path = 'cd /srv/sys_restore/deployment/local'
    full_cmd = cd_path + ' && ' + cmd
    print "CMD: " + white(full_cmd)
    run(full_cmd)

shipper.py

import pprint, re, os, sys, time, shlex, shutil, glob
from subprocess import PIPE, Popen
from datetime import datetime

from modenv_parser import ModEnvParser

class ModuleShipper(object):

    def __init__(self, env = None, branch = None):
        self.env = env
        self.branch = branch

        self.mop = ModEnvParser(self.env)


    def ship(self, mod):
        """ ship! (but don't deploy yet) """
        self.settings = self.mop.get_settings(mod)

        if self.branch is None:
            if 'default_branch' in self.settings:
                self.branch = self.settings['default_branch']
            else:
                self.branch = 'milestone_latest'

        # a few simple sanity checks
        if (not os.path.isdir(self.settings['target_path']) or
            not os.path.isdir(self.settings['repository_path']) or
            not os.path.isdir(self.settings['repository_path'] + '/' + self.settings['repository_dirname'])
           ):
            print "Some path settings are wrong; is this the right environment?"
            print "Halting shipment"
            sys.exit(1)

        print 'Shipping env: %s, module: %s, branch: %s' % (self.env, mod, self.branch)

        dt = datetime(2000,01,01)
        datetag = dt.today().strftime('%Y%m%d')
        folder = self.branch+'-'+datetag

        version = 1
        for item in os.listdir(self.settings['target_path']):
            if re.match(folder, item):
                print 'previous version found: %s' % item
                version += 1

        if version < 10:
            version = '0'+str(version)
        else:
            version = str(version)

        pretag = folder+'.'+version
        full_pretag_path = self.settings['target_path']+'/'+pretag

        os.chdir(self.settings['repository_path']+'/'+self.settings['repository_dirname'])

        if self.env != 'live':
            ''' If we're NOT shipping on production, we allow switching branches.
                Here we check if current branch is legit (dirty files? etc.) and
                switch branches accordingly if necessary '''

            p1 = Popen(['git', 'status'], stdout=PIPE)
            p2 = Popen(['head', '-1'], stdin=p1.stdout, stdout=PIPE)
            p1.stdout.close()
            cur_branch = p2.communicate()[0].strip()
            cur_branch = re.split(' ', cur_branch)[3]

            if self.branch != cur_branch:
                ''' we need to switch branches ...
                    but only if working directory is clean, and branch exists '''

                print 'Switching from branch %s to %s' % (cur_branch, self.branch)
                cmd = 'git status'
                p1 = Popen(['git', 'status'], stdout=PIPE)
                branch_clean = p1.communicate()[0]
                branch_clean = re.split('\n', branch_clean, 3)[1] # get second line
                branch_clean = re.split(' ', branch_clean, 3)
                if len(branch_clean) > 3:
                    branch_clean = branch_clean[3]
                else:
                    branch_clean = False

                if branch_clean != '(working directory clean)':
                    print "Working directory not clean! Exiting ..."
                    return False

                # current branch is clean, so we can change to another
                p1 = Popen(['git', 'branch'], stdout=PIPE)
                branches = p1.communicate()[0]
                branches = re.split('\n', branches)

                branch_exists = False
                for b in branches:
                    ''' current branch in list starts with '*', but we can ignore
                        that one since we're trying to switch branches anyways '''
                    b = b.strip()
                    if b == self.branch:
                        branch_exists = True

                if branch_exists:
                    p1 = Popen(['git', 'checkout', self.branch], stdout=PIPE)
                    p1.communicate() # wait for child process to return
                else:
                    # branch not tracked; make sure it's at least remote ...
                    p1 = Popen(['git', 'branch', '-r'], stdout=PIPE)
                    branches = p1.communicate()[0]
                    branches = re.split('\n', branches)

                    branch_exists = False
                    for b in branches:
                        b = b.strip()
                        if b == 'origin/'+self.branch:
                            branch_exists = True

                    if branch_exists:
                        cmd = 'git checkout --track -b '+self.branch+' origin/'+self.branch
                        p1 = Popen(shlex.split(cmd), stdout=PIPE)
                        p1.communicate() # wait for child process to return
                    else:
                        print "I don't know branch '%s' ... exiting ..." % self.branch
                        return False


        # we are ready to pull the repo now
        p1 = Popen(['git', 'pull'])
        p1.communicate()

        # archive / "export" code and unpack into folder outside the repo
        archive_file = pretag+'.zip'
        cmd = 'git archive --format=zip --prefix='+pretag+'/ --output='+archive_file+' refs/heads/'+self.branch
        # print cmd
        print 'archiving for export ...'
        p1 = Popen(shlex.split(cmd), stdout=PIPE)
        p1.communicate() # waits for child process (archiving) to finish

        shutil.move(archive_file, self.settings['target_path'])
        os.chdir(self.settings['target_path'])
        print 'archive moved; unzipping ...'
        p1 = Popen(['unzip', archive_file], stdout=PIPE)
        p1.communicate() # waits for child process (unzipping) to finish
        os.unlink(archive_file)
        p1 = Popen(['touch', pretag], stdout=PIPE) # folder timestamp sometimes must be updated

        # set permissions stuff; sucks but shell way is much ... faster ...
        os.chdir(self.settings['target_path'])
        p1 = Popen(['sudo', 'chgrp', '-R', 'www-data', pretag])
        p1.communicate() # waits for child process to finish
        p1 = Popen(['sudo', 'chmod', '-R', 'g+w', pretag])
        p1.communicate() # waits for child process to finish

        self.pretag = pretag

        # code 'shipped' but not deployed yet; caller should now setup symlinks, etc.
        print 'Initial shipment complete ...'
        return True


    def get_settings(self):
        return self.settings

    def get_env(self):
        return self.env

    def get_pretag(self):
        return self.pretag

    def get_modid(self):
        return self.settings['module_id']

    def get_shipped_tags(self, mod):
        settings = self.mop.get_settings(mod)
        shipped_regex = re.compile('(.*)-(\d+\.\d*)')
        tags = filter(os.path.isdir, glob.glob(settings['target_path']+'/*')) # get shipped folders
        tags = filter(shipped_regex.match, tags) # filter out api_logs, uploads, etc.
        tags.sort(key=lambda x: os.path.getmtime(x))
        tags.reverse()
        tags = [x.replace(settings['target_path']+'/', '') for x in tags] 
        return tags