Created
February 17, 2012 02:08
-
-
Save jravetch/1849832 to your computer and use it in GitHub Desktop.
My fabfile with shipper lib
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
import os, sys | |
from fabric.api import * | |
from fabric.colors import * | |
sys.path.insert( 0, os.path.abspath( os.path.join( os.path.realpath( os.path.curdir ), '../' ) ) ) | |
from lib.server_ip_map import ServerIpMap, ServerIpMapException | |
from lib.modenv_parser import ModEnvParser | |
sim = ServerIpMap('/srv/sys_restore/etc/server_ip_mappings.txt') | |
env.user = 'deployer' | |
env.roledefs = {'staging': sim.get_group('www-staging', 'external_ip'), | |
'live': sim.get_group('www-live', 'external_ip')} | |
env.roledefs['staging'].extend(env.roledefs['assets-staging']) | |
env.roledefs['live'].extend(env.roledefs['assets-live']) | |
env.roledefs['staging'].reverse() | |
env.roledefs['live'].reverse() | |
env.parallel = 'True' | |
#env.forward_agent = 'True' | |
def sshagent_run(cmd): | |
""" Helper function. | |
Runs a command with SSH agent forwarding enabled. | |
Note: Fabric (and paramiko) currently can't forward your SSH agent. | |
This helper uses your system's ssh to do so. | |
""" | |
local('ssh -A %s@%s "%s"' % (env.user, env.host, cmd)) | |
def get_env(): | |
""" This is really lame but Fabric doesn't tell you what role | |
it's currently running on. We must also compare with env.roles | |
because assets is actually in both staging and live. | |
""" | |
if env.host_string in env.roledefs['staging'] and 'staging' in env.roles: | |
return 'staging' | |
elif env.host_string in env.roledefs['live'] and 'live' in env.roles: | |
return 'live' | |
else: | |
return false | |
def clean_modules(mods): | |
""" converts and cleans modules list based on which instance we're on. """ | |
modules = [int(i) for i in mods.split(';')] | |
modenv = ModEnvParser(get_env()) | |
if 0 in modules: | |
# do all modules | |
modules = modenv.get_all_modids() | |
modules = modules.keys() | |
if env.host_string in env.roledefs['assets']: | |
# we handle assets (id 5) uniquely | |
modules = [x for x in modules if x == 5] | |
elif env.host_string not in env.roledefs['assets']: | |
modules = [x for x in modules if x != 5] | |
return modules | |
@task | |
def host_type(): | |
run('uname -a') | |
print "\n" | |
@task | |
def ship(mods='0', branch=''): | |
""" ships code for defined modules. | |
does not deploy though. | |
""" | |
if branch != '': | |
branch = ' -b '+branch | |
modules = clean_modules(mods) | |
if len(modules) < 1: | |
# don't run anything (wrong instance) | |
print blue("no modules to ship on this instance") | |
return | |
modenv = ModEnvParser(get_env()) | |
cmds = [] | |
for m in modules: | |
mod_script = modenv.get_ship_script(m) | |
cmds.append('./' + mod_script + ' -e ' + get_env() + branch) | |
full_cmd = ' && '.join(cmds) | |
cd_path = 'cd /srv/sys_restore/deployment/local/modules' | |
full_cmd = cd_path + ' && ' + full_cmd | |
print white(full_cmd) | |
# shipping needs key forwarding, so it can pull from github | |
sshagent_run(full_cmd) | |
# run(full_cmd) | |
@task | |
def deploy(mods='0'): | |
""" deploys latest code for defined modules. | |
order matters here (assets should deploy first), thought it seems | |
order is determined by the host list | |
""" | |
modules = clean_modules(mods) | |
if len(modules) < 1: | |
# don't run anything (wrong instance) | |
print blue("no modules to deploy on this instance") | |
return | |
modules = (',').join(map(str, modules)) # modules are ints; map to str first | |
cmd = './deploy-latest-release.py -e ' + get_env() + ' -m ' + modules | |
cd_path = 'cd /srv/sys_restore/deployment/local' | |
full_cmd = cd_path + ' && ' + cmd | |
print "CMD: " + white(full_cmd) | |
run(full_cmd) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pprint, re, os, sys, time, shlex, shutil, glob | |
from subprocess import PIPE, Popen | |
from datetime import datetime | |
from modenv_parser import ModEnvParser | |
class ModuleShipper(object): | |
def __init__(self, env = None, branch = None): | |
self.env = env | |
self.branch = branch | |
self.mop = ModEnvParser(self.env) | |
def ship(self, mod): | |
""" ship! (but don't deploy yet) """ | |
self.settings = self.mop.get_settings(mod) | |
if self.branch is None: | |
if 'default_branch' in self.settings: | |
self.branch = self.settings['default_branch'] | |
else: | |
self.branch = 'milestone_latest' | |
# a few simple sanity checks | |
if (not os.path.isdir(self.settings['target_path']) or | |
not os.path.isdir(self.settings['repository_path']) or | |
not os.path.isdir(self.settings['repository_path'] + '/' + self.settings['repository_dirname']) | |
): | |
print "Some path settings are wrong; is this the right environment?" | |
print "Halting shipment" | |
sys.exit(1) | |
print 'Shipping env: %s, module: %s, branch: %s' % (self.env, mod, self.branch) | |
dt = datetime(2000,01,01) | |
datetag = dt.today().strftime('%Y%m%d') | |
folder = self.branch+'-'+datetag | |
version = 1 | |
for item in os.listdir(self.settings['target_path']): | |
if re.match(folder, item): | |
print 'previous version found: %s' % item | |
version += 1 | |
if version < 10: | |
version = '0'+str(version) | |
else: | |
version = str(version) | |
pretag = folder+'.'+version | |
full_pretag_path = self.settings['target_path']+'/'+pretag | |
os.chdir(self.settings['repository_path']+'/'+self.settings['repository_dirname']) | |
if self.env != 'live': | |
''' If we're NOT shipping on production, we allow switching branches. | |
Here we check if current branch is legit (dirty files? etc.) and | |
switch branches accordingly if necessary ''' | |
p1 = Popen(['git', 'status'], stdout=PIPE) | |
p2 = Popen(['head', '-1'], stdin=p1.stdout, stdout=PIPE) | |
p1.stdout.close() | |
cur_branch = p2.communicate()[0].strip() | |
cur_branch = re.split(' ', cur_branch)[3] | |
if self.branch != cur_branch: | |
''' we need to switch branches ... | |
but only if working directory is clean, and branch exists ''' | |
print 'Switching from branch %s to %s' % (cur_branch, self.branch) | |
cmd = 'git status' | |
p1 = Popen(['git', 'status'], stdout=PIPE) | |
branch_clean = p1.communicate()[0] | |
branch_clean = re.split('\n', branch_clean, 3)[1] # get second line | |
branch_clean = re.split(' ', branch_clean, 3) | |
if len(branch_clean) > 3: | |
branch_clean = branch_clean[3] | |
else: | |
branch_clean = False | |
if branch_clean != '(working directory clean)': | |
print "Working directory not clean! Exiting ..." | |
return False | |
# current branch is clean, so we can change to another | |
p1 = Popen(['git', 'branch'], stdout=PIPE) | |
branches = p1.communicate()[0] | |
branches = re.split('\n', branches) | |
branch_exists = False | |
for b in branches: | |
''' current branch in list starts with '*', but we can ignore | |
that one since we're trying to switch branches anyways ''' | |
b = b.strip() | |
if b == self.branch: | |
branch_exists = True | |
if branch_exists: | |
p1 = Popen(['git', 'checkout', self.branch], stdout=PIPE) | |
p1.communicate() # wait for child process to return | |
else: | |
# branch not tracked; make sure it's at least remote ... | |
p1 = Popen(['git', 'branch', '-r'], stdout=PIPE) | |
branches = p1.communicate()[0] | |
branches = re.split('\n', branches) | |
branch_exists = False | |
for b in branches: | |
b = b.strip() | |
if b == 'origin/'+self.branch: | |
branch_exists = True | |
if branch_exists: | |
cmd = 'git checkout --track -b '+self.branch+' origin/'+self.branch | |
p1 = Popen(shlex.split(cmd), stdout=PIPE) | |
p1.communicate() # wait for child process to return | |
else: | |
print "I don't know branch '%s' ... exiting ..." % self.branch | |
return False | |
# we are ready to pull the repo now | |
p1 = Popen(['git', 'pull']) | |
p1.communicate() | |
# archive / "export" code and unpack into folder outside the repo | |
archive_file = pretag+'.zip' | |
cmd = 'git archive --format=zip --prefix='+pretag+'/ --output='+archive_file+' refs/heads/'+self.branch | |
# print cmd | |
print 'archiving for export ...' | |
p1 = Popen(shlex.split(cmd), stdout=PIPE) | |
p1.communicate() # waits for child process (archiving) to finish | |
shutil.move(archive_file, self.settings['target_path']) | |
os.chdir(self.settings['target_path']) | |
print 'archive moved; unzipping ...' | |
p1 = Popen(['unzip', archive_file], stdout=PIPE) | |
p1.communicate() # waits for child process (unzipping) to finish | |
os.unlink(archive_file) | |
p1 = Popen(['touch', pretag], stdout=PIPE) # folder timestamp sometimes must be updated | |
# set permissions stuff; sucks but shell way is much ... faster ... | |
os.chdir(self.settings['target_path']) | |
p1 = Popen(['sudo', 'chgrp', '-R', 'www-data', pretag]) | |
p1.communicate() # waits for child process to finish | |
p1 = Popen(['sudo', 'chmod', '-R', 'g+w', pretag]) | |
p1.communicate() # waits for child process to finish | |
self.pretag = pretag | |
# code 'shipped' but not deployed yet; caller should now setup symlinks, etc. | |
print 'Initial shipment complete ...' | |
return True | |
def get_settings(self): | |
return self.settings | |
def get_env(self): | |
return self.env | |
def get_pretag(self): | |
return self.pretag | |
def get_modid(self): | |
return self.settings['module_id'] | |
def get_shipped_tags(self, mod): | |
settings = self.mop.get_settings(mod) | |
shipped_regex = re.compile('(.*)-(\d+\.\d*)') | |
tags = filter(os.path.isdir, glob.glob(settings['target_path']+'/*')) # get shipped folders | |
tags = filter(shipped_regex.match, tags) # filter out api_logs, uploads, etc. | |
tags.sort(key=lambda x: os.path.getmtime(x)) | |
tags.reverse() | |
tags = [x.replace(settings['target_path']+'/', '') for x in tags] | |
return tags |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment