fabric deployment example

0. fabric

[assuming python is installed]
sudo pip install fabric
sudo pip install pyyaml

1. fabfile.py

import copy
import functools
import itertools
import tempfile
import time
import os.path
import yaml
import urllib

from fabric.api import task, run, env, local, put, get
from fabric.decorators import runs_once, with_settings, serial
from fabric.context_managers import cd, lcd
from fabric.utils import abort

import fabric.colors as colors
import fabric.contrib.console as console
import fabric.contrib.files

#----------- INIT --------------#

with open('conf/app.yaml', 'r') as f:
    c = yaml.load(f.read())
    appname = c['appname']
    base_app_config = c['base-app-config']
    release_repo = c['release-repo']
    config = c['envs']
for clustername in config:
    env.roledefs[clustername] = config[clustername]['hosts']

env.user = 'u'
env.disable_known_hosts = True

#----------- HELPER FUNCTIONS --------------#

def merge_config(user, default):
    if isinstance(user,dict) and isinstance(default,dict):
        for k,v in default.iteritems():
            if k not in user:
                user[k] = v
            else:
                user[k] = merge_config(user[k],v)
    return user

def get_current_cluster():
    for role in env.roledefs.keys():
        if env.host_string in env.roledefs[role]:
            return role
    return None

def _print(msg):
    print("[%s] %s" % (env.host, msg))

#----------- TASKS --------------#

@task
def copy_tarball(jarfile=None):
    remote_dir = '~/%s/jar/' % appname
    rfile = put(jarfile, remote_dir + jarfile)
    if rfile.failed:
        abort(colors.red('failed to put jarfile'))

@task
def deploy_config():
    """ deploy latest config """
    cluster = get_current_cluster()
    with tempfile.NamedTemporaryFile() as tmpf:
        new_conf = merge_config(config[cluster]['config-override'], base_app_config)
        _print(colors.green("resulting config:"))
        _print(yaml.dump(new_conf, default_flow_style=False))
        tmpf.write(yaml.dump(new_conf, default_flow_style=False))
        tmpf.flush()
        put(tmpf.name, '~/%s/conf/%s.yaml' % (appname, cluster))

@task
def update_symlink(jarfile=None):
    """ update the symlink to the latest version in the config """
    remotejar = '~/%s/jar/%s' % (appname, jarfile)
    symlink = '~/%s/jar/%s.jar' % (appname, appname)
    if not fabric.contrib.files.exists(remotejar):
        abort(colors.red(('target link dir %s does not exist!' % remotejar)))
    run('ln -sfn %s %s' % (remotejar, symlink))

@task
@serial
def download_jar(version):
    repo_jar = release_repo.replace("VER", version)
    lastslash = repo_jar.rfind("/")
    jarfile = repo_jar[lastslash+1:]
    if os.path.exists(jarfile):
        print("%s exists locally, no need to download" % jarfile)
    else:
        print("Downloading from %s..." % repo_jar)
        urllib.urlretrieve(repo_jar, jarfile)
    return jarfile
    
@task
def deploy_new_version(version=None, jarfile=None, symlink=False):
    """ Deploy a new version, but do not activate. """
    if version:
        jarfile = download_jar(version)

    remotejar = '~/%s/jar/%s' % (appname, jarfile)
    if fabric.contrib.files.exists(remotejar):
        _print(colors.red('dir %s exists, this version has been deployed already (and could be running)' %
                         (remotejar)))
        if not console.confirm('DANGER Continue anyway?', default=False):
            return 1

    copy_tarball(jarfile=jarfile)
    if symlink:
        update_symlink(jarfile=jarfile)

@task
def rolling_restart(force=False, max_wait=45, symlink=False, jarfile=None):
    """ take out of lb, restart, put back in """
    live_lb_off()
    stop_service()
    time.sleep(30)
    if symlink:
        update_symlink(jarfile=jarfile)
    start_service()
    live_lb_on()

def switch_lb_files(new, old):
    with cd('~/%s' % appname):
        if fabric.contrib.files.exists(new):
            _print('%s is already there' % new)
            if fabric.contrib.files.exists(old):
                _print('... but %s was found, removing' % old)
                run('rm %s' % old)
        else:
            run('echo "cool" > %s' % new)
            run('rm %s' % old)

@task
def live_lb_on():
    switch_lb_files('live-lb', 'live-lb-off')
    _print('waiting for 30 secs for live-lb to take effect...')
    time.sleep(30)

@task
def live_lb_off():
    switch_lb_files('live-lb-off', 'live-lb')
    _print('waiting for 30 secs for live-lb-off to take effect...')
    time.sleep(30)

@task
def tail_log(n='50'):
    run('tail -%s %s/log/%s.log' % (n, appname, appname))

2. app.yaml

appname: myapp
release-repo: http://buildbox/nexus/content/repositories/releases/myapp/VER/myapp-VER.jar
envs:
  myapp-dev:
    hosts: ['dev1']
    config-override:
      logging:
        level: DEBUG
      zookeeperConnection: zkd1:2181/d1
  myapp-prod:
    hosts: ['prod1','prod1']
    config-override:
      zookeeperConnection: zkp1:2181,zkp2:2181/p1
base-app-config:
  http:
    port: 8013
    adminPort: 8113
    minThreads: 8
    maxThreads: 1024
    requestLog:
      console:
        enabled: true
      file:
        enabled: true
        currentLogFilename: /home/prod/myapp/log/requests.log
        archivedLogFilenamePattern: /home/prod/myapp/log/requests-%d.log.gz
        timeZone: EST
        archivedFileCount: 10
    gzip:
      enabled: true
      minimumEntitySize: 256 bytes
      bufferSize: 8KiB
      excludedUserAgents: []
      compressedMimeTypes: []
  logging:
    level: INFO
    file:
      enabled: true
      timeZone: EST 
      currentLogFilename: log/myapp.log
      archivedLogFilenamePattern: log/myapp-%d.log.gz
  reporterConfig:
    ganglia:
      -
        groupPrefix: "myapp"
        period: 60
        timeunit: 'SECONDS'
        hosts:
          - host: 'localhost'
            port: 8649
  zookeeperConnection: REPLACE_ME

3. bash command

fab -R myapp-prod deploy_new_version:version=1.2,symlink=true rolling_restart