Fabric file to setup ceph quicky.

fabfile.py

"""
Deploy Ceph.
"""

import base64
import ConfigParser
import os
import StringIO
import struct
import time
import uuid

from fabric.api import (cd, settings, run, sudo, env, execute, roles,
                        put, runs_once, parallel, get)


# Populate these or use the target command below.
env.mon_hosts = ['diskbench0-0']
env.mon_ips = ['192.168.44.84']
env.osds = ['192.168.44.84', '192.168.44.85', '192.168.44.86', '192.168.44.87']
env.roledefs = {
    'mon': env.mon_ips,
    'osd': env.osds
}
env.hosts = list(set(env.mon_ips + env.osds))


# If your machines come online with drives mounted, you'll need to
# list the drives to be umount'd here or call the command `umount`
# with arguments.
UMOUNT = ['/dev/sdb1', '/dev/sdc1', '/dev/sdd1']


def osd(*osds):
    env.osds = osds
    env.roledefs['osd'].extend(osds)
    env.hosts.extend(osds)


def mon(*mons):
    env.mon_ips = mons
    env.hosts.extend(mons)
    env.roledefs['mon'].extend(mons)
    env.mon_hosts = execute(resolve_targets, hosts=mons).values()


@parallel
def resolve_targets():
    return run("uname -n")


@parallel
def install_cx_ceph(branch="master"):
    sudo("/bin/bash -c \"wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc' | sudo apt-key add -\"")
    sudo("/bin/bash -c \"echo deb http://gitbuilder.ceph.com/ceph-deb-$(lsb_release -sc)-armv7l-basic/ref/%s $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list\"" % branch)
    with settings(warn_only=True):
        sudo("apt-get -qq update")
    sudo("/bin/bash -c \"echo deb http://ceph.com/packages/google-perftools/debian  $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/google-perftools.list\"")
    with settings(warn_only=True):
        sudo("apt-get -qq update")
    sudo("apt-get -qq -y install google-perftools")
    with settings(warn_only=True):
        sudo("apt-get -qq update")
    sudo("apt-get -qq -y install ceph")


@runs_once
def config(fsid=None, key_string=None):
    cfg = ConfigParser.RawConfigParser()
    cfg.add_section("global")
    cfg.set("global", "fsid", str(fsid or uuid.uuid4()))
    cfg.set("global", "mon initial members", ", ".join(env.mon_hosts))
    cfg.set("global", "mon host", ", ".join(env.mon_ips))
    cfg.set("global", "auth supported", "cephx")
    cfg.set("global", "osd journal size", "1024")
    cfg.set("global", "filestore xattr use omap", "true")
    cfg_out = StringIO.StringIO()
    cfg.write(cfg_out)
    env.ceph_conf = cfg_out.getvalue()

    key = os.urandom(16)
    header = struct.pack('<hiih',
                         1,               # le16 type: CEPH_CRYPTO_AES
                         int(time.time()),  # le32 created: seconds
                         0,               # le32 created: nanoseconds,
                         len(key))        # le16: len(key)
    key_string = key_string or base64.b64encode(header + key)
    env.key = "[mon.]\nkey = %s\ncaps mon = allow *\n" % key_string

    with open("config_cache", "w") as f:
        f.write("fsid=%s\n" % fsid)
        f.write("key_string=%s\n" % key_string)

@parallel
def bootstrap():
    sudo("mkdir -p /etc/ceph")
    with cd("/etc/ceph"):
        put(StringIO.StringIO(env.ceph_conf), "ceph.conf", use_sudo=True)
        put(StringIO.StringIO(env.key), "ceph.mon.keyring", use_sudo=True)


@runs_once
def _require_bootstrap_osd_key():
    if 'bootstrap_osd' in env: return
    env.bootstrap_osd = sudo("cat /var/lib/ceph/bootstrap-osd/ceph.keyring") + "\n"


def umount(*disks):
    with settings(warn_only=True):
        for disk in disks:
            sudo("umount %s" % disk)


@roles('osd')
@parallel
def create_osd(disks=None):
    if not disks: return
    if UMOUNT:
        execute(umount, *UMOUNT)
    execute(_require_bootstrap_osd_key, hosts=env.roledefs['mon'][0])
    sudo("mkdir -p /var/lib/ceph/bootstrap-osd/")
    sudo("mkdir -p /var/lib/ceph/osd/")
    put(StringIO.StringIO(
        env.bootstrap_osd), "/var/lib/ceph/bootstrap-osd/ceph.keyring", use_sudo=True)

    for disk in disks.split(":"):
        sudo("ceph-disk-prepare --zap-disk -- %s" % disk)
        sudo("udevadm trigger --subsystem-match=block --action=add")
        sudo("ceph-disk-activate --mark-init upstart --mount %s" % disk)


@roles('mon')
def create_mon():
    sudo("mkdir -p /var/lib/ceph/mon/ceph-`uname -n`")
    sudo("ceph-mon --mkfs -i `uname -n` --keyring /etc/ceph/ceph.mon.keyring")
    sudo("initctl emit ceph-mon id=`uname -n`")
    time.sleep(10)  # HACK: just wait instead of actually looking for quorum.