example_fabfile.py

# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os

from fabric.api import cd, run, sudo, prefix, task, env, quiet, warn_only
from fabric.contrib.files import exists, sed
from fabric.tasks import Task

# The information to connecting to the server: `testserver` is contained
# in the local user's `~/.ssh/config` file
env.hosts = ['testserver']
env.use_ssh_config = True

env.project_name = 'app1'
env.site_root = "/home/webdev/apps/"
env.repo_url = "git@github.com:exampleorg/app1.git"
env.virtualenv_name = "virtualenv"
env.trusted_pypi_hosts = [
    'pypi.test.example.com'
]


def _make_link_cmd(source, link):
    """
    Generates the command string to make a link (if it doesn't exist)
    """
    return "ln -s %s %s" % (source, link)


class CreateTestInstance(Task):
    """
    Make a test instance using <branchname>, optionally calling it <name>
    """
    name = 'create'
    settings = 'settings.test'

    def run(self, branchname, name=""):
        self.env = env
        if not name:
            name = branchname
        self.branchname = branchname
        self.instance_name = name
        self.instance_dir = self.env.site_root + self.instance_name
        self.virtualenv_name = env.get('virtualenv_name', 'virtualenv')
        self.make_test_instance()

    def _make_systemd_config(self):
        """
        Make a systemd configuration from a template and copy it appropriately.
        Works idempotently. (Won't hurt anything if you run it multiple times)
        """
        systemd_conf_templ = os.path.join(self.instance_dir, 'conf', 'systemd-test.conf.template')
        systemd_conf = os.path.join(self.instance_dir, 'conf', 'systemd-test.conf')

        if not exists(systemd_conf):
            run('cp %s %s' % (systemd_conf_templ, systemd_conf))
            sed(systemd_conf, '\\{branchname\\}', self.instance_name)

        systemd_link = "/etc/systemd/system/%s.service" % self.instance_name
        if not exists(systemd_link):
            sudo("cp %s %s" % (systemd_conf, systemd_link), shell=False)

    def _make_settings(self):
        settings_template = os.path.join(self.instance_dir, 'settings', 'test.py.template')
        settings_path = os.path.join(self.instance_dir, 'settings', 'test.py')
        if not exists(settings_path):
            run('cp %s %s' % (settings_template, settings_path))
            sed(settings_path, '\\{branchname\\}', self.instance_name)

    def _make_web_config(self):
        """
        Make a web configuration from a template, link it to the web server configuration
        and enable it.
        Works idempotently. (Won't hurt anything if you run it multiple times)
        """
        web_config_templ = os.path.join(self.instance_dir, 'conf', 'nginx-test.conf.template')
        web_config = os.path.join(self.instance_dir, 'conf', 'nginx-test.conf')

        if not exists(web_config):
            run('cp %s %s' % (web_config_templ, web_config))
            sed(web_config, '\\{branchname\\}', self.instance_name)

        web_name = '/etc/nginx/sites-available/%s' % self.instance_name
        web_name_enabled = '/etc/nginx/sites-enabled/%s' % self.instance_name
        if not exists(web_name):
            sudo(_make_link_cmd(web_config, web_name), shell=False)
        if not exists(web_name_enabled):
            sudo(_make_link_cmd(web_config, web_name_enabled), shell=False)
        sudo('/etc/init.d/nginx reload', shell=False)

    def _checkout_repo(self):
        """
        Clone the repo into `instance_name` and checkout `branchname`.
        If the path to `instance_name` exists, checkout `branchname` and pull the
        latest changes down.
        Works idempotently. (Won't hurt anything if you run it multiple times)
        """
        run('mkdir -p %s' % self.env.site_root)
        if not exists(self.instance_dir):
            with cd(self.env.site_root):
                run('git clone %s %s' % (self.env.repo_url, self.instance_name))
            with cd(self.instance_dir):
                run('git checkout %s' % self.branchname)
        else:
            with cd(self.instance_dir):
                run('git checkout %s' % self.branchname)
                run("git pull")

    def _download_wheel(self, pkgname, directory):
        """
        Download the wheel for <pkgname> from PyPI to <directory>
        """
        import urllib2
        import json
        result = urllib2.urlopen("https://pypi.python.org/pypi/{0}/json".format(pkgname))
        result_json = json.loads(result.read())
        download_url = None
        for url in result_json['urls']:
            if url['packagetype'] == 'bdist_wheel':
                download_url = url['url']
                break

        with cd(directory):
            run('wget %s' % download_url)

    def _bootstrap(self):
        """
        Run the `bootstrap.py` to create the virtualenv and install requirements.
        Then setup the database and static media
        Works idempotently. (Won't hurt anything if you run it multiple times)
        """
        virtualenv = "%s/%s" % (self.instance_dir, self.virtualenv_name)
        venv_support_dir = "{0}/virtualenv_support".format(self.instance_dir)
        if not exists(venv_support_dir):
            run("mkdir -p {0}".format(venv_support_dir))
        with cd(venv_support_dir), quiet():
            has_pip = run("ls *pip*.whl").succeeded
            has_wheel = run("ls *wheel*.whl").succeeded
            has_setuptools = run("ls *setuptools*.whl").succeeded
        if not has_pip:
            self._download_wheel('pip', venv_support_dir)
        if not has_wheel:
            self._download_wheel('wheel', venv_support_dir)
        if not has_setuptools:
            self._download_wheel('setuptools', venv_support_dir)

        # Bootstrap the code
        with cd(self.instance_dir):
            if not exists(virtualenv):
                run("python bootstrap.py")
            with prefix('source %s/bin/activate' % virtualenv):
                run("pip install --upgrade pip")
                run("./manage.py migrate --noinput --settings %s" % self.settings)
                run("./manage.py collectstatic --noinput --verbosity 0 --settings %s" % self.settings)
                with warn_only():
                    run("./manage.py compress --force --verbosity 0 --settings %s" % self.settings)

        # place to put socket and pid files
        if not exists('/var/run/gunicorn'):
            sudo("mkdir /var/run/gunicorn")
            sudo("chown -R www-data:www-data /var/run/gunicorn")

        # place to put log files
        if not exists("/var/log/gunicorn"):
            sudo("mkdir /var/log/gunicorn")
            sudo("chown -R www-data:www-data /var/log/gunicorn")

    def make_test_instance(self):
        """
        Make a stand-alone instance with name of branchname on the test server
        """
        self._checkout_repo()
        self._make_settings()
        self._bootstrap()
        if self.use_systemd:
            self._make_systemd_config()
            sudo('sudo systemctl enable %s.service' % self.instance_name, shell=False)
            sudo('sudo systemctl start %s.service' % self.instance_name, shell=False)
        else:
            self._make_upstart_config()
            sudo('start %s' % self.instance_name, shell=False)
        self._make_web_config()

        sudo('chgrp -R www-data %s%s/staticmedia' % (self.env.site_root, self.instance_name), shell=False)
        sudo('chmod -R g+w %s%s/staticmedia' % (self.env.site_root, self.instance_name), shell=False)
        if not exists('media'):
            _make_link_cmd('/home/natgeo/media', 'media')

create = CreateTestInstance()


class RemoveTestInstance(Task):
    """
    Remove and clean all traces of test instance <name>
    """
    name = 'remove'

    def run(self, name):
        self.env = env
        self.instance_name = name
        self.instance_dir = self.env.site_root + self.instance_name
        self.virtualenv_name = env.get('virtualenv_name', 'virtualenv')
        self.remove_test_instance()

    def _clean_web_config(self):
        """
        Clean web configurations and reload the web server
        """
        web_name = '/etc/nginx/sites-available/%s' % self.instance_name
        web_name_enabled = '/etc/nginx/sites-enabled/%s' % self.instance_name
        if exists(web_name_enabled):
            sudo('rm %s' % web_name_enabled, shell=False)
            sudo('/etc/init.d/nginx reload', shell=False)
        if exists(web_name):
            sudo('rm %s' % web_name, shell=False)

    def _clean_systemd_config(self):
        """
        Clean systemd configurations
        """
        from fabric.api import settings

        systemd_link = "/etc/systemd/system/%s.service" % self.instance_name
        if exists(systemd_link):
            with settings(warn_only=True):
                sudo('service %s stop' % self.instance_name, shell=False)
                sudo('rm %s' % systemd_link, shell=False)

    def _clean_repo(self):
        """
        Clean the repo checkout
        """
        instance_dir = self.env.site_root + self.instance_name
        if exists(instance_dir):
            sudo('rm -Rf %s' % instance_dir, shell=False)

    def remove_test_instance(self):
        """
        Remove a test instance and remove all support scripts and configs
        """
        self._clean_web_config()
        self._clean_systemd_config()
        self._clean_repo()

remove = RemoveTestInstance()


class UpdateTestInstance(Task):
    """
    Pull down updates to test instance <name> and reload it
    """
    name = 'update'
    settings = 'settings.test'

    def run(self, name):
        self.env = env
        self.instance_name = name
        self.instance_dir = self.env.site_root + self.instance_name
        self.virtualenv_name = env.get('virtualenv_name', 'virtualenv')
        self.trusted_pypi_hosts = env.get('trusted_pypi_hosts', [])
        self.update_test_instance()

    def update_test_instance(self):
        """
        Pull down the latest stuff from the repo
        """
        with cd(self.instance_dir):
            run("git pull")

            with prefix('source %s/bin/activate' % self.virtualenv_name):
                trusted_hosts_args = ' '.join(['--trusted-host %s' % h for h in self.trusted_pypi_hosts])
                run("pip install -U pip")
                run("pip install %s -r requirements.txt" % trusted_hosts_args)
                run("./manage.py migrate --noinput --settings " + self.settings)
                run("./manage.py collectstatic --noinput --verbosity 0 --settings %s" % self.settings)
                with warn_only():
                    run("./manage.py compress --force --verbosity 0 --settings %s" % self.settings)
        sudo("service %s reload" % self.instance_name, shell=False)

update = UpdateTestInstance()


class StopTestInstance(Task):
    """
    Stop test instance <name> or all test instances if no name is given
    """
    name = 'stop'

    def run(self, name=""):
        env.warn_only = True
        if name:
            instances = [name]
        else:
            output = run('ls -1 %s' % env.site_root)
            instances = [x.strip() for x in output.split("\n")]
        for item in instances:
            sudo("service %s stop" % item.strip(), shell=False)

stop = StopTestInstance()


class StartTestInstance(Task):
    """
    Start test instance <name> or all test instances if no name is given
    """
    name = 'start'

    def run(self, name=""):
        env.warn_only = True
        if name:
            instances = [name]
        else:
            output = run('ls -1 %s' % env.site_root)
            instances = [x.strip() for x in output.split("\n")]
        for item in instances:
            sudo("service %s start" % item.strip(), shell=False)

start = StartTestInstance()


class ListTestInstance(Task):
    """
    List all test instances
    """
    name = 'list'

    def run(self):
        env.warn_only = True
        output = run('ls -1 %s' % env.site_root)
        instances = [x.strip() for x in output.split("\n")]

        for item in instances:
            print item

list_ = ListTestInstance()  # Fabric doesn't care about the name


@task
def reload(test_name):
    """
    Reload the <test_name> deployment on the test server
    """
    sudo("reload %s" % test_name, shell=False)