Skip to content

Instantly share code, notes, and snippets.

@SeanHayes
Last active September 27, 2020 20:10
Show Gist options
  • Save SeanHayes/4958077 to your computer and use it in GitHub Desktop.
Save SeanHayes/4958077 to your computer and use it in GitHub Desktop.
Example Fabric fabfile.py. It's a hodgepodge of old scripts and probably isn't best practice (uWSGI ought to have an init script), but it should give you a practical idea of how to use Fabric. This fabfile relies heavily on custom settings from a Django project in order to be more DRY. Includes commands for setting up a fresh Ubuntu Server insta…
# -*- coding: utf-8 -*-
#Copyright (C) 2013 Seán Hayes
import my_project.settings as dj_settings
from fabric.api import local, run, sudo, env, prompt, settings, cd, parallel, execute
from fabric.contrib.files import exists
from fabric.decorators import hosts, roles, runs_once
import json
import logging
import os
logging.getLogger('').setLevel(logging.INFO)
logger = logging.getLogger(__name__)
env.forward_agent = True
env.user = 'username'
role_list = (
'web',
'cache',
'db',
'celery',
)
host_dict = {
'100.100.100.100': ('web', 'cache',),
'100.100.100.101': ('db', 'celery',),
}
env.hosts = host_dict.keys()
for r in role_list:
env.roledefs[r] = []
for k_host, v_roles in host_dict.items():
for v_role in v_roles:
env.roledefs[v_role].append(k_host)
env.code_dir = '/srv/'
env.package_name = dj_settings.PACKAGE_MODULE
env.project_dir = '%s%s/' % (env.code_dir, dj_settings.PROJECT_NAME,)
env.package_dir = '%s%s/' % (env.project_dir, dj_settings.PACKAGE_MODULE,)
env.project_git_uri = 'git@github.com:some-user/my-project.git'
env.config_dir = '%sconfig/generated/' % env.project_dir
env.log_dir = '%slogs/' % env.project_dir
env.pip_dir = '%spip/' % env.code_dir
env.celery_script_dir = '%scelery/init.d/' % env.config_dir
main_dirs = [
env.pip_dir,
]
project_dirs = [
env.log_dir,
]
apt_packages = [
'debconf-utils',
'git',
'mercurial',
'subversion',
'ntp',
'gdebi-core',
'graphviz',
'graphviz-dev',
'libmemcached-tools',
'memcached',
'nginx',
'pkg-config',
'postfix',
'python-pip',
'python-virtualenv',
'python-all-dev',
'postgresql',
#'rabbitmq-server',
#TODO: try to get as many of these as possible in requirements.txt
#'python-django-doc',
#some require 1/4 GB of dependencies to build the PIP version, which is unacceptable for this kind of application
'python-imaging',
'python-psycopg2',
#'python-exactimage',
#'python-crypto',
]
#Run the following to make binary eggs when setuptools isn't used
#python -c "import setuptools; execfile('setup.py')" bdist_egg
# tasks
def create_user():
"Create admin user on fresh cloud instance."
username = prompt('Enter username to create: ', default=env.user)
with settings(user='root'):
run('useradd --groups sudo,www-data -d /home/%s -m %s' % (username, username))
run('passwd %s' % username)
def switch_to_bash():
"switch from dash (the Ubuntu default) to bash"
with cd('/bin'):
#has to be one command since each call to sudo() is a different session,
#and you can't login if sh isn't set
sudo('rm sh; ln -s bash sh')
@roles('db')
@runs_once
def setup_pgsql():
"Sets up PostgreSQL user and databases."
name = prompt('Enter PostgreSQL role/db to create: ', default=env.project_name)
sudo('createuser -s -P %s' % name, user='postgres')
sudo('createdb -O %s %s' % (name, name), user='postgres')
def mkdirs(dirs):
"Sets up the directories we need and sets the right permissions."
for d in dirs:
if not exists(d):
sudo('mkdir %s' % d)
sudo('chown %s:www-data %s' % (env.user, d))
sudo('chmod 775 %s' % d)
def upgrade_ubuntu():
"Probably shouldn't run this through Fabric, but here's the commands for it anyway."
sudo('apt-get install update-manager-core')
#edit /etc/update-manager/release-upgrades, set Prompt=normal
sudo('do-release-upgrade')
@parallel
def install_apt():
"Updates package list, upgrades all packages to latest available version, and installs Apt dependencies for this project."
sudo('apt-get update')
sudo('apt-get upgrade')
sudo('apt-get install -f %s' % ' '.join(apt_packages))
@parallel
def set_permissions():
sudo('chown :www-data %s' % (env.code_dir,))
sudo('chmod 775 %s' % env.code_dir)
@parallel
def install_pip():
"Installs the PIP requirements for this project."
with cd(env.pip_dir):
sudo('pip install -r %srequirements.txt' % env.project_dir)
@parallel
def install_project():
"Clones this project's Git repo if there's no copy on the target machine, else it pulls the latest version."
if exists(env.project_dir):
with cd(env.project_dir):
run('git pull origin master')
else:
with cd(env.code_dir):
run('git clone %s' % env.project_git_uri)
sudo('chown -R %s:www-data %s' % (env.user, env.project_dir))
sudo('chmod 775 %s' % env.project_dir)
mkdirs(project_dirs)
@hosts('')
def install():
"Runs the commands to create all necessary directories, install Apt and PIP dependencies, and install project files."
execute(mkdirs, main_dirs)
execute(install_apt)
execute(set_permissions)
execute(install_project)
execute(install_pip)
@roles('db')
@runs_once
def migrate():
with cd(env.package_dir):
run('./manage.py syncdb --migrate')
@roles('web')
def collectstatic():
with cd(env.project_dir):
run('./manage.py collectstatic -l')
def refresh_config_files():
"Regenerates dynamic config files using django-config-gen."
with cd(env.package_dir):
run('./manage.py config_gen')
def link_config_file(source, destination):
with settings(warn_only=True):
sudo('rm %s' % destination)
sudo('ln -s %s %s' % (source, destination))
@roles('web')
def config_nginx():
with settings(warn_only=True):
sudo('rm /etc/nginx/sites-available/*')
link_config_file(os.path.join(env.config_dir, 'nginx'), '/etc/nginx/sites-available/default')
@roles('db')
def config_postgresql():
link_config_file(os.path.join(env.config_dir, 'pg_hba.conf'), '/etc/postgresql/9.1/main/pg_hba.conf')
@roles('cache')
def config_memcached():
link_config_file(os.path.join(env.config_dir, 'memcached.conf'), '/etc/memcached.conf')
@roles('celery')
def config_celery():
"Links Celery's Debian init scripts to /etc/init.d/."
init_list=run('ls %s' % env.celery_script_dir).split()
for script in init_list:
p = '/etc/init.d/%s' % script
init_file = os.path.join(env.celery_script_dir, script)
link_config_file(init_file, p)
sudo('chmod +x %s' % init_file)
link_config_file(os.path.join(env.config_dir, 'celery/celeryd_default'), '/etc/default/celeryd')
def config_tzdata():
"Configures the time zone for the server."
run('echo \'America/New_York\'| sudo tee /etc/timezone')
sudo('dpkg-reconfigure -f noninteractive tzdata')
@hosts('')
def config():
"Runs the commands to generate config files using django-config-gen and symlinks the generated files to the normal config file locations for Apache, Nginx, Memcached, etc."
execute(refresh_config_files)
execute(config_nginx)
execute(config_memcached)
execute(config_celery)
execute(config_tzdata)
@roles('web')
def reload_nginx():
sudo('/etc/init.d/nginx reload')
@roles('web')
def reload_uwsgi():
sudo('kill -HUP `cat %suwsgi.pid`' % env.project_dir)
@roles('celery')
def restart_celery():
sudo('/etc/init.d/celeryd restart')
sudo('/etc/init.d/celerybeat restart')
sudo('/etc/init.d/celeryevcam restart')
@roles('cache')
def restart_memcached():
sudo('/etc/init.d/memcached restart')
@hosts('')
def reload_servers():
"Reloads Apache, Nginx, Rabbit MQ, and Celery where possible, otherwise it restarts them. Reloading config files is faster than restarting the processes."
execute(reload_nginx)
execute(reload_uwsgi)
#sudo('/etc/init.d/rabbitmq-server reload')
execute(restart_celery)
execute(restart_memcached)
#local development scripts
def setup_git_shortcuts():
#https://git.wiki.kernel.org/index.php/Aliases
shortcuts = {
'st': 'status',
}
for shortcut in shortcuts.items():
local('git config --global alias.%s %s' % shortcut)
def set_django_colors():
local('export DJANGO_COLORS="%s"' % dj_settings.DJANGO_COLORS)
def check_for_pdb():
"Easily check for instances of pdb.set_trace() in your code before committing."
local('find . -name \'*.py\'|xargs grep \'pdb.set_trace\'')
#TODO: handle exit code
default_dump_file_template = '%s-dump.sql'
def dump_db():
name = prompt('Enter PostgreSQL db to dump: ', default=env.project_name)
local('sudo -u postgres pg_dump %s > %s' % (name, default_dump_file_template % name))
def load_db_dump():
name = prompt('Enter PostgreSQL db to load: ', default=env.project_name)
local('sudo -u postgres psql %s < %s' % (name, default_dump_file_template % name))
#Python imports
import os
import sys
#Django imports
import django.conf.global_settings as DEFAULT_SETTINGS
from django.utils import timezone
PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT = PACKAGE_PARENT_DIR = os.path.dirname(PACKAGE_ROOT)
PROJECT_NAME = PROJECT_ROOT.split(os.sep)[-1]
PACKAGE_MODULE = __name__[:__name__.rfind('.')] if '.' in __name__ else PACKAGE_ROOT.split(os.sep)[-1]
LOG_DIR = os.path.join(PROJECT_ROOT, 'logs')
_colors_dict = {
'error': ['red', 'bold'],
'notice': ['red'],
'http_info': ['cyan'],
'http_success': ['blue'],
'http_not_modified': ['cyan'],
'http_redirect': ['cyan'],
'http_not_found': ['red'],
'http_bad_request': ['red'],
'http_server_error': ['red'],
}
#needs to be exported as an environment variable
DJANGO_COLORS = ''.join([''.join([k, '=', ','.join(_colors_dict[k]), ';']) for k in _colors_dict])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment