Skip to content

Instantly share code, notes, and snippets.

@trilopin
Created November 20, 2015 07:15
Show Gist options
  • Save trilopin/8e305575f50bb0c6396f to your computer and use it in GitHub Desktop.
Save trilopin/8e305575f50bb0c6396f to your computer and use it in GitHub Desktop.
fabfile with google compute engine (GCE) support + releases + virtualenv + supervisor
from fabric.api import env, run, sudo, local, runs_once
# Uncomment this two lines if you need debug mode
# import paramiko
# paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG)
# gce conf
GCE_PROJECT = 'my-project'
GCE_ZONE = 'europe-west1-b'
GCE_HOSTMATCH = 'frontend.*'
GCE_GS_URL = 'gs://bucket/dir/'
# vcs conf
VCS_REPOSITORY = 'repository-url'
VCS_BRANCH = None # None for master, otherwise branch name
# remote conf
REMOTE_BASE_DIR = '/opt/hosting/'
def APP_DOMAIN():
"""Preconf APP_DOMAIN to do action
Deploy with `fab APP_DOMAIN deploy`
"""
env.domain_name = "APP_DOMAIN"
gcloud()
env.hosts = gcloud_hosts(GCE_PROJECT, GCE_ZONE, GCE_HOSTMATCH)
def gcloud_hosts(project, zone, match):
"""Retrieve from google sdk tool the hosts we want to deploy code
Extracts all public ip addresses from hosts in project/zone
and name matching pattern (hosts in group should have groupname in hostname)
"""
hosts = []
data = {'project': project, 'zone': zone, 'match': match}
gcloud = local('gcloud --project "%(project)s" compute instances list --zone "%(zone)s" -r "%(match)s"' % data, capture=True)
for line in gcloud.split('\n')[1:]:
items = filter(lambda x: x, line.split(' '))
hosts.append(items[-2])
return hosts
@runs_once
def upload_to_gs():
"""After successful deploy, upload a tarball to Google Storage
Google Apis full access is needed in VM instances (write in google storage
at least). This process is really useful for autoscaling -> new hosts take
packages from google storage (instead of master or branch).
"""
sudo("cd %(releases_path)s/%(current_revision)s/ && tar czf /tmp/%(domain_name)s.tgz *" % {'releases_path': env.releases_path, 'current_revision': env.current_revision, 'domain_name': env.domain_name})
sudo("gsutil cp /tmp/%(domain_name)s.tgz %(gs_url)s" % {'domain_name': env.domain_name, 'gs_url': GCE_GS_URL})
def gcloud():
"""Defines google compute engine environment. Deploys will be done with
current user and current gcloud sdk auth.
"""
env.user = local('whoami', capture=True)
env.password = ''
env.key_filename = '~/.ssh/google_compute_engine'
env.base_dir = REMOTE_BASE_DIR
env.domain_path = "%(base_dir)s/%(domain_name)s" % {'base_dir': env.base_dir, 'domain_name': env.domain_name}
env.current_path = "%(domain_path)s/current" % {'domain_path': env.domain_path}
env.releases_path = "%(domain_path)s/releases" % {'domain_path': env.domain_path}
env.shared_path = "%(domain_path)s/shared" % {'domain_path': env.domain_path}
env.git_clone = VCS_REPOSITORY
env.env_file = "requirements.txt"
env.branch = VCS_BRANCH
def releases():
"""List a releases made"""
env.releases = sorted(run('ls -x %(releases_path)s' % {'releases_path': env.releases_path}).split())
if len(env.releases) >= 1:
env.current_revision = env.releases[-1]
env.current_release = "%(releases_path)s/%(current_revision)s" % {'releases_path': env.releases_path, 'current_revision': env.current_revision}
if len(env.releases) > 1:
env.previous_revision = env.releases[-2]
env.previous_release = "%(releases_path)s/%(previous_revision)s" % {'releases_path': env.releases_path, 'previous_revision': env.previous_revision}
def start():
"""Start the application servers"""
sudo("supervisorctl start %(domain_name)s:*" % {'domain_name': env.domain_name}, shell=False)
def restart():
"""Restarts your application"""
sudo("supervisorctl restart %(domain_name)s:*" % {'domain_name': env.domain_name}, shell=False)
def stop():
"""Stop the application servers"""
sudo("supervisorctl stop %(domain_name)s:*" % {'domain_name': env.domain_name}, shell=False)
def permissions():
"""Make the release group-writable"""
# sudo("chmod -R g+w %(domain_path)s" % { 'domain_path':env.domain_path }, shell=False)
# sudo("chown deploy:deploy -R %(domain_path)s" % {'domain_path': env.domain_path}, shell=False)
def setup():
"""Prepares one or more servers for deployment"""
run("mkdir -p %(domain_path)s/{releases,shared}" % {'domain_path': env.domain_path})
run("mkdir -p %(shared_path)s/log" % {'shared_path': env.shared_path})
permissions()
def checkout():
"""Checkout code to the remote servers"""
from time import time
env.current_release = "%(releases_path)s/%(time).0f" % {'releases_path': env.releases_path, 'time': time()}
if env.branch:
run("cd %(releases_path)s; git clone -q %(git_clone)s %(current_release)s" % {'releases_path': env.releases_path, 'git_clone': env.git_clone, 'current_release': env.current_release})
run("cd %(current_release)s && git checkout %(branch)s && cd .." % {'current_release': env.current_release, 'branch': env.branch})
else:
run("cd %(releases_path)s; git clone -q -o deploy --depth 1 %(git_clone)s %(current_release)s" % {'releases_path': env.releases_path, 'git_clone': env.git_clone, 'current_release': env.current_release})
run("rm -rf %(current_release)s/.git/" % {'current_release': env.current_release})
run("cp %(current_release)s/conf/settings-%(file)s.py %(current_release)s/settings.py" % {'current_release': env.current_release, 'file': env.domain_name})
def update():
"""Copies your project and updates environment and symlink"""
update_code()
update_env()
symlink()
permissions()
def update_code():
"""Copies your project to the remote servers"""
checkout()
permissions()
def symlink():
"""Updates the symlink to the most recently deployed version"""
if 'current_release' not in env:
releases()
sudo("ln -nfs %(current_release)s %(current_path)s" % {'current_release': env.current_release, 'current_path': env.current_path})
sudo("ln -nfs %(shared_path)s/log %(current_release)s/log" % {'shared_path': env.shared_path, 'current_release': env.current_release})
def update_env():
"""Update servers environment on the remote servers"""
if 'current_release' not in env:
releases()
run("cd %(current_release)s; virtualenv -p /usr/bin/python2.7 env" % {'current_release': env.current_release})
run("source %(current_release)s/env/bin/activate; cd %(current_release)s; pip -q install -r %(current_release)s/%(env_file)s; pip -q install aerospike --install-option=\"--lua-system-path=env/local/\"" % {'current_release': env.current_release, 'env_file': env.env_file})
permissions()
def cleanup():
"""Clean up old releases"""
if 'releases' not in env:
releases()
if len(env.releases) > 5:
directories = env.releases
directories.reverse()
del directories[:5]
env.directories = ' '.join(["%(releases_path)s/%(release)s" % {'releases_path': env.releases_path, 'release': release} for release in directories])
sudo("rm -rf %(directories)s" % {'directories': env.directories})
def rollback_code():
"""Rolls back to the previously deployed version"""
if 'releases' not in env:
releases()
if len(env.releases) >= 2:
env.current_release = env.releases[-1]
env.previous_revision = env.releases[-2]
env.current_release = "%(releases_path)s/%(current_revision)s" % {'releases_path': env.releases_path, 'current_revision': env.current_revision}
env.previous_release = "%(releases_path)s/%(previous_revision)s" % {'releases_path': env.releases_path, 'previous_revision': env.previous_revision}
sudo("rm %(current_path)s; ln -s %(previous_release)s %(current_path)s && rm -rf %(current_release)s" % {'current_release': env.current_release, 'previous_release': env.previous_release, 'current_path': env.current_path})
def rollback():
"""Rolls back to a previous version and restarts"""
rollback_code()
restart()
def deploy():
"""Deploys your project. This calls both `update' and `restart'"""
setup()
update()
restart()
cleanup()
upload_to_gs()
@dgboy2000
Copy link

This gist looks like it'd solve my problems. Do you give others permission to use it, eg under some permissive license like MIT?
https://opensource.org/licenses/MIT

Thanks in advance!
Danny

@trilopin
Copy link
Author

@dgboy2000 of course, do whatever you want with this piece of code

@Cylix
Copy link

Cylix commented Apr 9, 2019

Thanks a lot for this gist!

Note that the compute instances list parameter evolved a bit since then, I am now currently using 'gcloud --project "%s" compute instances list --filter "NAME=(%s) zone:(%s)"' % (project, match, zone), hope that helps some others :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment