Random compute engine deployment script

deploy.py

import subprocess
import json
import argparse
import random
import string


PROJECT = None
REGION = 'us-central1'
ZONE = 'us-central1-f'
INSTANCE_NAME = 'redis-google-directory'
MACHINE_TYPE = 'n1-standard-1'
REDIS_PASSWORD = ''


def random_password(length=128):
    r = random.SystemRandom()
    choices = string.letters + string.digits
    return ''.join(r.choice(choices) for i in range(length))


def call(*args):
    try:
        return subprocess.check_output(args, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        print "Error running %s:\n%s" % (e.cmd, e.output)
        raise e


def list_buckets():
    out = call('gsutil', 'ls', '-p', PROJECT)
    buckets = out.split()
    buckets = [x[5:-1] for x in buckets]
    return buckets


def create_bucket(name):
    out = call('gsutil', 'mb', '-p', PROJECT, 'gs://' + name)
    return out


def upload_to_bucket(file, bucket):
    out = call('gsutil', 'cp', file, 'gs://' + bucket)
    return out


def create_tar(folder, tarname):
    out = call('tar', '--exclude', '.git', '-zcf', tarname, folder)
    return out


def deployment_bucket_name():
    return PROJECT + '-gce-deployment'


def create_deployment_bucket():
    bucket_name = deployment_bucket_name()
    if bucket_name not in list_buckets():
        print "Creating deployment bucket %s" % bucket_name
        print create_bucket(bucket_name)
    else:
        print "Deployment bucket %s already exists. Good deal" % bucket_name


def create_instance():
    print 'Creating compute engine instance...'
    startup_script_url = 'gs://%s/startup_script.sh' % deployment_bucket_name()
    out = call(
        'gcloud', 'compute', 'instances', 'create', INSTANCE_NAME,
        '--project', PROJECT, '--zone', ZONE,
        '--image', 'debian-7-backports', '--machine-type', MACHINE_TYPE,
        '--scopes', 'storage-ro', '--metadata',
        'redis-password=' + REDIS_PASSWORD,
        'startup-script-url=' + startup_script_url)
    print out
    return out


def instance_info(name):
    out = call(
        'gcloud', 'compute', 'instances', 'describe', name,
        '--project', PROJECT, '--zone', ZONE,
        '--format', 'json')
    data = json.loads(out)
    return data


def promote_ip_address():
    info = instance_info(INSTANCE_NAME)
    external_ip = info['networkInterfaces'][0]['accessConfigs'][0]['natIP']
    call(
        'gcloud', 'compute', 'addresses', 'create',
        '--addresses', external_ip,
        '--project', PROJECT, '--region', REGION,)
    print 'External IP address is %s' % external_ip


def add_tag(instance, tag):
    call(
        'gcloud', 'compute', 'instances', 'add-tags', instance,
        '--tags', tag,
        '--project', PROJECT, '--zone', ZONE,)


def get_firewall_rules():
    out = call(
        'gcloud', 'compute', 'firewall-rules',
        'list', '--project', PROJECT, '--format', 'json')
    data = json.loads(out)
    return data


def add_redis_firewall_rule():
    names = [x['name'] for x in get_firewall_rules()]
    if 'default-allow-redis' in names:
        print 'Redis firewall rule already exists. Cool.'
        return

    call(
        'gcloud', 'compute', 'firewall-rules', 'create',
        'default-allow-redis',
        '--allow', 'tcp:6379', 'udp:6379',
        '--target-tags', 'redis-server',
        '--project', PROJECT)


def main():
    print 'Configuring buckets and uploading files...'
    create_deployment_bucket()
    upload_to_bucket('startup_script.sh', deployment_bucket_name())
    create_tar('.', '/tmp/redis-google-directory.tar.gz')
    upload_to_bucket('/tmp/redis-google-directory.tar.gz', deployment_bucket_name())
    create_instance()
    print 'Configuring networks and IP addresses...'
    promote_ip_address()
    add_tag(INSTANCE_NAME, 'redis-server')
    add_redis_firewall_rule()
    print 'All done!'
    print 'Redis password is: %s' % REDIS_PASSWORD

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Deploy redis-based google directory.')
    parser.add_argument('--project', help='Cloud project to deploy to', required=True)
    parser.add_argument('--region', help='GCE Region', default=REGION)
    parser.add_argument('--zone', help='GCE Zone', default=ZONE)
    parser.add_argument('--name', help='New instance name', default=INSTANCE_NAME)
    parser.add_argument('--type', help='GCE instance type', default=MACHINE_TYPE)

    args = parser.parse_args()

    PROJECT = args.project
    REGION = args.region
    ZONE = args.zone
    INSTANCE_NAME = args.name
    MACHINE_TYPE = args.type
    REDIS_PASSWORD = random_password()

    main()

startup-script.sh

#!/bin/bash

PROJECTID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
REDISPASSWORD=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/redis-password" -H "Metadata-Flavor: Google")

sudo apt-get update
sudo apt-get install -y build-essential tcl8.5 python-dev python-pip python-gevent python-crypto python-openssl
sudo pip install --upgrade pip
sudo pip install --upgrade gevent google-api-python-client redis raven geventhttpclient

# redis
if [ ! $(which redis-cli) ]; then
    cd /tmp
    wget http://download.redis.io/redis-stable.tar.gz
    tar xvzf redis-stable.tar.gz
    cd redis-stable
    make
    sudo make install
    cd utils
    echo -n | sudo ./install_server.sh
    echo -e "\n\n# Password\nrequirepass $REDISPASSWORD" >> /etc/redis/6379.conf
    /etc/init.d/redis_6379 restart
fi

# app
if [ ! -d "/opt/redis-google-directory" ]; then 
    cd /opt
    sudo chmod -R a+wx /opt
    gsutil cp "gs://$PROJECTID-gce-deployment/redis-google-directory.tar.gz" /opt/redis-google-directory.tar.gz
    mkdir -p /opt/redis-google-directory
    tar xzf /opt/redis-google-directory.tar.gz -C /opt/redis-google-directory
    rm /opt/redis-google-directory.tar.gz
    sudo crontab /opt/redis-google/directory/crontab
    sudo chmod -R a+wx /opt
    cd /opt/redis-google-directory
    python main.py
fi