Fabric script to deploy staticly built tables from ProPublica's table-setter application to S3.

fabfile.py

from fabric.api import *

"""
Base configuration
"""
env.project_name = 'tables'

"""
Environments
"""
def production():
    """
    Work on production environment
    """
    env.settings = 'production'
    env.s3_bucket = 'production_s3_goes_here'

def staging():
    """
    Work on staging environment
    """
    env.settings = 'staging'
    env.s3_bucket = 'staging_s3_goes_here'
    
"""
Commands - deployment
"""
def deploy():
    """
    Deploy built tables to S3.
    
    Does not perform the functions of load_new_data().
    """
    require('settings', provided_by=[production, staging])
    
    build_tables()
    gzip_assets()
    deploy_to_s3()
    
def build_tables():
    """
    Rebuilds all tables.
    """
    local('rm -rf out')
    local('table-setter build . -p tables')
    local('python rename_tables.py')

def gzip_assets():
    """
    GZips every file in the assets directory and places the new file
    in the gzip directory with the same filename.
    """
    local('python gzip_assets.py')

def deploy_to_s3():
    """
    Deploy the latest built tables to S3.
    """
    env.gzip_path = 'gzip/tables/'
    local(('s3cmd -P --add-header=Content-encoding:gzip --guess-mime-type sync %(gzip_path)s s3://%(s3_bucket)s/%(project_name)s/') % env)

"""
Deaths, destroyers of worlds
"""
def shiva_the_destroyer():
    """
    Remove all directories, databases, etc. associated with the application.
    """
    require('settings', provided_by=[production, staging])
    
    local('s3cmd del --recursive s3://%(s3_bucket)s/%(project_name)s' % env)

gzip_assets.py

#!/bin/env python

import os
import gzip
import shutil

class FakeTime:
    def time(self):
        return 1261130520.0

# Hack to override gzip's time implementation
# See: http://stackoverflow.com/questions/264224/setting-the-gzip-timestamp-from-python
gzip.time = FakeTime()

shutil.rmtree('gzip', ignore_errors=True)
shutil.copytree('out', 'gzip')

for path, dirs, files in os.walk('gzip'):
    for filename in files:
        if filename[-3:] in ['gif', 'png', 'jpg']:
            continue
        
        file_path = os.path.join(path, filename)
        
        f_in = open(file_path, 'rb')
        contents = f_in.readlines()
        f_in.close()
        f_out = gzip.open(file_path, 'wb')
        f_out.writelines(contents)
        f_out.close();

rename_tables.py

#!/bin/env python

import os
import shutil

SKIP = [
    'favicon.ico',
    'images',
    'javascripts',
    'stylesheets',
    ]

try:
    os.remove('out/tables/index.html')
except:
    pass

for i in os.listdir('out/tables'):    
    if i in SKIP:
        continue
        
    folder = os.path.join('out/tables', i)
    index = os.path.join(folder, 'index.html')
    shutil.copyfile(index, 'out/tables/%s.html' % i)
    
    os.remove(index)
    os.rmdir(folder)