NVZ uploader

nvz_upload.py

import boto
import cStringIO
import gzip
import hashlib
import logging
import os
import sys

from boto.s3.connection import S3Connection
from boto.s3.key import Key

dir_name = 'nicholas.veznat.com'
base_path = 'nicholas.veznat.com'

c = S3Connection()
b = c.create_bucket(dir_name)

content_types = {
    'html': 'text/html',
    'css': 'text/css',
    'js': 'text/javascript',
}

boto.set_file_logger('boto', 'nvz_upload.out', level=logging.DEBUG)


def process_dir(dir_path):
    for filename in os.listdir(dir_path):
        complete_path = os.path.join(dir_path, filename)
        if os.path.isdir(complete_path):
            complete_path = complete_path + '/'
            process_dir(complete_path)
            file_id = None
            continue
        else:
            hasher = hashlib.new('sha256')
            with open(complete_path, 'r') as f:
                file_contents = f.read()
                file_id = hashlib.new('sha256', file_contents).hexdigest()

        key_name = complete_path.replace(base_path, '')
        existing_key = b.get_key(key_name)
        if existing_key:
            old_file_id = existing_key.get_metadata('file_id')
            if old_file_id == file_id:
                print 'hashes match', complete_path
                continue

        headers={'Cache-Control': 'max-age=900'}

        if complete_path[-3:] not in ('jpg', 'png', 'gif'):
            compressed_file = cStringIO.StringIO()
            zipper = gzip.GzipFile(fileobj=compressed_file, mode='wb')
            zipper.write(file_contents)
            zipper.close()
            compressed_file.seek(0)
            file_contents = compressed_file.read()
            headers['Content-Encoding'] = 'gzip'
            extension = complete_path[complete_path.rfind('.') + 1:]
            headers['Content-Type'] = content_types[extension]
            compressed = True
        else:
            compressed = False

        def start_progress():
            sys.stderr.write('\n')
        def upload_progress(bytes_done, bytes_total):
            compressed_str = ''
            if compressed:
                compressed_str = '(compressed)'
            sys.stderr.write('\r%s%s: %s / %s' % (
                complete_path, compressed_str, bytes_done, bytes_total))
            sys.stderr.flush()
        def finish_progress(new_key):
            sys.stderr.write('\nFinished: %r\n' % (new_key,))


        start_progress()
        new_key = Key(b, key_name)
        headers['x-amz-meta-file_id'] = file_id
        if compressed:
            new_key.set_contents_from_string(
                file_contents,
                policy='public-read',
                headers=headers,
                cb=upload_progress,
                num_cb=20,
            )
        else:
            with open(complete_path, 'r') as f:
                new_key.set_contents_from_file(
                    f,
                    policy='public-read',
                    headers=headers,
                    cb=upload_progress,
                    num_cb=20
                )
        finish_progress(new_key)

process_dir(dir_name)