sync_to_s3_with_boto.py

gistfile1.txt

#! /usr/bin/env python
# coding=utf-8

import os
import sys
import boto
import boto.s3
from boto.s3.key import Key


def print_usage():
    usage_text = '''USAGE:
        {} bucket_name directory'''.format(sys.argv[0])

    print(usage_text)


if len(sys.argv) == 3:
    bucket_name = sys.argv[1]
else:
    print_usage()
    sys.exit(1)

AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']

FILES_TO_SYNC = [
]

DIRS_TO_SYNC = [
    sys.argv[2],
]

conn = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)

try:
    bucket = conn.get_bucket(bucket_name)
except boto.exception.S3ResponseError:
    print('Couldn\'t find bucket "{}", creating it'.format(bucket_name))
    bucket = conn.create_bucket(bucket_name)


def percent_cb(complete, total):
    if total > 0:
        sys.stdout.write('{:.2f}%\r'.format((float(complete)/total)*100))
        sys.stdout.flush()


def upload_file(filename):
    uploaded_filename = filename
    if filename.startswith('git/pelican/output/'):
        uploaded_filename = filename[19:]
    print('Uploading {} to Amazon S3 bucket {} as {}'.format(filename, bucket_name, uploaded_filename))

    k = Key(bucket)
    k.key = uploaded_filename.lower()
    k.set_contents_from_filename(filename,
                                 cb=percent_cb, num_cb=100)
    bucket.set_acl('public-read', k.key)
    print("\nFile completed...")


for filename in FILES_TO_SYNC:
    upload_file(filename)

for dirname in DIRS_TO_SYNC:
    for path, subdirs, files in os.walk(dirname):
        for name in files:
            upload_file(os.path.join(path, name))