S3用のrsync的なPythonスクリプト。botoが必要。excludeパターンの指定可。

gistfile1.py

# -*- coding: utf-8 -*-
"""
File Name: upload_s3.py
Usage: python upload_s3.py
"""

import sys
import os
import os.path
import fnmatch
from boto.s3.connection import S3Connection
import hashlib


AWS_ACCESS_KEY = ''
AWS_SECRET_KEY = ''
S3_TARGET_BUCKET = ''
LOCAL_TARGET_FOLDER = ''
EXCLUDE_PATTERNS = ['.*', '*~', '*.coffee', '*.sass', '*.psd']
EXECUTE_OPERATION = "Please execute!!!" 


def is_exclude(key):
    for file_or_dir_name in key.split(os.sep):
        for exclude_file_pattern in EXCLUDE_PATTERNS:
            if fnmatch.fnmatch(file_or_dir_name, exclude_file_pattern):
                return True
    return False

local_keys = []
for dirpath, dirnames, filenames in os.walk(LOCAL_TARGET_FOLDER):
    for filename in filenames:
        key = os.path.join(dirpath, filename)
        key = key.replace(LOCAL_TARGET_FOLDER, '')
        if not is_exclude(key):
            local_keys.append(key)


s3 = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
bucket = s3.get_bucket(S3_TARGET_BUCKET)

s3_keys = [key.name for key in bucket.list() if not key.name.endswith("/")]


upload_keys = []
delete_keys = []

for local_key in local_keys:
    if local_key not in s3_keys:
        upload_keys.append(local_key)
    else:
        s3_key_object = bucket.get_key(local_key)
        with open(os.path.join(LOCAL_TARGET_FOLDER, local_key)) as local_key_content:
            s3_key_md5 = hashlib.md5(s3_key_object.get_contents_as_string()).hexdigest()
            local_key_md5 = hashlib.md5(local_key_content.read()).hexdigest()
            if s3_key_md5 != local_key_md5:
                upload_keys.append(local_key)

delete_keys = [s3_key for s3_key in s3_keys if s3_key not in local_keys]


print ""
print "Upload ({0}) ---------------------".format(len(upload_keys))
for upload_key in upload_keys:
    print upload_key

print ""
print "Delete ({0}) ---------------------".format(len(delete_keys))
for delete_key in delete_keys:
    print delete_key

print ""
if len(sys.argv) > 1:
    operation = sys.argv[1]
else:
    operation = raw_input("Execute? (If want to execute, input '{0}') > ".format(EXECUTE_OPERATION))
if operation == EXECUTE_OPERATION:
    print ""
    print "Start Upload ({0}) -------------------".format(len(upload_keys))
    for upload_key in upload_keys:
        key_object = bucket.new_key(upload_key)
        with open(os.path.join(LOCAL_TARGET_FOLDER, upload_key)) as upload_file:
            key_object.set_contents_from_file(upload_file)
            key_object.make_public()
        print "Uploaded: {0}".format(upload_key)

    print ""
    print "Start Delete ({0}) -------------------".format(len(delete_keys))
    for delete_key in delete_keys:
        bucket.delete_key(delete_key)
        print "Deleted: {0}".format(delete_key)
else:
    print ""
    print "Cancelled -------------------".format(len(delete_keys))