Update s3 metadata concurrently with futures

s3_update_futures.py

from boto.s3.connection import S3Connection
from futures import ThreadPoolExecutor
import futures

connection = S3Connection('id', 'key')

import logging
logging.basicConfig(level=logging.INFO)


def mod_key(key):
    """Rewrite the headers of js files in js/
    """
    if not key.name.startswith("js/") or not key.name.endswith('.js'):
        logging.info("not doing %s", key)
        return

    key.metadata.update({
        'Content-Type': 'application/javascript',
        'Cache-Control': 'max-age=2592000',  # 30 days. Rewritten at fastly
    })
    key.copy(
        key.bucket.name,
        key.name,
        key.metadata,
        preserve_acl=True
    )

    return "Done %s" % key.name


def main():
    """Use future threads to run concurrent s3 updates
    """
    bucket_name = 'mybucket.com'
    bucket = connection.get_bucket(bucket_name)

    logging.info("starting")

    with ThreadPoolExecutor(40) as executor:
        future_s3 = dict((executor.submit(mod_key, key), key)
                         for key in bucket.list())

        for future in futures.as_completed(future_s3):
            key = future_s3[future]
            if future.exception() is not None:
                print('%r generated an exception: %s' % (key,
                                                         future.exception()))
            else:
                print('%s' % future.result())


if __name__ == '__main__':
    main()