Python Multiprocessing S3 Uploads

#!/usr/bin/env python
from multiprocessing import Pool
import os

# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the <organization> nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY <copyright holder> ''AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <copyright holder> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

def put_dir(bucket_name, directory, num_processes = 20):
    import boto
    file_list = []
    for root, folders, files in os.walk(directory):
        for file in files:
            file_list.append(os.path.join(root,file))
    p = Pool(num_processes)
    bucket = boto.lookup('s3', bucket_name)
    file_args = [(bucket_name, f, f) for f in file_list]
    p.map(public_put_file, file_args)

   
def put_files(bucket_name, files, num_processes = 20):
    import boto
    p = Pool(num_processes)
    bucket = boto.lookup('s3', bucket_name)
    file_args = [(bucket_name, f, f) for f in files]
    p.map(public_put_file, file_args)

def public_put_file(input_tuple):
    import boto
    (bucket_name, file_path, key_name) = (input_tuple[0], input_tuple[1], input_tuple[2])
    bucket = boto.lookup('s3', bucket_name)
    key = bucket.new_key(key_name)
    key.set_contents_from_filename(file_path)
    key.make_public()