s3cp.py

s3cp.py

#!/usr/bin/env python2
import argparse
import logging
import os
import sys

from boto.s3.connection import S3Connection

LOG_FILENAME = '/home/rharding/.s3cp.log'
NEVER = 'Thu, 31 Dec 2037 23:59:59 GMT'

class S3Exception(Exception): pass

# set up logging to file - see previous section for more details
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename=LOG_FILENAME,
                    filemode='a')

# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)


def parse_args():
    """Parse out the options we support.

    Arg options:
        - bucket
        - folder path
        - public (bool)
        - no_overwrite (bool)
    """
    parser = argparse.ArgumentParser(description='Copy a file over to a S3 location')

    parser.add_argument('--bucket', dest='bucket',
                            action='store',
                            default=False,
                            help='What S3 bucket namespace should files be placed')


    parser.add_argument('--public', dest='public',
                            action='store_true',
                            default=False,
                            help='if set files uploaded will be made public')

    parser.add_argument('--no-overwrite', dest='no_overwrite',
                            action='store_true',
                            default=False,
                            help='if set files that exist will not be overwritten')

    parser.add_argument('--folder-prefix', dest='folder_prefix',
                            action='store',
                            default=None,
                            help='set a nesting path for the file name /sample/folder')

    parser.add_argument('files', metavar='N', type=str, nargs='+',
                       help='one or more files to upload')

    args = parser.parse_args()
    return args


def handle_progress(transmitted, pending):
        sys.stdout.write("Upload progress: %d%%   \r" % (transmitted/float(pending))*100)
        sys.stdout.flush()


def send_file(filename, bucketname, folder_prefix=None, no_overwrite=True, public=True):
    connection = S3Connection()
    bucket = connection.get_bucket(bucketname)

    if folder_prefix:
        basefilename = os.path.join(folder_prefix, filename)
        k = bucket.new_key(basefilename)
    else:
        k = bucket.new_key(filename)

    if bucket.get_key(k.key):
        if not no_overwrite:
            logging.error('Duplicate filename %s! I hope that is OK.' % k.key)
        else:
            logging.error('Duplicate filename %s! Skipping file.' % k.key)
            raise Exception('Skipping existing filename: %s')

    logging.info('Uploading %s ' % k.key)

    headers={}
    #if never_expire:
    #    headers['Expires'] = NEVER
    #headers = {'x-amz-storage-class': 'REDUCED_REDUNDANCY'}

    fd = open(filename)
    k.set_contents_from_file(fd, headers=headers, cb=handle_progress)

    sys.stdout.flush()
    sys.stdout.write("...Upload Complete \n")
    sys.stdout.flush()

    if public:
        k.set_acl('public-read')

    return k

def main(args):
    """Run the process of uploading given the args provided"""
    if len(args.files) > 0:
        for filename in args.files:
            send_file(filename, args.bucket,
                    folder_prefix=args.folder_prefix,
                    no_overwrite=args.no_overwrite,
                    public=args.public,
            )
    else:
        raise Exception("No files given")


if __name__ == "__main__":
    args = parse_args()

    if not args.bucket:
        """We need at least a bucket to dump into"""
        logging.error("ERROR: you must supply a bucket to upload to")
        sys.exit(1)

    try:
        main(args)
    except Exception, exc:
        logging.error(str(exc))