Skip to content

Instantly share code, notes, and snippets.

@erans
Last active April 25, 2020 12:27
Show Gist options
  • Save erans/1275085 to your computer and use it in GitHub Desktop.
Save erans/1275085 to your computer and use it in GitHub Desktop.
Clone S3 Bucket
import argparse
import time
from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
def run(args):
s3_connection = S3Connection(args.aws_access_key, args.aws_secret_access_key)
source_bucket = s3_connection.get_bucket(args.source_bucket)
destination_bucket = None
try:
destination_bucket = s3_connection.get_bucket(args.destination_bucket)
except S3ResponseError, ex:
if ex.status == 404:
destination_bucket = s3_connection.create_bucket(args.destination_bucket)
if args.verbose:
print "- Copying from %s to %s" % (source_bucket.name, destination_bucket.name)
counter = 0
rs = source_bucket.list()
for key in rs:
if args.verbose:
print "\tCopying %s" % key.key
i = 0
while(True):
try:
key.copy(destination_bucket.name, key.key)
break
except boto.exception.S3CopyError:
print 'exception during copying... will try again...'
time.sleep(2)
if i > 10:
raise
counter += 1
if args.verbose:
print "- Copied %d items" % counter
print "- Done"
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--source_bucket', help='Source Bucket')
parser.add_argument('-d', '--destination_bucket', help='Destination Bucket')
parser.add_argument('-k', '--aws_access_key', help='AWS Access Key. Default can be taken from environment variable "AWS_ACCESS_KEY_ID"', default=None)
parser.add_argument('-sk', '--aws_secret_access_key', help='AWS Secret Access Key. Default can be taken from environment variable "AWS_SECRET_ACCESS_KEY"', default=None)
parser.add_argument('-v', '--verbose', help="Be verbose", action='store_true')
args = parser.parse_args()
if args.source_bucket and args.destination_bucket:
run(args)
else:
parser.print_help()
if __name__ == "__main__":
main()
@grayaii
Copy link

grayaii commented Aug 30, 2013

Very well done. This script has been very useful for me. Thank you.

@grayaii
Copy link

grayaii commented Aug 30, 2013

I would replace key.copy with the following code, and "import time":

        i = 0
        while(True):
            try:
                key.copy(destination_bucket.name, key.key)
                break
            except boto.exception.S3CopyError:
                print 'exception during copying... will try again...'
                time.sleep(2)
                if i > 10:
                    raise

@alexey
Copy link

alexey commented Apr 28, 2014

you made my day, thanks !

@grayaii
Copy link

grayaii commented Apr 2, 2015

I also would replace the key.copy line with:
key.copy(destination_bucket.name, key.key, preserve_acl=True)
Or have preserve_acl be an option to pass in. I'll create a pull request sometime (it looks like there are tabs-vs-spaces issues right now too)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment