public
Created

Migrate from S3 to Rackspace Cloud Files

  • Download Gist
s3_rackspace_migrate.py
Python
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
#!/usr/bin/python
 
"""
 
S3 to Rackspace Cloud Files Migration
 
This script will copy the contents of a S3
bucket to to a Rackspace Cloud Files container.
 
Author: Tony Landis
Website: www.tonylandis.com
License: Do whatever you want with this code
 
Usage: Just define the S3_* and CF_* settings below before running.
If you have a s3 bucket name with characters that are not
valid for a file name on your system, you will need to change
the tmp_file and mrk_file as well to avoid issues.
 
 
"""
 
import cloudfiles
from boto.s3.connection import S3Connection
from boto.s3.key import Key
 
# the s3 api key, secret, and bucket to copy from
S3_KEY = ''
S3_SECRET = ''
S3_BUCKET = ''
 
# the rackspakce cloud files user, api, and container to copy to
CF_USER = ''
CF_API_KEY = ''
CF_CONTAINER = ''
 
# connect to s3
s3_conn = S3Connection(S3_KEY, S3_SECRET, is_secure=False)
s3_bucket = s3_conn.create_bucket(S3_BUCKET)
# connect to cf
cf_conn = cloudfiles.get_connection(CF_USER, CF_API_KEY, serviceNet=True)
cf_container = cf_conn.get_container(CF_CONTAINER)
# setup temp files
tmp_file = '/tmp/%s' % S3_BUCKET
mrk_file = '/tmp/key_%s' % S3_BUCKET
#see if we have a file with the key marker for s3 get_all_keys()
key_marker=''
try:
fp = open(mrk_file, 'r')
lines = fp.readlines()
fp.close()
if lines:
key_marker = lines[-1]
except Exception:
pass
 
def handle(name):
"try to do the copy"
try:
#get tmp file
key = Key(s3_bucket, name)
#copy to tmp
fp = open(tmp_file, "w")
key.get_file(fp)
fp.close()
#copy to cf
fp = open(tmp_file, "r")
#create the object to copy to
o = cf_container.create_object(name)
o.write(fp)
#cleanup
fp.close()
return True
except Exception:
print ' retrying'
return False
 
i = 0
rs = True
while rs:
"get all the keys"
rs = s3_bucket.get_all_keys(marker=key_marker)
for s3_key in rs:
name = s3_key.name
print "%i %s" % (i, name)
done, tries = False, 0
while done == False:
#keep retrying, sometimes things time out
done = handle(name)
#reset key marker, save last processed
key_marker = name
fp = open(mrk_file, 'w')
fp.write(key_marker)
fp.close()
i+=1
 
print "All done!"

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.