Skip to content

Instantly share code, notes, and snippets.

@chicagobuss
Last active March 9, 2016 23:18
Show Gist options
  • Save chicagobuss/898ff4cf8b70dda30c00 to your computer and use it in GitHub Desktop.
Save chicagobuss/898ff4cf8b70dda30c00 to your computer and use it in GitHub Desktop.
how to get boto working with s3proxy (with GCS on the backend)
#!/usr/bin/python
from boto import connect_s3
from boto.s3.connection import OrdinaryCallingFormat
from boto.s3.key import Key
import sys
s3 = connect_s3(aws_access_key_id = 'access',
aws_secret_access_key = 'secret',
host = '127.0.0.1', port = 7000, is_secure = False,
calling_format = OrdinaryCallingFormat())
bucket_name = 'dabucket'
bucket = s3.get_bucket(bucket_name)
filename = "%s" % sys.argv[1]
print 'Downloading %s from Amazon S3 bucket %s' % (filename, bucket_name)
def percent_cb(complete, total):
sys.stdout.write('.')
sys.stdout.flush()
k = Key(bucket)
k.key = filename
k.get_contents_to_file(open(filename, 'w'), cb=percent_cb, num_cb=10)
#!/usr/bin/python
from boto import connect_s3
from boto.s3.connection import OrdinaryCallingFormat
from boto.s3.key import Key
import sys
s3 = connect_s3(aws_access_key_id = 'access',
aws_secret_access_key = 'secret',
host = '127.0.0.1', port = 7000, is_secure = False,
calling_format = OrdinaryCallingFormat())
bucket_name = 'dabucket'
bucket = s3.get_bucket(bucket_name)
filename = "%s" % sys.argv[1]
print 'Uploading %s to Amazon S3 bucket %s' % (filename, bucket_name)
def percent_cb(complete, total):
sys.stdout.write('.')
sys.stdout.flush()
k = Key(bucket)
k.key = filename
k.set_contents_from_filename(filename, cb=percent_cb, num_cb=10)
java -DLOG_LEVEL=info -jar s3proxy-master.jar --properties proxy-gcs-with-auth.conf
# Local proxy settings
s3proxy.authorization=aws-v2
s3proxy.identity=access
s3proxy.credential=secret
s3proxy.endpoint=http://127.0.0.1:7000
jclouds.provider=google-cloud-storage
# This next option is needed if your identity is an iam role, because
# otherwise the project # is in the identity string
jclouds.googlecloud.project-name=<project #>
jclouds.identity=<service account name>@<project_id>.iam.gserviceaccount.com
# If you use the default service account, you don't need to set the project-name:
jclouds.identity=<project #>-compute@developer.gserviceaccount.com
jclouds.credential=/home/jbuss/gce.key
# This key file looks like:
## ---- BEGIN ...
## ..............
## ----- END ....
# Local proxy settings
s3proxy.authorization=aws-v2
s3proxy.identity=access
s3proxy.credential=secret
s3proxy.endpoint=http://127.0.0.1:7000
# Jclouds settings - https://github.com/jclouds/jclouds-site/blob/master/guides/aws-s3.md
jclouds.provider=s3
jclouds.endpoint=https://storage.googleapis.com
jclouds.s3.virtual-host-buckets=false
jclouds.strip-expect-header=true
jclouds.identity=GOOGLEACCESSKEY
jclouds.credential=GOOGLESECRETKEY
jclouds.regions=us-east-1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment