futzing with google storage

goog_storage_notes.py

#!/usr/bin/python

# easy_install boto
# created .boto config file

# read: http://thurloat.com/2010/06/07/google-storage-and-app-engine/
# https://gist.github.com/425480
# https://code.google.com/apis/storage/docs/gspythonlibrary.html

# Google storage video from cloudstock: http://www.cloudstockevent.com/10/sessions/Introduction-to-Google-s-Cloud-Platform-Technologies
# Starts around minute 31

# Now getting this error:

#File "gstest.py", line 49, in <module>
#    key = dst_uri.new_key()
#  File "build/bdist.macosx-10.6-universal/egg/boto/storage_uri.py", line 120, in new_key
#  File "build/bdist.macosx-10.6-universal/egg/boto/storage_uri.py", line 105, in get_bucket
#  File "build/bdist.macosx-10.6-universal/egg/boto/s3/connection.py", line 325, in get_bucket
#  File "build/bdist.macosx-10.6-universal/egg/boto/s3/bucket.py", line 295, in get_all_keys
#  File "build/bdist.macosx-10.6-universal/egg/boto/s3/bucket.py", line 262, in _get_all
#boto.exception.GSResponseError: GSResponseError: 400 Bad Request
#<?xml version='1.0' encoding='UTF-8'?><Error><Code>InvalidArgument</Code><Message>Invalid argument.
#</Message><Details>Invalid header: max-keys</Details></Error>
# Error was from not specifying 'gs' as the API type

# ASIDE: Couldn't find my default packages location.. Found it with:
# python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"



import StringIO
import os
import shutil
import tempfile
import time

import boto

# Read developer keys from the ~/.boto config file.
config = boto.config

# URI scheme for Google Storage.
GOOGLE_STORAGE = 'gs'
# URI scheme for accessing local files.
LOCAL_FILE = 'file'

now = time.time()
CHESNOK_BUCKET = 'chesnok-%d' % now

for name in (CHESNOK_BUCKET,):
    # Instantiate a BucketStorageUri object.
    print name
    uri = boto.storage_uri(name, GOOGLE_STORAGE)
    print uri
    # Try to create the bucket.
    try:
        uri.create_bucket()
        print 'Successfully created bucket "%s"' % name
    except boto.exception.StorageCreateError, e:
        print 'Failed to create bucket:', e

# Upload deez files
tempfiles = { 'access.log': 'chesnok.com access logs' }
for filename, descr in tempfiles.iteritems():
    contents = file(filename, 'r')
    print GOOGLE_STORAGE + '/' + CHESNOK_BUCKET + '/' + filename
    dst_uri = boto.storage_uri( CHESNOK_BUCKET + '/' + filename, GOOGLE_STORAGE)
    # The key-related functions are a consequence of boto's
    # interoperability with Amazon S3 (which employs the
    # concept of a key mapping to contents).
    try:
        dst_uri.new_key().set_contents_from_file(contents)
        print 'Successfully created "%s/%s"' % (dst_uri.bucket_name, dst_uri.object_name)
    except boto.exception.GSResponseError, e:
        print 'Failed to create new key:', e

    contents.close()

uri = boto.storage_uri(CHESNOK_BUCKET, GOOGLE_STORAGE)
for obj in uri.get_bucket():
  print '%s://%s/%s' % (uri.scheme, uri.bucket_name, obj.name)
  print '  "%s"' % obj.get_contents_as_string()