Skip to content

Instantly share code, notes, and snippets.

@bbengfort
Created October 8, 2013 15:02
Show Gist options
  • Save bbengfort/6886128 to your computer and use it in GitHub Desktop.
Save bbengfort/6886128 to your computer and use it in GitHub Desktop.
A wrapper for Boto that provides a posix-like management interface to S3 with Read, Write, and Delete.
#!/usr/bin/env python
# graffiti.breakdance.s3
# Handler class for writing to Amazon S3
#
# Author: Benjamin Bengfort <ben@cobrain.com>
# Created: Fri Sep 27 08:48:19 2013 -0400
#
# Copyright (C) 2013 Cobrain Company
# For license information, see LICENSE.txt
#
# ID: s3.py [] ben@cobrain.com $
"""
A client that performs a RESTful request to the S3 API with Boto to write
data files to a long term data store.
>>> manager = S3Manager()
>>> manager.write("test.txt", "Shopping list: Apples, Oranges, Pears")
>>> manager.read("test.txt")
'Shopping list: Apples, Oranges, Pears'
>>> manager.delete("test.txt")
"""
##########################################################################
## Imports
##########################################################################
import boto
import random
import string
from datetime import datetime
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
##########################################################################
## Exception Heirarchy
##########################################################################
class AWSClientError(Exception):
"""
There was some problem connecting to AWS
"""
pass
##########################################################################
## Writer Class
##########################################################################
class S3Manager(object):
def __init__(self, bucket=None,
aws_access_key_id=None, aws_secret_access_key=None,
**kwargs):
self.bucket = bucket
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.connect()
##////////////////////////////////////////////////////////////////////
## Instance properties
##////////////////////////////////////////////////////////////////////
@property
def bucket(self):
"""
Note: could return either a string or a boto.Bucket object. Always
perform type checking or duck checking (try/except) when using this
property.
"""
return self._bucket
@bucket.setter
def bucket(self, bucket):
"""
Setter for the bucket object, if connected to AWS, this creates a
boto.Bucket object and sets that, otherwise simply saves the bucket
name as a string for use during connection.
"""
bucket = bucket or getattr(settings, "S3_BUCKET", "")
if not bucket:
raise ImproperlyConfigured("No S3_BUCKET has been specified.")
if self.is_connected():
self._bucket = self._connection.lookup(bucket)
if not self._bucket:
self.close()
raise ImproperlyConfigured("Could not connect to S3 Bucket "
" named %s" % bucket)
self._test_write()
else:
self._bucket = bucket
@property
def aws_access_key_id(self):
return self._aws_access_key_id
@aws_access_key_id.setter
def aws_access_key_id(self, key):
"""
TODO: Make a settings fetch descriptor
"""
key = key or getattr(settings, "AWS_ACCESS_KEY_ID", "")
if not key:
raise ImproperlyConfigured("No AWS_ACCESS_KEY_ID has been specified")
self._aws_access_key_id = key
@property
def aws_secret_access_key(self):
return self._aws_secret_access_key
@aws_secret_access_key.setter
def aws_secret_access_key(self, key):
"""
TODO: make a settings fetch descriptor
"""
key = key or getattr(settings, "AWS_SECRECT_ACCESS_KEY", "")
if not key:
raise ImproperlyConfigured(("No AWS_SECRECT_ACCESS_KEY "
"has been specified"))
self._aws_secret_access_key = key
##////////////////////////////////////////////////////////////////////
## API Methods
##////////////////////////////////////////////////////////////////////
def read(self, path):
"""
Read data from a path - specified in the posix style. If the path
does not exist in S3 an error will be raised.
NOTE: When dealing with large files, it is going to be better to
save the contents to a file using other boto methods.
:param path: A posix style path to a location in the bucket
:returns: A string with the data from the path
"""
key = self.get_s3_key(path)
if not key: raise AWSClientError("Could not read from S3 bucket at '%s'" % path)
return key.get_contents_as_string()
def write(self, path, data, overwrite=True):
"""
Write data to a path - specified in the posix style. This will
cause a write to the bucket as defined in the settings or the init
of the manager.
NOTE: When dealing with large files, it is going to be better to
write the content from a file using other boto methods, and ensure
that the file is compressed.
:param path: A posix style path to a location in the bucket
:param data: The data to write to the bucket at the path.
:param overwrite: Specify whether or not to overwrite existing data
:returns: None
"""
if data:
key = self.get_s3_key(path, overwrite=overwrite)
if not key: raise AWSClientError("Could not write to S3 bucket at '%s'" % path)
key.set_contents_from_string(data)
def delete(self, path):
"""
Delete a file in S3 at a path - specified in the posix style. If
the path does not exist in S3 an error will be raised.
:param path: A posix style path to a location in the bucket
:returns: None
"""
key = self.get_s3_key(path)
if not key: raise AWSClientError("Could not delete '%s' from S3 bucket" % path)
key.delete()
def is_connected(self):
"""
Always ensure that if there is an error, call `self.close`
:returns: Boolean describing state of connection
"""
return (hasattr(self, '_connection') and
getattr(self, '_connection'))
##////////////////////////////////////////////////////////////////////
## Helper methods
##////////////////////////////////////////////////////////////////////
def connect(self, bucket=None):
"""
Creates a connection object to AWS and instantiates the Bucket
object, which in turn tests the writability of the bucket in S3
"""
try:
self._connection = boto.connect_s3( self.aws_access_key_id,
self.aws_secret_access_key )
except boto.exception.BotoClientError as e:
raise AWSClientError(str(e))
self.bucket = bucket or self.bucket
def close(self):
"""
Close the connection and clean up the class.
"""
try:
self._connection.close()
except:
pass
self._connection = None
def get_s3_key(self, name, overwrite=True):
"""
Get the key (sort of like the path) in s3. This is an object that
is like a file handler to a particular point in s3. If you want to
overwrite (the default), simply use it, if not overwrite, test for
None.
"""
key = boto.s3.key.Key(self.bucket, name)
if not overwrite:
if key.exists(): return None
return key
def _test_write(self):
"""
Create a randomly named key with the timestamp, then attempt to
set the contents from a string and delete it. If failure, raise
AWSClientError.
"""
def random_chars(size=6):
chars = string.ascii_letters + string.digits
return ''.join(random.choice(chars) for x in xrange(size))
try:
key = "%s-%s.txt" % (datetime.now().strftime("%Y%m%d%H%M%S"),
random_chars())
key = self.get_s3_key(key, overwrite=True)
key.set_contents_from_string(random_chars(32))
key.delete()
except Exception as e:
raise AWSClientError(str(e))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment