Skip to content

Instantly share code, notes, and snippets.

@davidhariri
Created January 14, 2017 15:45
Show Gist options
  • Save davidhariri/1d00bfcca6bfe0491f65af027e7b46f7 to your computer and use it in GitHub Desktop.
Save davidhariri/1d00bfcca6bfe0491f65af027e7b46f7 to your computer and use it in GitHub Desktop.
class ModelStore(object):
def __init__(
self,
s3_id,
s3_key,
s3_bucket,
should_compress=True,
cache_expiry=180,
profile=False,
should_cache=True
):
self.conn = S3Connection(
s3_id,
s3_key
)
self.bucket = self.conn.get_bucket(s3_bucket)
self.should_compress = should_compress
self.cache_expiry = cache_expiry
self.should_cache = should_cache
def __make_s3_file(self, key):
new_file = Key(self.bucket)
new_file.key = key
return new_file
def put(self, key, obj):
stringed_obj = pickle.dumps(obj)
if self.should_compress:
stringed_obj = zlib.compress(stringed_obj)
new_file = self.__make_s3_file(key)
new_file.set_contents_from_string(stringed_obj)
if self.should_cache:
cache.set(key, stringed_obj, self.cache_expiry)
return True
def get(self, key):
pickled_obj = cache.get(key) if self.should_cache else None
if pickled_obj is None:
found_file = self.__make_s3_file(key)
pickled_obj = found_file.get_contents_as_string()
if self.should_cache:
cache.set(key, pickled_obj, self.cache_expiry)
if self.should_compress:
pickled_obj = zlib.decompress(pickled_obj)
ti = time.time() if self.profile else None
obj = pickle.loads(pickled_obj)
return obj
def delete(self, key):
self.bucket.delete_key(key)
return True
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment