Last active
August 29, 2015 14:13
-
-
Save abelsonlive/10f52502e2a272d59c5b to your computer and use it in GitHub Desktop.
Abstract Cache With Class
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import hashlib | |
import s3plz | |
import os | |
import requests | |
class Cache(object): | |
""" | |
An Abstract class for caching | |
urls. | |
""" | |
def __init__(self, directory='cache', **kw): | |
if not directory.endswith('/'): | |
directory += '/' | |
self.directory = directory | |
self.extension = kw.get('extension', 'cache') | |
self.ttl = kw.get('ttl') | |
self.configure(**kw) | |
def url_to_filepath(self, url): | |
md5 = hashlib.md5(url).hexdigest() | |
return "{}{}.{}".format(self.directory, md5, self.extension) | |
def get(self, url, **kw): | |
filepath = self.url_to_filepath(url) | |
if not self.exists(filepath): | |
contents = self.get_url(url, **kw) | |
self.cache(contents, filepath) | |
else: | |
contents = self.load(filepath) | |
return contents | |
def get_url(self, url, **kw): | |
raise NotImplementedError | |
def configure(self, **kw): | |
pass | |
def cache(self, contents, filepath): | |
raise NotImplementedError | |
def exists(self, filepath): | |
raise NotImplementedError | |
def load(self, filepath): | |
raise NotImplementedError | |
class LocalCache(Cache): | |
""" | |
Local Cache | |
""" | |
def configure(self, **kw): | |
if not os.path.exists(self.directory): | |
os.mkdir(self.directory) | |
def cache(self, contents, filepath): | |
with open(filepath, 'wb') as f: | |
f.write(contents) | |
def exists(self, filepath): | |
return os.path.exists(filepath) | |
def load(self, filepath): | |
return open(filepath).read() | |
class S3Cache(Cache): | |
""" | |
S3 Cache | |
""" | |
def configure(self, **kw): | |
""" | |
Optionally pass in s3 conn | |
""" | |
assert(s3plz.utils.is_s3_uri(self.directory)) | |
self.s3 = s3plz.connect(self.directory) | |
def cache(self, contents, filepath): | |
self.s3.put(contents, filepath) | |
def exists(self, filepath): | |
return self.s3.exists(filepath) | |
def load(self, filepath): | |
return self.s3.get(filepath) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Usage
Inherit from
S3Cache
orLocalCache
and overwrite theget_url
method.This function takes a url and returns the html of its page.
Now, use it:
Examples