Simple file-based caching for requests module. For scrapers and other creatures.

requests_cache.py

import requests as requests_
import json
import os
from datetime import date

cache_dir = 'cache' + os.sep
cache_dir_mode = 0755

internal_methods = ['method', 'debug']
supported_methods = ['get', 'post', 'put', 'patch', 'head', 'options', 'put']

class RequestsWrapper(object):
    pass

def proxy_method(method):
    def func(self, farg, **kargs):
        kargs['method'] = method
        return request(farg, **kargs)
    return func

for k in dir(requests_):
    fn = getattr(requests_, k)
    if callable(fn):
        setattr(RequestsWrapper, k, proxy_method(k) if k in supported_methods
            else fn)

def get_cache_path(farg, args):
    cache_key = [farg]
    for key, val in args.iteritems():
        cache_key.append(key)
        cache_key.append(str(hash(json.dumps(val, sort_keys=True))))

    cache_key = hash(''.join(cache_key))
    return '%scache%s_%s' % (cache_dir, cache_key,
        date.today().strftime("%Y-%m-%d"))

def request(farg, **kwargs):
    cache_path = get_cache_path(farg, kwargs)
    if os.path.isfile(cache_path):
        with open(cache_path) as f:
            cache_content = {'status_code': 200, '_content': f.read()}
            r = requests_.models.Response()
            r.__setstate__(cache_content)

            if kwargs.get('debug', False):
                print 'using cached'
            return r

    method = kwargs.get('method', 'get')
    fn = getattr(requests_, method)
    if not callable(fn):
        raise ValueError('Invalid method: ' + method)

    args = {k: v for k, v in kwargs.items() if k not in internal_methods}
    r = fn(farg, **args)
    # print 'response', r, farg, kwargs
    if r.status_code == 200:
        if not os.path.isdir(cache_dir):
            os.makedirs(cache_dir, cache_dir_mode)

        with open(cache_path, 'w') as f:
            f.write(r.__getstate__()['_content'])
    return r

requests = RequestsWrapper()