using shelve as cache for request

request_cache.py

import logging
import shelve

import requests


_HEADERS = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) '
                          'AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/53.0.2785.143 Safari/537.36'}
_TIMEOUT = 5
TESTING = True


def get_url(url):
    try:
        if TESTING:
            cache = shelve.open('requests_cache')
            if str(url) in cache:
                logging.info('return from cache: %s', url)
                return cache[str(url)]  # if exists, return page from cache
        page = requests.get(url, headers=_HEADERS, timeout=_TIMEOUT)
        if page.status_code != 200:
            raise requests.RequestException
        logging.info('request page: %s', url)
        if TESTING:
            cache[str(url)] = page  # save page on cache
        return page
    except requests.RequestException as ex:
        logging.error(ex)
        raise
    finally:
        if TESTING:
            cache.close()