Skip to content

Instantly share code, notes, and snippets.

@nb-programmer
Created May 8, 2021 07:23
Show Gist options
  • Save nb-programmer/a59da2a1734453f42577f57479c93bd6 to your computer and use it in GitHub Desktop.
Save nb-programmer/a59da2a1734453f42577f57479c93bd6 to your computer and use it in GitHub Desktop.
Simple Request caching to file, for python requests module
import pickle
import hashlib
import requests
#Make sure the cache directory exists
class CachedSession(requests.Session):
cache_format = "{cache_dir}/res_{hash}_obj.bin"
cache_dir = "./cache"
def send_cached(self, request):
req_pkl = pickle.dumps(request)
req_hash = hashlib.sha1(req_pkl).hexdigest()
cache_file_name = self.cache_format.format(cache_dir = self.cache_dir, hash = req_hash)
try:
with open(cache_file_name, "rb") as f:
return pickle.load(f)
except (FileNotFoundError, EOFError):
prep = self.prepare_request(request)
res = None
try:
res = self.send(prep)
with open(cache_file_name, "wb") as f:
pickle.dump(res, f)
finally:
return res
#Example usage
if __name__ == "__main__":
sess = CachedSession()
request = requests.Request('GET', 'https://www.google.com/')
res = sess.send_cached(request)
print("Got data:")
print(res.text)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment