Last active
October 17, 2020 03:03
-
-
Save sonhanguyen/b36fb5fc2106d0f45d5b00bcd4a38fa2 to your computer and use it in GitHub Desktop.
A filesystem-based memoization decorator
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import collections | |
import pickle | |
import json | |
import urllib.parse | |
import re | |
from functools import wraps | |
from typing import Callable, Any | |
from dataclasses import dataclass, fields | |
@dataclass | |
class Key: | |
function: Callable | |
args: tuple | |
kwarg: dict | |
def __str__(self): | |
stringify = lambda key: re.sub( | |
r'[{}"[\]]', | |
'', | |
json.dumps( | |
key, | |
default=lambda unserializable: '', | |
separators=('_', '-') | |
) | |
) | |
args = stringify(self.args) | |
kwarg = stringify(self.kwarg) | |
params = '_'.join(filter(lambda s: s, [args, kwarg])) | |
func = self.function.__name__ | |
return os.path.join( | |
func, | |
urllib.parse.quote(params)) if params\ | |
else func | |
@dataclass | |
class CacheOptions: | |
read: Callable[[Key], Any] | |
write: Callable[[Key], None] | |
invalidate: Callable[[Any, dict, tuple], bool] | |
key: Callable[[Callable, tuple, dict], Key] | |
def __post_init__(self): | |
self.invalidate = self.invalidate or (lambda *_: False) | |
self.key = self.key or Key | |
def cache(options: CacheOptions): | |
def cached(func): | |
@wraps(func) | |
def with_cache(*args, **kwarg): | |
key = options.key(func, args, kwarg) | |
try: | |
result = options.read(key) | |
except KeyError: | |
result = func(*args, **kwarg) | |
if not options.invalidate(result, kwarg, args): options.write(key, result) | |
return result | |
return with_cache | |
return cached | |
def file_cache(**options): | |
def dumps(fname, serialized): | |
os.makedirs(os.path.dirname(fname), exist_ok=True) | |
with open(fname, 'wb') as file: | |
file.write(serialized) | |
def loads(fname): | |
with open(fname, 'rb') as file: | |
return file.read() | |
@dataclass | |
class Options(CacheOptions): | |
dir: str | |
marshal: Callable | |
unmarshal: Callable | |
def __init__(self, **kwarg): | |
attrs = self.__dict__ | |
attrs.update(kwarg) | |
path = lambda key: os.path.join(attrs.get('dir', '.'), str(key)) | |
unmarshal = attrs.get('unmarshal', | |
lambda serialized, key: pickle.loads(serialized)) | |
marshal = attrs.get('marshal', lambda val, key: pickle.dumps(val)) | |
def get(key): | |
try: | |
return unmarshal(loads(path(key)), key) | |
except (FileNotFoundError, NotADirectoryError): | |
raise KeyError('Cache misses for ' + str(key)) | |
props = map(lambda field: field.name, fields(super())) | |
super().__init__(**{ | |
**{ key: kwarg.get(key) for key in props }, | |
**{ | |
'write': lambda key, val: dumps(path(key), marshal(val, key)), | |
'read': get | |
} | |
}) | |
return cache(Options(**options)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment