Skip to content

Instantly share code, notes, and snippets.

@wassname
Created July 29, 2019 07:30
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save wassname/3909d87306b45435dd83a11d1bfc9d8f to your computer and use it in GitHub Desktop.
how to cache with non immutable function inputs
def cache_load_utturances(ttl=360000):
"""
Decorator for wrapping simple cache around load_utterances.
Since some arguments are unhashable (tokenizer) or immutable (list) we need to make the key manually
"""
def decorate(func):
@simple_cache.wraps(func)
def wrapper(**kwargs):
# key = (args, tuple_kwargs(kwargs))
filename = f"data/.simple.cache"
tokenizer = kwargs["tokenizer"]
# We must use immutaable, hashable args as keys, so no lists, sets, or tokenizer
key = simple_cache.tuple_kwargs(
dict(
data_dir=kwargs["data_dir"],
resnet_name=type(resnet).__name__,
tokenizer_name=type(tokenizer).__name__,
vocab_size=len(tokenizer.encoder),
special_tokens=tuple(sorted(tokenizer.special_tokens)),
)
)
value = simple_cache.load_key(filename, key)
if value is None:
value = func(**kwargs)
simple_cache.save_key(filename, key, value, ttl)
else:
logger.info(f'Loaded utturances from cache for {kwargs["personality"]}')
return value
return wrapper
return decorate
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment