How to cache large sets of data that are greater than memcache block size (which is around 1MB)

readme

In the Django settings you need to specify that you want to use this cache backend.

fat_cache.py

import cPickle as pickle
from collections import OrderedDict
import logging

from django.conf import settings
from newcache import CacheClass as _CacheClass

# the cache block is set to 512KB because we are doing a serialization bellow
# and also another serialization is done by memcache. So the size of the data
# to be saved in cache is greater.
CACHE_CHUNK_SIZE = getattr(settings, 'CACHE_CHUNK_SIZE', 2**19)

logger = logging.getLogger(__name__)

__all__ = ("CacheClass", )


class CacheClass(_CacheClass):

    def get(self, key, default=None):
        cached_keys_or_data = super(CacheClass, self).get(key, default)
        if not cached_keys_or_data:
            return default

        # we are trying to unpickle data from cache. in case of success
        # it means that the data is not a list of cache ids
        try:
            return pickle.loads(cached_keys_or_data)
        except (TypeError, pickle.UnpicklingError):
            pass

        # we have a list of ids so we can get all the data chunks
        chunks = self.get_many(cached_keys_or_data)
        if not chunks:
            return default
        chunks = OrderedDict(sorted(chunks.items()))

        try:
            return pickle.loads(''.join(chunks.values()))
        except (TypeError, pickle.UnpicklingError):
            logger.error("Unable to deserialize cache data for key '%s'" % key)
            return default

    def set(self, key, cache_data, timeout=0):
        data = pickle.dumps(cache_data)

        cache_keys = []

        chunks = xrange(0, len(data), CACHE_CHUNK_SIZE)
        for chunks_count, position in enumerate(chunks):
            if len(chunks) == 1:
                cache_key = key
            else:
                cache_key = "%s-%d" % (key, chunks_count)
                cache_keys.append(cache_key)
            super(CacheClass, self).set(
                cache_key,
                data[position:position+CACHE_CHUNK_SIZE],
                timeout
            )

        if len(chunks) > 1:
            super(CacheClass, self).set(key, cache_keys, timeout)