memcacheset

gistfile1.py

from mymem import cache_set, cache_get

########################
#### private functions


def chunk_seq(file_name, chunk_size):
    '''
        Create a lazy sequence for bytes each up to chunk_size (default 1048576)
    '''
    f = open(file_name)
    while True:
        data = f.read(chunk_size)
        if not data:
            break

        yield data


def debug(s):
    print s

########################
#### public api


## TODO all debug output to stdout err or turnoff with logging

def cache_big_file(ctx, key, file_name, chunk_size=1024):
    '''
     Write a big file to the ctx cache in 1 mb chunks of size chunk_size
    '''
    i = 0

    ## TODO Provide crc check on each data chunk
    for data in chunk_seq(file_name, chunk_size=chunk_size):
        chunk_key = "%s_%s" % (key, i)
        cache_set(ctx, chunk_key, data)
        i = i + 1
        debug("Data set %s of size %s " % (chunk_key, len(str(data))))

    #set final count
    debug("Done set %s chunks" % i)
    cache_set(ctx, "__%s" % key, i)

def cache_big_file_seq(ctx, key, error=True):
    '''
        Return a lazy sequence of chunks from the cache
        if a chunk is not available and error=True then an exception
        is raised, otherwise the sequence stops
    '''
    chunk_count = int(cache_get(ctx, "__%s" % key))

    if chunk_count:
        debug("Reading file with chunks %s" % chunk_count)
        i = 0
        while True:
            chunk_key = "%s_%s" % (key, i)

            data = cache_get(ctx, chunk_key)

            debug("reading key %s " % chunk_key)

            if i < chunk_count and data and not data == 'None':
                yield data
            elif i < chunk_count - 1 and error:
                raise Exception("Chunk %s is of %s is missing " % (i, chunk_count))
            # else nothing
            else:
                break

            i = i + 1
    else:
        ## TODO might want to error here or not
        debug("No file available for %s" % key)

def count_blocks(ctx, key):
    '''
     Helper function for testing
    '''
    i = 0
    for data in cache_big_file_seq(ctx, key):
        i = i + 1

    return i