memory_trials

memory_trials.py

#!/usr/bin/env python

from coverage_model import *
import numpy as np
import time
import objgraph
import gevent
from clint.textui import progress
from guppy import hpy
import time

def _build_coverage(num_params=1, value_encoding='float32', brick_size=1000, in_memory=False, verbose=False):
    if verbose:
        print '>> Build Coverage: num_params=%s  value_encoding=%s  brick_size=%s  in_memory=%s' % (num_params, value_encoding, brick_size, in_memory)
    # Instantiate a ParameterDictionary
    pdict = ParameterDictionary()

    # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
    t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=value_encoding))
    t_ctxt.axis = AxisTypeEnum.TIME
    t_ctxt.uom = 'seconds since 01-01-1970'
    pdict.add_context(t_ctxt)

    # Construct temporal and spatial Coordinate Reference System objects
    tcrs = CRS([AxisTypeEnum.TIME])
    scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])

    # Construct temporal and spatial Domain objects
    tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
    sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)

    if num_params > 1:
        for x in range(num_params - 1):
            pdict.add_context(ParameterContext(str(x), param_type=QuantityType(value_encoding=value_encoding)))

    # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
    bricking_scheme = {'brick_size':brick_size,'chunk_size':500}

    return SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, bricking_scheme=bricking_scheme, in_memory_storage=in_memory)

def insert_values(cov=None, num_iters=100, insert_size=10, verbose=False):
    if cov is None:
        cov = _build_coverage()

    ori = cov.num_timesteps
    if verbose:
        print ">>>>> Insert Loop: insert_size=%s <<<<<" % insert_size
        iover = progress.bar(xrange(num_iters))
    else:
        iover = xrange(num_iters)

    for x in iover:
        cov.insert_timesteps(insert_size)
        for p in cov.list_parameters():
            cov.set_parameter_values(p, np.arange(ori,ori+insert_size), slice(ori,ori+insert_size))
        ori += insert_size
#        time.sleep(0.1)

    return cov

def _print_heap(heap, msg='', verbose=False):
    print ">>>> %s: %s" % (msg, heap.size)
    if verbose:
        print heap

def create_looped_insert():
    t_hp = hpy()
    p_hp = hpy()

    verbose = False

    if verbose:
        print ">> Waiting 5 seconds..."
    gevent.sleep(5)

    if verbose:
        print ">> Reset Total Heap"
    t_hp.setrelheap()
    _print_heap(t_hp.heap(), 'Initial Total Heap')

    # Coverage Creation
    if verbose:
        print ">> Reset Partial Heap"
    p_hp.setrelheap()
    if verbose:
        print ">> Build Coverage"
    cov = _build_coverage()
    ph=p_hp.heap()
    _print_heap(p_hp.heap(), 'Partial Heap after coverage creation')
    th = t_hp.heap()
    _print_heap(t_hp.heap(), 'Total Heap after coverage creation')

    # Value Insertion
    if verbose:
        print ">> Reset Partial Heap"
    p_hp.setrelheap()
    if verbose:
        print ">> Insert Values"
    insert_values(cov=cov)
    _print_heap(p_hp.heap(), 'Partial Heap after value insertion')
    _print_heap(t_hp.heap(), 'Total Heap after value insertion')

    ppath = cov.persistence_dir
    for i in xrange(10):
        if verbose:
            print ">> Close coverage \'%s\'" % ppath
        cov.close()

        if verbose:
            print ">> Load coverage \'%s\'" % ppath
            print ">> Reset Partial Heap"
        p_hp.setrelheap()
        cov = SimplexCoverage.load(ppath)
        _print_heap(p_hp.heap(), 'Partial Heap after load')

        # Value Insertion
        if verbose:
            print ">> Reset Partial Heap"
        p_hp.setrelheap()
        if verbose:
            print ">> Insert Values"
        insert_values(cov=cov)
        _print_heap(p_hp.heap(), 'Partial Heap after value insertion')
        _print_heap(t_hp.heap(), 'Total Heap after value insertion')

    #        gevent.sleep(2)

    if verbose:
        print ">> Close coverage \'%s\'" % ppath
    cov.close()

    _print_heap(t_hp.heap(), 'Total Heap')

def param_stats():
    hp = hpy()
    cov=_build_coverage() # Throwaway - removes 1st initialization overhead
    insert_values(cov=cov)
    cov.close()
    headers = ['# Params','Create Heap (bytes)','Create Time (s)','Insertion Heap (bytes)','Insertion Time (s)','Close Time (s)']
    stats = []
    for p in xrange(1,21): # 1-20 parameters
        hp.setrelheap()
        bts=time.time()
        cov = _build_coverage(num_params=p)
        bt=time.time()-bts
        bld_size = hp.heap().size
        hp.setrelheap()
        its=time.time()
        insert_values(cov=cov, num_iters=20, insert_size=100)
        it=time.time()-its

        cts=time.time()
        cov.close()
        ct=time.time()-cts
        stats.append((p, bld_size, bt, hp.heap().size, it, ct))

    return headers, stats

def _print_stats(headers, stat_table):
    print ''.join(['%s\t' % x for x in headers])
    for r in stat_table:
        print ''.join(['%s\t' % x for x in r])

if __name__ == '__main__':
#    create_looped_insert()

    _print_stats(*param_stats())

    #def leaker_heapy(cov=None, iters=100, inserts=10):
    #    p_hp=hpy()
    #    if cov is None:
    #        cov = _build_coverage()
    #
    #    print ">>>>> Init <<<<<"
    #    #    objgraph.show_refs(cov, filename='init.png', refcounts=True)
    #    #    objgraph.show_growth()
    #    #    p_hp.setrelheap()
    #    hi = p_hp.heap()
    #
    #    ori = cov.num_timesteps
    #
    #    print ">>>>> Insert Loop <<<<<"
    #    for x in progress.bar(xrange(iters)):
    #        cov.insert_timesteps(inserts)
    #        cov.set_parameter_values('time', np.arange(ori,ori+inserts), slice(ori,ori+inserts))
    #        ori += inserts
    #        #        print ">>>>> iter %s <<<<<" % x
    #    #        time.sleep(0.1)
    #
    #    print ">>>>> Final <<<<<"
    #    #    objgraph.show_refs(cov, filename='final.png', refcounts=True)
    #    #    objgraph.show_growth()
    #    hf = p_hp.heap()
    #
    #    return cov, hi, hf