Shows leak with ParameterFunctionType

memhunt.py

#!/usr/bin/env python
import gevent
import gc
from load_datasets import *
from coverage_model import AbstractCoverage, ViewCoverage, ParameterContext, ParameterFunctionType, create_guid

def get_mem():
    import resource
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024.0**2)

root = '/tmp/ion/run/cache/datasets/'

pdict_name = 'ctd_parsed_param_dict'

for i in xrange(5):
    if i == 0:
        print '\nVia create_data_product'
        data_product_id, stream_id, route, stream_def_id, dataset_id = create_data_product(pdict_name)
        populate_data_product(data_product_id,1)
    elif i == 1:
        print '\nManual coverage creation with pdict:', pdict_name
        from scratch.memory_trials import make_cov
        import os
        if not os.path.exists(root):
            os.makedirs(root)
        pdict_id = pn.dataset_management.read_parameter_dictionary_by_name(pdict_name, id_only=True)
        pdict = pn.dataset_management.get_parameter_dictionary(pdict_id)
        params = [p[1] for p in pdict.itervalues()]
        ntimes = 3600
        vals = range(ntimes)
        ddict = {}
        for p in params:
            if isinstance(p.param_type, ParameterFunctionType):
                continue
            ddict[p] = vals
        scp = make_cov(root, params, nt=ntimes, data_dict=ddict, make_temporal=False)
        vc = ViewCoverage(root, create_guid(), 'test', reference_coverage_location=scp)
        dataset_id = vc.persistence_guid
        vc.close()
    elif i == 2:
        print '\nManual coverage creation with manually created pdict using PC\'s from', pdict_name
        from scratch.memory_trials import make_cov
        import os
        if not os.path.exists(root):
            os.makedirs(root)
        pdict_id = pn.dataset_management.read_parameter_dictionary_by_name(pdict_name, id_only=True)
        pcs = pn.dataset_management.read_parameter_contexts(pdict_id, id_only=False)
        # pdict = pn.dataset_management.get_parameter_dictionary(pdict_id)
        params = [ParameterContext.load(p.parameter_context) for p in pcs]
        ntimes = 3600
        vals = range(ntimes)
        ddict = {}
        for p in params:
            if isinstance(p.param_type, ParameterFunctionType):
                continue
            ddict[p] = vals
        scp = make_cov(root, params, nt=ntimes, data_dict=ddict, make_temporal=False)
        vc = ViewCoverage(root, create_guid(), 'test', reference_coverage_location=scp)
        dataset_id = vc.persistence_guid
        vc.close()
    elif i == 3:
        print '\nManual coverage creation with manually created pdict using PC\'s from', pdict_name, 'NO PFuncTypes!'
        from scratch.memory_trials import make_cov
        import os
        if not os.path.exists(root):
            os.makedirs(root)
        pdict_id = pn.dataset_management.read_parameter_dictionary_by_name(pdict_name, id_only=True)
        pcs = pn.dataset_management.read_parameter_contexts(pdict_id, id_only=False)
        # pdict = pn.dataset_management.get_parameter_dictionary(pdict_id)
        params = []
        for i, p in enumerate(pcs):
            pc = ParameterContext.load(p.parameter_context)
            if not isinstance(pc.param_type, ParameterFunctionType):
                params.append(pc)
            else:
                params.append('p%i'%i)
        ntimes = 3600
        vals = range(ntimes)
        ddict = {}
        for p in params:
            if isinstance(p, ParameterContext) and isinstance(p.param_type, ParameterFunctionType):
                continue
            ddict[p] = vals
        scp = make_cov(root, params, nt=ntimes, data_dict=ddict, make_temporal=False)
        vc = ViewCoverage(root, create_guid(), 'test', reference_coverage_location=scp)
        dataset_id = vc.persistence_guid
        vc.close()
    else:
        print '\nManual coverage creation with 64 simple params'
        from scratch.memory_trials import make_cov
        import os
        if not os.path.exists(root):
            os.makedirs(root)
        params = ['p%i' % i for i in xrange(64)]
        ntimes = 3600
        vals = range(ntimes)
        ddict = {}
        for p in params:
            ddict[p] = vals
        scp = make_cov(root, params, nt=ntimes, data_dict=ddict, make_temporal=False)
        vc = ViewCoverage(root, create_guid(), 'test', reference_coverage_location=scp)
        dataset_id = vc.persistence_guid
        vc.close()

    gevent.sleep(0.5)

    cov = AbstractCoverage.load(root, dataset_id, mode='r')
    print '\nCov has %i parameters and %i timesteps' % (len(cov.list_parameters()), cov.num_timesteps)

    print '\nWith dataset_management._get_coverage'
    for x in xrange(15):
        s = get_mem()
        cov = pn.dataset_management._get_coverage(dataset_id)
        e = get_mem()
        print e-s, 'mb'
        cov.close()

        gc.collect()
        gevent.sleep(0.2)

    print '\nWith AbstractCoverage.load'
    for x in xrange(15):
        s = get_mem()
        cov = AbstractCoverage.load(root, dataset_id, mode='r')
        e = get_mem()
        print e-s, 'mb'
        cov.close()

        gc.collect()
        gevent.sleep(0.2)

    print "=================="