Skip to content

Instantly share code, notes, and snippets.

@blazetopher
Last active December 17, 2015 11:09
Show Gist options
  • Save blazetopher/5599795 to your computer and use it in GitHub Desktop.
Save blazetopher/5599795 to your computer and use it in GitHub Desktop.
trying to find rtree segfault
#!/usr/bin/env python
"""
@package
@file memory_trials.py
@author Christopher Mueller
@brief
"""
from coverage_model import *
import numpy as np
import os
import resource
def make_cov(root_dir='test_data', params=['a','b'], nt=10, data_dict=None, make_temporal=True, in_memory=False, val_cache=True, return_cov=False):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
if isinstance(params, ParameterDictionary):
pdict = params
else:
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
if make_temporal:
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
for p in params:
if isinstance(p, ParameterContext):
pdict.add_context(p)
elif isinstance(params, tuple):
pdict.add_context(ParameterContext(p[0], param_type=QuantityType(value_encoding=np.dtype(p[1]))))
else:
pdict.add_context(ParameterContext(p, param_type=QuantityType(value_encoding=np.dtype('float32'))))
scov = SimplexCoverage(root_dir, create_guid(), 'sample coverage_model', parameter_dictionary=pdict,
temporal_domain=tdom, spatial_domain=sdom,
in_memory_storage=in_memory, value_caching=val_cache)
if nt != 0:
scov.insert_timesteps(nt)
for p in scov.list_parameters():
if data_dict is not None and p in data_dict:
dat = data_dict[p]
else:
dat = range(nt)
try:
scov.set_parameter_values(p, dat)
except Exception as ex:
import sys
raise Exception('Error setting values for {0}: {1}'.format(p, data_dict[p])), None, sys.exc_traceback
if return_cov:
return scov
scov.close()
return os.path.realpath(scov.persistence_dir) if scov._in_memory_storage is False else 'in-memory'
def looped_insert_data(cov, num_times=200000, num_loops=10):
for x in xrange(num_loops):
cov = insert_data(cov, num_times)
return cov
def descoped_loop(cov, num_times=200000, num_loops=10):
cov = looped_insert_data(cov, num_times, num_loops)
return
def insert_data(cov, num_times=200000):
if isinstance(cov, basestring):
cov = AbstractCoverage.load(cov, mode='w')
strt = cov.num_timesteps
cov.insert_timesteps(num_times)
sl = slice(strt, strt+num_times)
dat = range(num_times)
for p in cov.list_parameters():
if isinstance(cov.get_parameter_context(p).param_type, QuantityType):
cov.set_parameter_values(p, dat, sl)
return cov
pd=[]
def ctd_make_no_close():
if len(pd) == 0:
import yaml
pd.append(ParameterDictionary.load(yaml.load(open('pddump.yml'))))
cov = make_cov(params=[pc[1] for pc in pd[0].itervalues()], make_temporal=False, return_cov=True)
return
def ctd_make_and_close():
if len(pd) == 0:
import yaml
pd.append(ParameterDictionary.load(yaml.load(open('pddump.yml'))))
cov = make_cov(params=[pc[1] for pc in pd[0].itervalues()], make_temporal=False, return_cov=True)
cov.close()
return
def make_no_close():
cov = make_cov(return_cov=True)
return
def load_no_close(cpth):
cov = AbstractCoverage.load(cpth)
return
def make_and_close():
cov = make_cov(return_cov=True)
cov.close()
return
def load_and_close(cpth):
cov = AbstractCoverage.load(cpth)
cov.close()
return
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment