writable MDS store for dask

writeable_mds_store.py

class writable_mds_store:
    
    def __init__(self, prefix, iters, suffix='data', dtype='>f4'):
        self.prefix = prefix
        self.iters = iters
        self.suffix = suffix
        self.dtype = dtype
    
    def __setitem__(self, idx, data):
        # first slice should be the time index
        tslice = idx[0]
        # make sure it is just one single time slice        
        assert tslice.step is None
        assert (tslice.stop - tslice.start) == 1
        n = tslice.start
        fname = '%s.%010d.%s' % (self.prefix, self.iters[n], self.suffix)
        #print("Writing %s" % fname)
        data.astype(self.dtype).tofile(fname)
        
# to use
# write all the data to disk
outdir = '/vega/physo/users/jb3210/offline_velocities/aviso_DUACS2014_daily_msla/div_corrected/'
uvel_store = writable_mds_store(outdir + 'uvelCorr', m.iter.values)
with ProgressBar():
    m.UVEL_Psi.data.store(uvel_store)
    

See my fork for a f.seek implimentation that is probably fragile, but works: https://gist.github.com/jklymak/282aa1fece49167af68997bdf2f95780