writable MDS store for dask

writeable_mds_store.py

class writable_mds_store:
    '''
    This will write each chunk to a binary file.  Really only works if chunks are the k-index of the matrix
    '''
    def __init__(self, prefix, itern, suffix='data', dtype='>f4'):
        self.prefix = prefix
        self.itern = itern
        self.suffix = suffix
        self.dtype = dtype
        self.fname = '%s.%010d.%s' % (self.prefix, self.itern, self.suffix)
        # zero the file:
        with open(self.fname,'wb+') as f:
            pass
        

    def __setitem__(self, idx, data):
        import os
        tslice = idx[0]
        assert tslice.step is None
        assert (tslice.stop - tslice.start) == 1
        n = tslice.start
        nbytes = data.astype(self.dtype).nbytes
        kslice = idx[1]
        with open(self.fname,'rb+') as f:
            f.seek(nbytes*kslice.start,os.SEEK_SET)
            f.write(data.astype(self.dtype).tobytes())
        
tnew_store = writable_mds_store('Tnew',0,dtype=tm.dtype)
#print(tm[:,0:10,:,:])
tm.store(tnew_store,lock=True)