h5_concatenation_snippet.py

# every input file has a different length,
# so I first scan through the files, compute their length,
# `self.lengths` here is an array of cumulative lengths
# so first I create the dataset for full length with resize(lengths[-1])
# then I fill it in.

    def run(self):
        with h5py.File(self.filename, mode='w') as h5f:
            for n, fitsfile in enumerate(self.files):
                print("Processing %s" % fitsfile)
                data = self.get_data(fitsfile)
                print("Data read, length %d" % len(data))
                try:
                    h5f['data'][self.lengths[n]:self.lengths[n+1]] = data
                except exceptions.KeyError: #if dataset not created yet

                    h5f.create_dataset('data', data=data, maxshape=(None,))
                    h5f['data'].resize((self.lengths[-1],))

                h5f.flush()