HDF5 direct chunk write example

chunker.py

import h5py
import numpy

import hdf5plugin

import time

NN = 1024
NY = 4096
NX = 4096

FRAME = numpy.random.poisson(0.1, size=(NY, NX)).astype(numpy.uint16)


def write_1k_images(filename):

    with h5py.File(filename, "w") as f:
        data = f.create_dataset(
            "data",
            shape=(NN, NY, NX),
            chunks=(1, NY, NX),
            dtype=numpy.uint16,
            **hdf5plugin.Bitshuffle(),
        )
        t0 = time.time()
        for j in range(NN):
            data[j, :, :] = FRAME
        t1 = time.time()

        print(f"Writing {NN} frames simple way took {t1-t0:.1f}s")


def write_1k_chunks(filename):
    with h5py.File(filename, "w") as f:
        data = f.create_dataset(
            "data",
            shape=(NN, NY, NX),
            chunks=(1, NY, NX),
            dtype=numpy.uint16,
            **hdf5plugin.Bitshuffle(),
        )
        data[0, :, :] = FRAME
        filter, chunk = data.id.read_direct_chunk((0, 0, 0))

        t0 = time.time()
        for j in range(1, NN):
            data.id.write_direct_chunk((j, 0, 0), chunk, filter)
        t1 = time.time()

        print(f"Writing {NN} frames with direct chunk took {t1-t0:.1f}s")


write_1k_images("junk.h5")
write_1k_chunks("chunk.h5")