Make lots of small datasets and one large one

make_small_dsets.py

import h5pyd as h5py
import time
import uuid
import numpy as np

class ThingItem:

    def __init__(self, name, age, version, data):
        self.name = name
        self.age = age
        self.version = version
        self.data = data


#Storing approaches

def store(group, items):
    for key, val in items.items():
        if type(val) == dict:
            g = group.require_group(key)
            store(g, val)
        elif type(val) == ThingItem:
            group.attrs["name"] = val.name
            group.attrs["age"] = val.age
            group.attrs["version"] = val.version
        else:
            group.create_dataset(key, data=val)


#
# main
#

totRunningTime = 0

# Creating test data

child= {}
child["name"] = "John"
child["age"] = "32"
child["address"] = "some street"

itm = ThingItem("Jens", 42, 1, child)

things = {}
things["item1"] = 42
things["item2"] = "string test"
things["child1"] = itm
things["child2"] = itm
things["child3"] = itm
things["child4"] = itm

#
# Running the test
#
filename = "test1.h5"
fqdn = "/home/test_user1/small_dset_test/" + filename
N = 100
timingsData = np.zeros(N)
timingsIm = np.zeros(N)

with h5py.File(fqdn, "a") as f:
    start = time.time()
    g = f.require_group("/test")
    for i in range(N):
        print(f"iteration {i} file:"+fqdn) 
        grp_name = str(uuid.uuid4()) 
        sub_grp = g.require_group(grp_name)
        store(sub_grp, things)
    end = time.time()
    timingsData[i] = end-start
    print(f"Saving small datasets: {timingsData[i]} sek" )

    im = np.random.randint(0,10,size=[6000,4000], dtype=np.int16)

    things["im"] = im
    start = time.time()
    f["im"] = im

    end = time.time()
    timingsIm[i] = end-start
    print(f"Saving image: {timingsIm[i]} sek" )
print("")