generate.py

import numpy as np
import h5py

ADDRESS_COUNT = 100_000
BATCH_COUNT = 500

matrix = np.random.rand(ADDRESS_COUNT // BATCH_COUNT, ADDRESS_COUNT)

for i in range(BATCH_COUNT - 1):
    print(f'Generating matrix {i} of {BATCH_COUNT}')
    random_matrix = np.random.rand(ADDRESS_COUNT // BATCH_COUNT, ADDRESS_COUNT)
    matrix = np.concatenate((matrix, random_matrix))

with h5py.File('random.hdf5', 'w') as f:
    f.create_dataset('test', data=matrix)