Work in progress

keras_sequence.py

# Keras Sequence
import os
import tempfile
from itertools import product
import numpy as np
from random import shuffle
import time
import sys

f_rows = 30000
f_columns = 9

s_length = 300

batch_size = 32

if __name__ == '__main__':
    with tempfile.TemporaryDirectory() as d:
        files = [os.path.join(d, f"file{file}") for file in range(2)]
        for file in files:
            arr = np.random.random((f_rows, f_columns))
            a = '1'
            f = np.memmap(file, dtype='float32', mode='w+', shape=arr.shape)
            f[:] = arr[:]
            f.flush()
            del f
        fos = [np.memmap(file, dtype="float32", mode='r', shape=(f_rows, f_columns)) for file in files]
        a = 1

        shuf = list(product(range(len(fos)), range(f_rows - s_length + 1)))
        shuffle(shuf)

        t = time.time()
        batch = []
        for fo_ind, r_ind in shuf:
            r = fos[fo_ind][r_ind:r_ind+s_length]
            batch.append(r)
            if len(batch) >= batch_size:
                print(batch)
                pass
                # yield np.array(batch)
                batch = []
        print(np.array(batch).shape)
        print(time.time() - t)