Streaming recipes

iter_batches.py

import pandas as pd


def iter_batches(X_y, batch_size):

    x_batch = [None] * batch_size
    y_batch = [None] * batch_size
    j = 0

    for i, (x, y) in enumerate(X_y, start=1):

        x_batch[j] = x
        y_batch[j] = y

        j += 1

        if j == batch_size:
            index = pd.RangeIndex(i - batch_size, i)
            yield (
                pd.DataFrame(x_batch, index=index),
                pd.Series(y_batch, index=index)
            )
            j = 0