Data generators

generators.py

from random import shuffle


def example_generator(*lists, random_order=True):
    """Create infinite generator of examples.

    example_generator([3, 6, 8], [0, 0, 1]) -> (6,0), (3,0), (8,1), (3,0)...
    """

    assert len(lists) > 0, 'no lists provided'
    length = len(lists[0])
    assert length > 0, 'empty lists provided'
    for i, l in enumerate(lists):
        assert len(l) == length, ('list %s has different length'
                                  ' (%s instead of %s)' % (i, len(l), length))

    examples = list(zip(*lists))

    while True:
        if random_order:
            shuffle(examples)
        for example in examples:
            yield example


def batch_generator(examples, batch_size=5, num_batches=200000):
    """Group examples into batches (expects generator)"""

    for _ in range(num_batches):
        batch = []
        for _ in range(batch_size):
            batch.append(next(examples))
        yield list(zip(*batch))

generators_example.py

import generators


X1 = range(10)
X2 = range(50, 60)
Y = [0]*5 + [1]*5


print('original order')
examples = generators.example_generator(X1, X2, Y, random_order=False)
for _ in range(15):
    print(next(examples))
print()

print('random order')
examples = generators.example_generator(X1, X2, Y)
for _ in range(15):
    print(next(examples))
print()

print('in batches')
batches = generators.batch_generator(examples, batch_size=4, num_batches=10)
for b in batches:
    print(b)