Reading files in minibatches

minibatch.py

def get_minibatch(stream, size):
    """A minibatch is a stream slice."""
    return [doc for doc in itertools.islice(stream, size)]


def iter_minibatches(stream, minibatch_size):
    """Generator of minibatches."""
    minibatch = self.get_minibatch(stream, minibatch_size)
    while len(minibatch):
        yield minibatch
        minibatch = self.get_minibatch(stream, minibatch_size)

        
def stream_files(input_path, file_token):
    """Stream files one by one."""
    for filename in glob.glob(os.path.join(input_path, file_token)):
        yield self.parse(open(filename, 'r').read())


stream = stream_files('path', '*.json')
minibatch_size = 10
       
for minibatch in iter_minibatches(stream=stream, minibatch_size=minibatch_size):
    print(len(minibatch)) # Should be equal to 10 (the minibatch size)