Skip to content

Instantly share code, notes, and snippets.

@Ab1992ao
Created May 17, 2021 09:23
Show Gist options
  • Save Ab1992ao/8bed4d2ab4bc20e01237ddb8e4532501 to your computer and use it in GitHub Desktop.
Save Ab1992ao/8bed4d2ab4bc20e01237ddb8e4532501 to your computer and use it in GitHub Desktop.
complex data generator for multitask pipe
class MultitaskDataGenerator:
def __init__(self, generators):
self.generators = generators
self.generator = self.generate_batch()
def generate_batch(self, batch_size=None):
while True:
batch = self.__next__()
yield batch
def __next__(self):
data_arrays = []
for gen in self.generators:
gen_data, gen_labels = next(gen.generator)
if type(gen_data) not in {list, tuple}:
gen_data = [gen_data]
data_arrays += gen_data
return data_arrays, gen_labels
def __iter__(self):
return self.__next__()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment