iterable_dataloader_v0.py

#Creating the iterable dataset object
dataset     = CustomIterableDataset('path_to/somefile')
#Creating the dataloader
dataloader  = DataLoader(dataset, batch_size = 64)

for data in dataloader:
    #Data is a list containing 64 (=batch_size) consecutive lines of the file
    print(len(data)) #[64,]
    
    #We still need to separate the text and labels from each other and preprocess the text
    X, y = []
    for i in range(len(data)):
        
        text, label = data[i].split(',')
        text = preprocess(text) #Defined somewhere outside
        
        X.append(text)
        y.append(label)
    
    ### Do something with X and y

    ###