Make torchtext training examples from pandas.DataFrame on the fly

make_training_examples.py

import pandas as pd
from torchtext import data
from tqdm import tqdm


def make_examples(df: pd.DataFrame, fields: Dict[str, data.Field]):
    fields = {field_name: (field_name, field)
              for field_name, field in fields.items()}
    for _, row in tqdm(df.iterrows()):
        example = data.Example.fromdict(row, fields)
        yield example