verify-tsv-files.py

import json
import t5
import tensorflow as tf
import functools

ds = tf.data.TextLineDataset("data-path/train.tsv")
ds = ds.map(functools.partial(tf.io.decode_csv, 
                              record_defaults=["", ""], # expects two columsn 
                              field_delim="\t", use_quote_delim=False),
                                num_parallel_calls=tf.data.experimental.AUTOTUNE)

iterator = ds.make_one_shot_iterator()
next_element = iterator.get_next()
with tf.Session() as sess:
    for i in range(1000000):
        print((i, x))
        x = sess.run(next_element)