gpt_tokenize.py

def get_tokenized_dataset(dataset, tokenizer, batch_size, num_proc=None,):
    def process(samples):
        batch_inputs = tokenizer(samples['text'], 
                                max_length=None, 
                                padding=True, 
                                truncation=False, 
                                return_tensors='pt')     
        return batch_inputs

    tokenizer.padding_side = "left"
    remove_columns = dataset.column_names
    dataset = dataset.map(  
                process,
                batched=True,
                num_proc=num_proc,
                load_from_cache_file=False,
                desc="Tokenizing dataset...",
                batch_size=batch_size,
                remove_columns= remove_columns
        )
    return dataset