repro_poss_memory_leak.py

from torchtune.models.llama3 import llama3_tokenizer
from torchtune.datasets import instruct_dataset

tokenizer = llama3_tokenizer("./model/original/tokenizer.model")
dataset = instruct_dataset(
    tokenizer=tokenizer,
    source="TIGER-Lab/WebInstructSub",
    template="torchtune.data.AlpacaInstructTemplate",
    column_map={
        "instruction": "question",
        "output": "answer",
    },
    max_seq_len=3072,    
    packed=True,
    split="train",
)