gistfile1.txt

from tiktoken._educational import SimpleBytePairEncoding
from tiktoken.load import dump_tiktoken_bpe

# copy-paste from torchtune/modules/tokenizers/_tiktoken.py
pattern = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""

# in_path corresponds to https://github.com/google/sentencepiece/blob/master/data/botchan.txt
with open(in_path, 'r') as f:
    data = f.read()
enc = SimpleBytePairEncoding.train(data, vocab_size=2000, pat_str=pattern)
dump_tiktoken_bpe(enc.mergeable_ranks, out_path)