Create CLIP Tokenizer

tokenizer.py

from tokenizers import (
    models,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)
from pathlib import Path
import os
from transformers import PreTrainedTokenizerFast

paths = [str(x) for x in Path("../sacorpus/Sankrit_Corpus/").glob("**/*.txt")]

tokenizer = Tokenizer(models.BPE(end_of_word_suffix="</w>"))
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
    [pre_tokenizers.ByteLevel(add_prefix_space=False)])
trainer = trainers.BpeTrainer(vocab_size=25000, special_tokens=["<|endoftext|>"], min_frequency=1,
                              show_progress=True)

tokenizer.train(files=paths, trainer=trainer)

tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>"
)

wrapped_tokenizer.save_pretrained("cliptok")
tokenizer.save("cliptok.json")