Llama tokenizer test

tk.py

import sys

from llama import Tokenizer

tokenizer_path = '../llama.cpp/models/tokenizer.model'
tokenizer = Tokenizer(model_path=tokenizer_path)

text = sys.argv[1]

tokens = tokenizer.encode(text, bos=True, eos=False)
print(tokens)

for tid in tokens:
    tk = tokenizer.decode([tid])
    print(f"{tid:>6} -> '{tk}'")

print(tokenizer.decode(tokens))