debug tokenizer of lmsys/vicuna-13b-v1.3

debug-tokenizer-vicuna-13b.py

#
# debug tokenizer of lmsys/vicuna-13b-v1.3
#

from transformers import AutoTokenizer


tokenizer = AutoTokenizer.from_pretrained("lmsys/vicuna-13b-v1.3")

def encode_decode(string: str):
    ids = tokenizer.encode(string, add_special_tokens=False)
    decoded = tokenizer.decode(ids)
    u8 = decoded.encode('utf-8')

    print(ids, decoded, u8)

def encode_decode_each_id(string: str):
    ids = tokenizer.encode(string, add_special_tokens=False)
    for i in ids:
        decoded = tokenizer.decode([i])
        u8 = decoded.encode('utf-8')

        print(i, decoded, u8)


string = '神々の黄昏'
encode_decode(string)
encode_decode_each_id(string)