Get original words of tokens in HF Tokenizers

get_words_of_tokens.py

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
text = "It 's a pre-tokenized , silly sentence !"
words = text.split()
encoded = tokenizer(words, is_split_into_words=True)

for token, wordid in zip(encoded.tokens(), encoded.word_ids()):
    if wordid is not None:
        print(token, words[wordid])

"""
# Output (subword unit - original word)
It It
' 's
s 's
a a
pre pre-tokenized
- pre-tokenized
token pre-tokenized
##ized pre-tokenized
, ,
silly silly
sentence sentence
! !
"""