Working with OOV Tokens

oov_tokens.py

from tensorflow.keras.preprocessing.text import Tokenizer

#Let's add custom sentences 
sentences = [
    "Apples are red",
    "Apples are round",
    "Oranges are round",
    "Grapes are green"
]

#Tokenize the sentences using OOV
myTokenizer = Tokenizer(num_words=100, oov_token="<some-word>")
myTokenizer.fit_on_texts(sentences)
print(myTokenizer.word_index)

# Unseen Words
test_data = [
    'Grapes are sour but oranges are sweet',
]

test_seq = myTokenizer.texts_to_sequences(test_data)
print("\nTest Sequence = ", test_seq, " => ", [x for x in myTokenizer.sequences_to_texts_generator(test_seq)])