Adding Padding to the tokenization process

tokenizer_padding.py

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

#Let's add custom sentences 
sentences = [
    "Apples are red",
    "Apples are round",
    "Oranges are round",
    'Grapes are sour, oranges are sweet'
]

#Tokenize the sentences
myTokenizer = Tokenizer(num_words=100)
myTokenizer.fit_on_texts(sentences)
sequences = myTokenizer.texts_to_sequences(sentences)

#Padding
padded = pad_sequences(sequences, maxlen=len(sentences[3].split(" ")))

#Display the output
print("\nWord Index = " , myTokenizer.word_index)
print("\nSequences = " , sequences)
print("\nPadded Sequences:")
print(padded)
print("\nOriginal Sentences: \n",[x for x in myTokenizer.sequences_to_texts_generator(padded)])

#Pre and Post Padding
padded = pad_sequences(sequences)
print("\nPre Padded Sequences:")
print(padded)
padded = pad_sequences(sequences, padding="post")

print("\nPost Padded Sequences:")
print(padded)