sequencing.py

from tensorflow.keras.preprocessing.sequence import pad_sequences

sequences = tokenizer.texts_to_sequences(sentences)

padded = pad_sequences(sequences, maxlen=5)
print("\nThe Word Index = " , word_index)
print("\nThe Sequences = " , sequences)
print("\nThe Padded Sequences:")
print(padded)


# Test data with sentences we have never seen 
test_data = [
    'i really love fish',
    'you love to eat fish so much'
]

test_seq = tokenizer.texts_to_sequences(test_data)
print("\nTest Sequence = ", test_seq)

padded = pad_sequences(test_seq)
print("\nPadded Test Sequence: ")
print(padded)