Text Tokenization using NLTK

nltk_tokenize.py

# Text Tokenization using NLTK

from nltk.tokenize import sent_tokenize, \
    word_tokenize, WordPunctTokenizer

in_text = 'Use this option to select your font. The Show only monospaced fonts option if selected shortens the list of available fonts.'

# Sentence Tokenization
print(sent_tokenize(in_text))

# Word Tokenization

print(word_tokenize(in_text))

# Word Punct Tokenization
print(WordPunctTokenizer().tokenize(
    in_text))