Blog - NLP - Word Tokenization Example

WordTokenization.py

from nltk.tokenize import word_tokenize
from nltk.tokenize import wordpunct_tokenize

test_sentence = "Hi Mr. Sam, today's a good day to learn NLP. It's a well-known field of study."

#Method 1 - Without using the NLTK - Splitting at white spaces
words = test_sentence.split()
print words

#Method 2 - Using word_tokenize of NLTK
words = word_tokenize(test_sentence)
print words

#Method 2 - Using wordpunct_tokenize of NLTK
words = wordpunct_tokenize(test_sentence)
print words