hbr-article-frequency-nltk

import os 
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer

porter_stemmer = PorterStemmer()

def get_dir():
    dir_path = os.path.dirname(os.path.realpath(__file__))
    return dir_path

def get_frequency(filename):

    content = ''
    with open(filename, encoding="utf8") as f:
        for line in f:
            #print(line, end = '') 
            content = content + line.lower()

    #print(content)

    tokens = word_tokenize(content)

    freq = nltk.FreqDist(tokens)

    clean_tokens = [] 

    stop_words = stopwords.words('english')

    for token in tokens:

        # ignore string less than 4 characters
        if(len(token) < 4):
            continue

        if(token in stopwords.words('english')):
            continue

        token = porter_stemmer.stem(token)

        clean_tokens.append(token)

    freq = nltk.FreqDist(clean_tokens)
    freq.plot(20, cumulative=False)

            
get_frequency(get_dir() + "/article1.txt")