Process text

processing.py

import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

def process_txt(input, stemm = False,lemm = True):

    ### Clean input data
    processed_text = clean(input)

    ### Tokenization
    processed_text = word_tokenize(processed_text)

    ### remove stop words
    processed_text = [word for word in processed_text if word not in stopwords.words('english')]

    ### Stemming
    if stemm == True:
      ps = nltk.stem.porter.PorterStemmer()
      processed_text = [ps.stem(word) for word in processed_text]

    ### Lemmatization
    if lemm == True:
      lem = nltk.stem.wordnet.WordNetLemmatizer()
      processed_text = [lem.lemmatize(word) for word in processed_text]

    text = " ".join(processed_text)
    
    return text