text_preprocessing.py

def text_preprocessing(df):
  #Remove punctuations
  df['clean_text'] = df['comment_text'].apply(lambda x:remove_punctuation(x))
  #make lower case
  df['clean_text_lower']= df['clean_text'].apply(lambda x: x.lower())
  #toenize the string
  df['text_tokenied']= df['clean_text_lower'].apply(lambda x: tokenization(x))
  #remove stop words
  df['text_tokenized_no_stopwords']= df['text_tokenied'].apply(lambda x:remove_stopwords(x))   
  #lemmatize
  df['text_lemmatized']=df['text_tokenized_no_stopwords'].apply(lambda x:lemmatizer(x))  
  return df