Preprocessing

preprocess.py

import pandas as pd 
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer

'''
    Return array of word
'''
def preprocess(teks):
    # lowercase
    teks = teks.lower()
    
    # Token + remove punctuation 
    tokenizer = RegexpTokenizer(r'\w+') # \w : angka atau huruf kecil & besar termasuk underscore | + : 1 atau lebih, sebanyak-banyaknya
    teks = tokenizer.tokenize(teks)

    # Stopwords
    stop_words = set(stopwords.words('english')) 
    word_tokens = teks

    teks = [w for w in word_tokens if not w in stop_words] 
    # sama aja kayak yang diatas
    # teks = [] 
    # for w in word_tokens: 
    #     if w not in stop_words: 
    #         teks.append(w) 

    # Stemming pake porter stemmer
    ps = PorterStemmer() # inisiasi porter stemmer 
    teks = [ps.stem(kata) for kata in teks] 

    return teks

# ambil data di csv pake pandas
df = pd.read_csv("data/dataset.csv")
# iterasi data dari csv
for index, row in df.iterrows():
    print(preprocess(row["news"]))