cria X.pkl

parseX.py

import os
import re
import pickle
from nltk.stem import RSLPStemmer
from sklearn.feature_extraction.text import TfidfVectorizer

def pre_process():
    vanilla = u'[^\u0041-\u005A \
                  \u0061-\u007A \
                  \u00C0-\u00D6 \
                  \u00D8-\u00F6 \
                  \u00F8-\u00FF \
                  \u0100-\u017F \
                  \u0020]'
    regex = re.compile(vanilla)
    st = RSLPStemmer()
    basepath = '/caminho/ate/arquivos/CSVs/' # altere conforme necessario
    flist = [fname for fname in os.listdir(basepath) if '.csv' in fname]
    counter = 0
    for fname in flist:
        fbuffer = open(basepath + fname, mode = 'rb')
        for line in fbuffer:
            counter += 1
            idx1 = line.index(',')
            classe = line[:idx1]
            if classe[0] == '2':
                continue
            print counter
            subline1 = line[idx1 + 1:]
            idx2 = subline1.index(',')
            descricao = subline1[idx2 + 1:]
            lowercased = descricao.decode('utf8').lower()
            regexed = regex.sub(' ', lowercased)
            tokenized = regexed.split()
            singularized = [st.apply_rule(token, 0) for token in tokenized]
            remerged = ''
            for word in singularized:
                if len(word) > 1:
                    remerged += word + ' '
#            print remerged.encode('utf-8')
#            raw_input()
            yield remerged

tfidf_maker = TfidfVectorizer(lowercase = False,
                              min_df = 2,
                              norm = 'l2',
                              smooth_idf = True)
tfidf = tfidf_maker.fit_transform(pre_process())
pickle.dump(tfidf, open('X.pkl', 'wb'))