sklearn model test

sklearn-hash-rf.py

from revscoring.utilities import util
from revscoring.dependencies import solve
from revscoring.datasources import revision_oriented
import yamlconf

from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import MultiLabelBinarizer

#features=hbow
observations = read_observations(open('enwiki.labeled_wikiprojects.w_text.json'))
label_name = 'mid_level_categories'
#value_labels = [(list(solve(features, cache=ob['cache'])), ob[label_name])
#               for ob in observations]
#labels, lw, pr = util.read_labels_and_population_rates(None, 
#                                            None, None, 'labels-config.yaml')
#
#model = RandomForest(features, labels, multilabel=True)
#model.train(value_labels)

hv = HashingVectorizer()
mlb = MultiLabelBinarizer()
labels = []
X = []
for ob in observations:
    labels.append(ob[label_name])
    X.append(solve(english.stopwords.revision.datasources.non_stopwords,
                   cache={revision_oriented.revision.text: ob['text']}))
    X[-1] = " ".join(X[-1])
label_matrix = mlb.fit_transform(labels)

print("Preprocessing done, classifying...")

clf = RandomForestClassifier()
t1 = time.time()
clf.fit(hv.transform(X), label_matrix)
t2 = time.time()
print("Time:{}".format(t2-t1))