| genc = pickle.loads(open(settings.DATA_FOLDER + id + "/GenderClassify.pkl").read()) | |
| from urlparse import urlparse | |
| def raw2feature(line): | |
| r = [] | |
| try : | |
| dictf = ["hour" , "category_id" , "cookie_pta" , "timestamp" , "url" , "country" , "city" , "resolution" , "browser" , "browser_version" , "os" , "os_version" , "device_model" , "device_marketing" , "device_brand" , "search_keyword" , "referrer_host"] | |
| parsedline = dict() | |
| index = 0 |
http://machinelearningmastery.com/an-introduction-to-feature-selection/ http://machinelearningmastery.com/feature-selection-in-python-with-scikit-learn/ http://stats.stackexchange.com/questions/40576/is-using-the-same-data-for-feature-selection-and-cross-validation-biased-or-not
http://www.slideshare.net/irecsys/matrix-factorization-in-recommender-systems
| from pyspark.mllib.feature import ChiSqSelector | |
| model = ChiSqSelector(5000).fit(sc.parallelize(lc)) | |
| chi_l = l.mapValues(lambda x : model.transform (x)) | |
| print chi_l.first() | |
| 出現以下的訊息 | |
| --------------------------------------------------------------------------- | |
| Exception Traceback (most recent call last) |
| def cuttext(text): | |
| stop_words = stop_sc.value | |
| import jieba | |
| from jieba import analyse | |
| jieba.load_userdict("../data/new.dict_all") | |
| bag_word = dict() | |
| for word in jieba.cut(text,cut_all=False): | |
| if word in stop_words : continue | |
| if len(word) == 1 and word != '$' : continue | |
| else: |
| just |