bag_of_words.py

def pickle_words_features(microblogType):
    microblogs = Microblog.objects(microblogType=microblogType)

    all_words = []
    for microblog in microblogs:
        all_words.extend(microblog.words)
    all_words = nltk.FreqDist(all_words)

    words_features = list(all_words.keys())

    with open(WORDS_FEATURES_PATH, 'wb') as output_file:
        pickle.dump(words_features, output_file)


def feature_filter(document, words_features):
    words = set(document)
    features = {}
    for w in words_features:
        features[w] = (w in words)

    return features


def get_feature_set(microblogType):

    microblogs = Microblog.objects(microblogType=microblogType)

    words_features = get_words_features_pickle()

    feature_sets = [(feature_filter(microblog.words, words_features), microblog.polarity) for microblog in microblogs]

    return feature_sets