parse_html_and_get_content.py

def cuttext(text):
    stop_words = stop_sc.value
    import jieba
    from jieba import analyse
    jieba.load_userdict("../data/new.dict_all")
    bag_word = dict()
    for word in jieba.cut(text,cut_all=False):
        if word in stop_words : continue
        if len(word) == 1 and word != '$' : continue
        else:
            if word in bag_word :
                bag_word[word] = bag_word[word] + 1
            else :
                bag_word[word] = 1
    return bag_word
def analysis_content(content):
    print '*',
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(content)
#     text_len = len(soup.getText().replace('\n','').replace('\r','').replace(' ','').replace('\t',''))
    cuts = cuttext(soup.getText())
    return cuts