gistfile1.py

import time
import re
import urllib2
import datetime
import sys
import os, errno
from BeautifulSoup import BeautifulSoup
from pymongo import MongoClient
import nltk   

reload(sys)
sys.setdefaultencoding('utf-8')

def visible(element):
    if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
        return False
    elif re.match('<!--.*-->', str(element)):
        return False
    return True

def getData(source):
    
    response = urllib2.urlopen(source[1])
    if source[2] == "utf-8":
        html = response.read().decode().encode("utf-8")
    else:
        html = response.read().decode(source[2]).encode("utf-8")
    soup = BeautifulSoup(html)
    pageText = soup.findAll(text=True)
    visible_texts = filter(visible, pageText)
    clearRaw =''.join(visible_texts).replace("&nbsp;", " ").replace("&gt;","").lower()
    return clearRaw
    


timestampForProcess =  time.time()
client = MongoClient('localhost', 27017)
portalData = client.portalData

portals = [["terra","http://terra.com.br/","utf-8"],["uol","http://uol.com.br","utf-8"],["folha","http://folha.com.br","cp1252"],["estadao","http://estadao.com.br","utf-8"]]


for portal in portals:
    print "processing "+portal[0]+" from "+ portal[1]
    data = getData(portal)
    gram  = nltk.word_tokenize(data)
    bgram = nltk.bigrams(gram)
    tgram = nltk.trigrams(gram)
    portalData.portal.insert({"name":portal[0],"momentum":timestampForProcess, "source": data,"gram":gram,"bgram":bgram,"tgram":tgram});