Crawl webpage to restore success records , calculate today's WCG points and list every host's points.

crawl.py

#!/usr/bin/python
# -*- coding:utf-8 -*-

from datetime import datetime
import sqlite3
import cookielib
import requests
from BeautifulSoup import *

device_final_score_url = "https://secure.worldcommunitygrid.org/ms/viewBoincResults.do?filterDevice=0&filterStatus=4&projectId=-1&pageNum=%s&sortBy=returnedTime"
data = {"j_username": "your_wcg_name", "j_password": "your_wcg_password"}

conn = sqlite3.connect('wcg_points.db')
s = requests.session()

################
# Utilities
################
def perse_date(date_str):
    date = datetime.strptime(date_str, '%m/%d/%y %H:%M:%S')
    return date

################
# SQLite Related
################
def create_database():
    c = conn.cursor()
    # Create table
    c.execute('''CREATE TABLE points
                     (result_name text, device_name text, result_time text, cpu_time real, point real)''')

def already_exist(result_name, device_name):
    c = conn.cursor()
    args = (result_name, device_name)
    result = c.execute('SELECT * FROM points WHERE result_name=? AND device_name=?', args)
    data = c.fetchall()
    if len(data) != 0:
        return True
    return False

def insert_record(args):
    c = conn.cursor()
    c.execute('INSERT INTO points VALUES (?,?,?,?,?)', args)

###############
# Print Realted
###############
def print_today_score():
    c = conn.cursor()
    today = str(datetime.now().date())
    today_sum = 0
    for record in  c.execute('SELECT * FROM points WHERE result_time=?',(today,)):
        today_sum += record[4]
    print today + ': ' + str(today_sum) + ' ' + str(today_sum*7)

def print_today_device_scores():
    c = conn.cursor()
    c2 = conn.cursor()
    today = str(datetime.now().date())
    print 'Device score list:'
    for device_name_record in  c.execute('SELECT distinct device_name FROM points WHERE result_time=? ORDER BY device_name', (today,)):
        device_score = 0
        device_name = device_name_record[0]
        for score in c2.execute('SELECT point FROM points WHERE result_time=? and device_name=?', (today, device_name)):
            device_score += score[0]
        print '\t\t%s: %s' % (device_name, device_score)

#################
# Network Related
#################
def login_wcg():
    url = "https://secure.worldcommunitygrid.org/j_security_check"
    s.post(url, data)

def get_score_sum_from_page(page_num):
    url = device_final_score_url % str(page_num)
    text = s.get(url).text
    soup = BeautifulSoup(text)
    
    results = soup.findAll("tr", {"height": "54"})
    if len(results) == 0:
        print 'You should login again.'
        return False
    for result in results:
        result_name = result.contents[1].contents[0].contents[0].contents[0]
        device_name = result.contents[3].contents[0].contents[0]
        date = perse_date(result.contents[9].contents[0].contents[0])
        cup_time = result.contents[11].contents[0].contents[0]
        cup_time = cup_time[cup_time.find('/')+2:]
        score = result.contents[13].contents[0].contents[0]
        score = score[score.find('/')+7:]
        print result_name, device_name
        if already_exist(result_name, device_name):
            return False
        else:
            insert_record((result_name, device_name, str(date.date()), float(cup_time), float(score)))
    return True


if __name__ == "__main__":

    if os.path.isfile('wcg_points.db') == False:
        create_database()
    
    print 'Start trying login.'
    login_wcg()
    
    for i in range(1,47):
        print 'Page ' + str(i) + ' start.'
        if get_score_sum_from_page(i) == False:
            print 'Meet a saved record'
            break;
    conn.commit()
    
    print_today_score()
    print_today_device_scores()
    
    conn.close()