Data Aggregator [ currently only hacker news ]

ycombinator_db.py

#!/usr/bin/python

from mongoengine import *
import json
import urllib2
import sys

class Post(Document):
    item_id = IntField(required=True,unique=True)
    title = StringField(required=True)
    url = StringField(required=True)
    score = IntField(required=True)
    comment_count = IntField(required=True)

class DBHelper:
    def __init__(self):
        connect('ycombinator')

    def add(self,field):
        if "score" in field and \
            len(field["comments"].split()) > 1 :
            posts = Post.objects(item_id = int(field["item_id"]))

            if len(posts) == 0:
                post = Post()
                print "new"
            else:
                post = posts[0]
                print "updated"

            if post.score < int(field["score"].split()[0]):
                post.item_id = int(field["item_id"])
                post.title = field["title"]
                post.url = field["url"]
                post.score = int(field["score"].split()[0])
                post.comment_count = int(field["comments"].split()[0])

            post.save()

    def automate(self):
        j = json.loads(urllib2.urlopen("http://hndroidapi.appspot.com/news/format/json/page/?appid=ad").read())
        for i in j["items"]:
            self.add(i)

    def listAll(self):
        for i in Post.objects:
            print i

    def clean(self):
        pass

if __name__=="__main__":
    dbhelper = DBHelper()
    if sys.argv[1] == "ADD":
        dbhelper.automate()
    elif sys.argv[1] == "TEST":
        print Post.objects(item_id=4101992)[0].title
    else:
        dbhelper.listAll()