gened-search

# -*- coding: utf-8 -*-

from BeautifulSoup import BeautifulSoup
import json
import os
import requests

def max(x,y):
    return x if x > y else y

def main():
    cnt = 1
    gened = {'HU': [], 'SO': [], 'SC': [], 'IN': [], 'UNKNOWN': []}
    filenum = 6757
    logs = []
    for dept in range(1,60):
        for i in range(1,1000):
            os.system('clear')
            print "processing ", cnt, "/", filenum, "(", cnt*100/filenum, "%)"
            filename = "%02d-%03d.CourseScheduleDtlNewServlet.html" % (dept, i)
            print filename
            print "-"*80
            for j in range(len(logs)-1,max(len(logs)-20,-1),-1):
                print logs[j]
            try:
                with open(filename) as f:
                    cnt += 1
                    html = reduce(lambda x,y: x+y,f.readlines())
                    soup = BeautifulSoup(html)
                    tables = soup.findAll('table')

                    timetable = tables[4]
                    timetable_text = timetable.text
                    if "GENED-HU" in timetable_text or "GENED - HU" in timetable_text:
                        category = "HU"
                    elif "GENED-SO" in timetable_text or "GENED - SO" in timetable_text:
                        category = "SO"
                    elif "GENED-SC" in timetable_text or "GENED - SC" in timetable_text:
                        category = "SC"
                    elif "GENED-IN" in timetable_text or "GENED - IN" in timetable_text:
                        category = "IN"
                    elif "GENED" in timetable_text:
                        category = "UNKNOWN"
                    else:
                        continue

                    trs = tables[1].findAll('tr')
                    subject_tr = trs[3].text.split("  ")
                    course_id = subject_tr[0]
                    subject = subject_tr[1]

                    if '<' in subject and '>' in subject:
                        idx = subject.find('<')
                        subject = subject[:idx]

                    course = {'course_id': course_id, 'subject': subject}
                    print course

                    gened[category].append(course)
                    logs.append(course)

            except IOError:
                break
    w = open("geneddict.py","w")
    w.write("geneddict = ")
    w.write(json.dumps(gened))

if __name__ == "__main__":
    main()