Skip to content

Instantly share code, notes, and snippets.

@iheanyi
Created December 6, 2012 16:39
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save iheanyi/4225886 to your computer and use it in GitHub Desktop.
Save iheanyi/4225886 to your computer and use it in GitHub Desktop.
update_db.py command, not working
from django.db import models
from django.core.management.base import BaseCommand, CommandError
from IrishSchedule.models import Department, Course, Section
import mechanize
import requests
from BeautifulSoup import BeautifulSoup
#import os
#os.environ['DJANGO_SETTINGS_MODULE'] = "myproject.settings"
class Command(BaseCommand):
def handle(self, *args, **options):
process_shit();
def process_shit():
class DepartmentPy:
def __init__(self, dept, key):
self.deptname = dept
self.deptkey = key
self.CourseList = []
def add(self, course):
self.CourseList.append(course)
# def getInfo(self, br):
# #br.open("https://was.nd.edu/reg/srch/ClassSearchServlet")
# print self.key
# br.select_form(nr=0)
# forms = [f for f in br.forms()]
# form = forms[0]
# #form["SUBJ"].clear
# control = form.find_control("SUBJ")
# #control.value = [None]
# # Deselect all items in the select, so not to error out
# for item in control.items:
# item.selected = False
# form["SUBJ"] = [self.key]
# response = br.submit()
# html = response.read()
# self.addCourses(html)
def postRequest(self):
print self.deptkey
payload = {'TERM':'201220', 'DIVS': 'A', 'CAMPUS':'M', 'SUBJ':self.deptkey, 'ATTR':'0ANY', 'CREDIT':'A' }
r = requests.post("https://was.nd.edu/reg/srch/ClassSearchServlet", data=payload)
#print r.text
#print r.status_code
html = r.text
if(len(html) > 30000):
self.addCourses(html)
def addCourses(self, html):
soup = BeautifulSoup(html)
result_table = soup.find('table', {'id': 'resulttable'})
body_result = result_table.find('tbody')
#select = soup.find('select', {'name': 'SUBJ'})
prev = None
for row in body_result.findAll('tr'):
cells = row.findAll('td')
section = cells[0].text.partition('-')[2]
course = cells[0].text.partition('-')[0]
section = re.sub('[^0-9]', '', section)
course = re.sub('[^A-Za-z0-9]+', '', course)
title = cells[1].text
credits = cells[2].text
max_spots = cells[4].text
open_spots = cells[5].text
crn = cells[7].text
instructor = cells[9].text
time = cells[10].text
location = cells[13].text
if title == prev:
s = SectionPy(section, open_spots, max_spots, crn, instructor, time, location)
sect = Section(section_num = section, open_spots = open_spots, max_spots = max_spots, crn = crn, instructor = instructor, time = time, location = time, Course = cour)
sect.save()
self.CourseList[-1].add(s)
else:
c = CoursePy(title, credits, course)
s = SectionPy(section, open_spots, max_spots, crn, instructor, time, location)
cour = Course(title = title, credits = credits, course = course, department = d)
cour.save()
sect = Section(section_num = section, open_spots = open_spots, max_spots = max_spots, crn = crn, instructor = instructor, time = time, location = time, Course = cour)
sect.save()
c.add(s)
self.add(c)
if title is not None:
prev = title
class CoursePy:
def __init__(self, name, credit, coursenum):
self.title = name
self.credits = credit
self.course_no = coursenum
self.sections = []
def add(self, section):
self.sections.append(section)
class SectionPy:
def __init__(self, section_num, open_spots, max_spots, crn, instructor, time, location):
self.section_num = section_num
self.open_spots = open_spots
self.max_spots = max_spots
self.crn = crn
self.instructor = instructor
self.time = time
self.location = location
if __name__ == "__main__":
departments = []
br = mechanize.Browser()
br.set_handle_equiv(True)
#br.set_handle_gzip(True)
br.set_handle_redirect(mechanize.HTTPRedirectHandler)
br.set_handle_referer(True)
br.open("https://was.nd.edu/reg/srch/ClassSearchServlet")
forms = [f for f in br.forms()]
subj_form = forms[0]
control = subj_form.find_control("SUBJ")
for item in control.items:
#item.selected = False
#print item.attrs['label'] + item.name
dept = DepartmentPy(item.attrs['label'], item.name)
d = Department(deptname=item.attrs['label'], deptkey=item.name)
d.save()
#dept.getInfo(br)
dept.postRequest()
#pickled = jsonpickle.encode(dept)
#f = open('data.json', 'a')
#f.write(simplejson.dumps(simplejson.loads(pickled),indent=4))
#print pickled
#print json.dumps(dept)
#pprint(vars(dept.CourseList[1].sections[0]), indent=4)
departments.append(dept)
item.selected = False
#yaml.dump(dept, sys.stdout)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment