public
Created

update_db.py command, not working

  • Download Gist
update_db.py
Python
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
from django.db import models
from django.core.management.base import BaseCommand, CommandError
from IrishSchedule.models import Department, Course, Section
import mechanize
import requests
from BeautifulSoup import BeautifulSoup
#import os
#os.environ['DJANGO_SETTINGS_MODULE'] = "myproject.settings"
 
class Command(BaseCommand):
 
def handle(self, *args, **options):
 
process_shit();
 
def process_shit():
class DepartmentPy:
def __init__(self, dept, key):
self.deptname = dept
self.deptkey = key
self.CourseList = []
 
def add(self, course):
self.CourseList.append(course)
 
# def getInfo(self, br):
# #br.open("https://was.nd.edu/reg/srch/ClassSearchServlet")
# print self.key
# br.select_form(nr=0)
# forms = [f for f in br.forms()]
# form = forms[0]
# #form["SUBJ"].clear
# control = form.find_control("SUBJ")
# #control.value = [None]
 
 
# # Deselect all items in the select, so not to error out
# for item in control.items:
# item.selected = False
 
# form["SUBJ"] = [self.key]
 
# response = br.submit()
# html = response.read()
 
# self.addCourses(html)
 
def postRequest(self):
print self.deptkey
payload = {'TERM':'201220', 'DIVS': 'A', 'CAMPUS':'M', 'SUBJ':self.deptkey, 'ATTR':'0ANY', 'CREDIT':'A' }
r = requests.post("https://was.nd.edu/reg/srch/ClassSearchServlet", data=payload)
#print r.text
#print r.status_code
 
html = r.text
if(len(html) > 30000):
self.addCourses(html)
 
def addCourses(self, html):
soup = BeautifulSoup(html)
 
result_table = soup.find('table', {'id': 'resulttable'})
body_result = result_table.find('tbody')
 
#select = soup.find('select', {'name': 'SUBJ'})
prev = None
 
for row in body_result.findAll('tr'):
cells = row.findAll('td')
 
section = cells[0].text.partition('-')[2]
course = cells[0].text.partition('-')[0]
 
section = re.sub('[^0-9]', '', section)
course = re.sub('[^A-Za-z0-9]+', '', course)
 
title = cells[1].text
credits = cells[2].text
max_spots = cells[4].text
open_spots = cells[5].text
crn = cells[7].text
instructor = cells[9].text
time = cells[10].text
location = cells[13].text
 
if title == prev:
s = SectionPy(section, open_spots, max_spots, crn, instructor, time, location)
sect = Section(section_num = section, open_spots = open_spots, max_spots = max_spots, crn = crn, instructor = instructor, time = time, location = time, Course = cour)
sect.save()
self.CourseList[-1].add(s)
 
else:
c = CoursePy(title, credits, course)
s = SectionPy(section, open_spots, max_spots, crn, instructor, time, location)
 
cour = Course(title = title, credits = credits, course = course, department = d)
cour.save()
sect = Section(section_num = section, open_spots = open_spots, max_spots = max_spots, crn = crn, instructor = instructor, time = time, location = time, Course = cour)
sect.save()
c.add(s)
self.add(c)
 
if title is not None:
prev = title
 
class CoursePy:
def __init__(self, name, credit, coursenum):
self.title = name
self.credits = credit
self.course_no = coursenum
self.sections = []
 
def add(self, section):
self.sections.append(section)
 
class SectionPy:
def __init__(self, section_num, open_spots, max_spots, crn, instructor, time, location):
self.section_num = section_num
self.open_spots = open_spots
self.max_spots = max_spots
self.crn = crn
self.instructor = instructor
self.time = time
self.location = location
 
if __name__ == "__main__":
departments = []
br = mechanize.Browser()
br.set_handle_equiv(True)
#br.set_handle_gzip(True)
br.set_handle_redirect(mechanize.HTTPRedirectHandler)
br.set_handle_referer(True)
 
br.open("https://was.nd.edu/reg/srch/ClassSearchServlet")
 
forms = [f for f in br.forms()]
subj_form = forms[0]
 
control = subj_form.find_control("SUBJ")
 
for item in control.items:
#item.selected = False
#print item.attrs['label'] + item.name
dept = DepartmentPy(item.attrs['label'], item.name)
d = Department(deptname=item.attrs['label'], deptkey=item.name)
d.save()
#dept.getInfo(br)
dept.postRequest()
#pickled = jsonpickle.encode(dept)
#f = open('data.json', 'a')
#f.write(simplejson.dumps(simplejson.loads(pickled),indent=4))
#print pickled
#print json.dumps(dept)
#pprint(vars(dept.CourseList[1].sections[0]), indent=4)
departments.append(dept)
item.selected = False
#yaml.dump(dept, sys.stdout)

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.