marks_scraper_full.py

import requests
from bs4 import BeautifulSoup
import copy



class ID:
	def __init__(self):
		self.let1 = ord('A')
		self.let2 = ord('A')
		self.dig3 = 0
		self.dig4 = 0
		self.dig5 = 1

	def __str__(self):
		return chr(self.let1) + chr(self.let2) + str(self.dig3) + str(self.dig4) + str(self.dig5)

	def __repr__(self):
		return self.__str__()

	def next(self):
		new_id = copy.deepcopy(self)
		new_id.dig5 = new_id.dig5 + 1

		if new_id.dig5 > 9:
			new_id.dig5 = 0
			new_id.dig4 = new_id.dig4 + 1

		if new_id.dig4 > 9:
			 new_id.dig4 = 0
			 new_id.dig3 = new_id.dig3 + 1

		if new_id.dig3 > 9:
			new_id.dig3 = 0
			new_id.let2 = new_id.let2 + 1

		if new_id.let2 > ord('Z'):
			new_id.let2 = ord('A')
			new_id.let1 = new_id.let1 + 1

		return new_id

if __name__ == "__main__":

	current_id = ID()
	num_iters = 26 * 26 * 10 * 10 * 10

	continuous_exceptions = 0 #number of expections that occur continuiously. if > 100, break
	had_prev_exception = False

	for i in xrange(0, num_iters):

		payload = {'reg': str(current_id), 'frm_tokens' :0.6877558 }
		doc =  requests.post("http://karresults.nic.in/cetresult2014.asp", data=payload).text

		current_id = current_id.next()

		#file("a.html", "w").write(doc)
		
		try:
			soup = BeautifulSoup(doc)
			soup.prettify()

			number = soup.select('td[width=146]')[0]
			print "Number " + number.font.text

			name = soup.select('td[width=224]')[0]
			print "Name" + name.text

			marks_table = soup.select('table[border="1"]')[0]

			#one next_siblingfor whitespace, other for actual sibling
			phys_row = marks_table.tr.next_sibling.next_sibling
			chem_row = phys_row.next_sibling.next_sibling
			maths_row = chem_row.next_sibling.next_sibling

			try:
				phys_marks = phys_row.td.next_sibling.next_sibling.next_sibling.text
			except Exception, e:
				phys_marks = "Unknown"
			print "Physics"  + phys_marks


			try:
				chem_marks = chem_row.td.next_sibling.next_sibling.next_sibling.next_sibling.text
			except Exception, e:
				chem_marks = "Unknown"
			print "Chem"  + chem_marks


			try:
				math_marks = maths_row.td.next_sibling.next_sibling.next_sibling.next_sibling.text
			except Exception, e:
				math_marks = "Unknown"
			print "Maths"  + math_marks


			ranks_table = soup.select('table[border="0"]')[2]
			
			try:
				engg_rank = ranks_table.tbody.tr.td.table.tr.next_sibling.next_sibling.td.next_sibling.next_sibling.text
			except Exception, e:
				engg_rank = "Unknown"
			print "Engg rank: " + engg_rank

			print "-------------\n\n"

			had_prev_exception = False

		except Exception, e:
			print e
			print "done will all numbers"

			if had_prev_exception:
				continuous_exceptions = continuous_exceptions + 1
			else:
				continuous_exceptions = 0
				had_prev_exception = True

			if continuous_exceptions > 100:
				break