davelester

Website code: https://svn.apache.org/repos/asf/mesos/site
Requirements: Ruby 2+

## Setup

		gem install bundler
		bundle install

## Generating the site
To generate the site one only needs to run `rake` after performing the setup

davelester

import urllib.robotparser
import urllib.request
import urllib.parse
from bs4 import BeautifulSoup
import time
import re
import string
from collections import deque
import os
import fileinput

davelester

year | # syllabi
2001  - 13729
2002  - 26001
2003  - 34601
2004  - 41163
2005  - 39927
2006  - 39599
2007  - 42413
2008  - 42992
2009  - 18057

davelester

http://web.mit.edu/firstyear/2006/rightnow/folder.html
http://classof2006.uchicago.edu/mailings/ies/course_descriptions.cfm
http://www.unc.edu/depts/comm/CourseSyllabi/Fall%202002/029%20Comm%20063.1%20Syllabus.doc
http://www.alvernia.edu/2006/schwangersyllabus.pdf
http://www.alvernia.edu/2006/css.htm
http://www.uncp.edu/home/albrecht/pspa101_%2006f02.htm
http://www4.nas.edu/cls/ijhome.nsf/44bf87db309563a0852566f2006d63bb/796c3396bb2d51e9852568dd005fe4e2?OpenDocument
http://parents2006.sacredheart.edu/FS101.html
http://pirate.shu.edu/~zinicode/2006/edst2006syllabus.html
http://www.du.edu/~ddelffs/advancedstand/mysyllabusENGG%200611.html

davelester

http://sobek.colorado.edu/SOC/syllabi_files/SOCY2004-AR.htm
http://www.gmu.edu/depts/psychology/homepage/syllabi/syllfall01/372004.doc
http://studyat.anu.edu.au/DisplayCourse.asp?role=1&course=HIST2004
http://www.majbill.vt.edu/history/hirsh/hist2004/hist2004syl-f00.htm
http://reach.ucf.edu/~dep2004/syllabus.html
http://www.majbill.vt.edu/history/hirsh/hist2004/hist2004syl-f00.pdf
http://www.lrdc.pitt.edu/schunn/courses/0420/cogpsy%200421%20syllabus.pdf
http://www.lrdc.pitt.edu/schunn/courses/0420/cogpsy%200420%20syllabus.pdf
http://www.cvm.msstate.edu/Academics/2004/Radiology02-03.pdf
http://studyat.anu.edu.au/DisplayCourse-a.asp?role=1&course=HIST2004

davelester

http://religion.rutgers.edu/courses/317/317spring2002.html
http://religion.rutgers.edu/courses/212/212summer2002.html
http://www.ling.upenn.edu/courses/Summer_2002/ling001/
http://religion.rutgers.edu/courses/325/325spring2002.htm
http://religion.rutgers.edu/courses/212/212spring2002.htm
http://www.stcloudstate.edu/~plamen/2001_2002/im260/spring02/syllabus.html
http://webct.millersville.edu:8900/public/200220_9500/
http://www.washburn.edu/cas/psychology/pmacdonald/py620syllabusSpring2002.html
http://www2.h-net.msu.edu/~urban/teach/syllabi/seligman2002syl5.htm
http://religion.rutgers.edu/courses/324/324spring2002.html

davelester

http://faculty.oxy.edu/horowitz/courses/Hist221/hist221f2001.html
http://religion.rutgers.edu/courses/212/212_winter2001.html
http://www.music.utk.edu/keathley/MH220.2001.syllabus.html
http://www.stcloudstate.edu/~plamen/2001_2002/im260/spring02/syllabus.html
http://www.lib.iastate.edu/commons/hist307/2001spring/syllabus.html
http://www.math.grin.edu/~rebelsky/Courses/CS362/2001S/Handouts/instructions.html
http://www.math.grin.edu/~rebelsky/Courses/CS151/2001S/Handouts/instructions.html
http://www.unc.edu/courses/2001fall/anth/010/006/Syllabus1.html
http://cas-courses.buffalo.edu/classes/psy/segal/4212001/421Syl2001.htm
http://bss.sfsu.edu/tygiel/Hist427/427s2001online.html

davelester

<?php

$mysqli = new mysqli('localhost', 'root', '', 'syllabi_scraper');


if ($mysqli->connect_error) {
    die('Connect Error (' . $mysqli->connect_errno . ') '
            . $mysqli->connect_error);
}

davelester

http://www.phys.tcu.edu/~dingram/phys20073/su02_syll.html
http://www.phys.tcu.edu/~dingram/phys20073/su02_syll_lab.html
http://web.gc.cuny.edu/ees/weather_climate/Fall%202002%20LAB%2007%20Syllabus%20for%20PGEOG130.pdf
http://www2.semo.edu/foreignlang/plan_del_cursosn100%2007.htm
http://www.uncp.edu/home/albrecht/pspa101_%2007f02.htm
http://www.cambridgecollege.edu/eforum/upload/PM%20II%20Syllabus%20072002.doc
http://www.eiu.edu/~multilit/english2007-f00/syllabus.htm
http://www.unc.edu/depts/comm/CourseSyllabi/Fall%202002/029%20Comm%20072.1%20Syllabus.doc
http://somc.csudh.edu/fa02_pub512/Week%2007%20Locke/Locke%20new.htm
http://cc.ysu.edu/~triley/2610/2610SyllabusFall%202002%200706.htm

davelester

http://faculty.oxy.edu/horowitz/courses/Hist221/hist221f2001.html
http://sunset.usc.edu/classes/cs599_2001/
http://religion.rutgers.edu/courses/212/212_winter2001.html
http://www.ling.upenn.edu/courses/Spring_2001/ling001/
http://www.music.utk.edu/keathley/MH220.2001.syllabus.html
http://www.stcloudstate.edu/~plamen/2001_2002/im260/spring02/syllabus.html
http://www.lib.iastate.edu/commons/hist307/2001spring/syllabus.html
http://www.math.grin.edu/~rebelsky/Courses/CS362/2001S/Handouts/instructions.html
http://www.math.grin.edu/~rebelsky/Courses/CS151/2001S/Handouts/instructions.html
http://www.southalabama.edu/history/faculty/monheit/101-f2001/