CS101 - Simple Web Crawler (week 3)

gistfile1.py

import urllib2

def web_request(url):
	response = urllib2.urlopen(url)
	return response.read()

def get_next_link(page):
	href = '<a href="';
	start_pos = page.find(href)
	if start_pos > 0:
		url_start = start_pos + len(href)
		url_end = page.find('"', url_start)
		url = page[url_start:url_end]
		return url, url_end

	return None, -1

def get_links(page):
	links = []
	while True:
		url, end_pos = get_next_link(page)
		if not url:
			break

		links.append(url)
		page = page[end_pos:]

	return links

def crawl(seed):
	to_crawl = [seed]
	crawled = []

	print 'Crawling started...'
	while to_crawl:
		current = to_crawl.pop()

		if current not in crawled:
			page = web_request(current)
			links = get_links(page)
			if len(links) > 0:
				to_crawl.extend(links)

			crawled.append(current)

	print 'Crawling done...'
	return crawled

c = crawl('http://www.udacity.com/cs101x/index.html')
print c