jameshartig

//before running make sure you setup a GOPATH env variable and ran: "go get code.google.com/p/go.net/html"

//to run: go run ioCrawler.go -url="http://developers.google.com/"
//also try http://developer.android.com/index.html
//output goo.gl links to try and redeem will be sent to foundLinks.txt

//by the way there's an artificial "rate limit" in func crawler, you can lower that or raise it as you wish
//You can also comment out the onlyGoogleRegex code if you don't want to limit to google.com/youtube
//if you're getting I/O timeout errors, then you might need to increase the timeouts on line 231

CharlesCai930

#!/usr/local/bin/python

import sys
from lxml import html
from urlparse import urljoin
import urllib2
import requests
import logging

visited_links = []