superjose

# Reference: https://www.exclamationlabs.com/blog/continuous-deployment-to-npm-using-gitlab-ci/

# GitLab uses docker in the background, so we need to specify the 
# image versions. This is useful because we're freely to use
# multiple node versions to work with it. They come from the docker
# repo.
# Uses NodeJS V 9.4.0
image: node:9.4.0

# And to cache them as well.

jameshartig

//before running make sure you setup a GOPATH env variable and ran: "go get code.google.com/p/go.net/html"

//to run: go run ioCrawler.go -url="http://developers.google.com/"
//also try http://developer.android.com/index.html
//output goo.gl links to try and redeem will be sent to foundLinks.txt

//by the way there's an artificial "rate limit" in func crawler, you can lower that or raise it as you wish
//You can also comment out the onlyGoogleRegex code if you don't want to limit to google.com/youtube
//if you're getting I/O timeout errors, then you might need to increase the timeouts on line 231