Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
package main
import "code.google.com/p/go.net/html"
import "log"
import "strings"
import "io/ioutil"
import "net/http"
import "runtime"
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
WORKER_COUNT := 25
jobChannel := make(chan string)
for id := 0; id < WORKER_COUNT; id++ {
go fetchPage(id, jobChannel)
}
// Not most efficient way to read file, but isn't gonna use up all
// available RAM. This isn't the problem.
content, err := ioutil.ReadFile("urls.txt")
if err != nil {
log.Panicln("FILE READ ERROR")
}
urls := strings.Split(string(content), "\n")
for _, url := range urls {
jobChannel <- url
}
}
func fetchPage(id int, jobChannel chan string) {
for url := range jobChannel {
resp, err := http.Get(url)
if err != nil {
log.Println("http error")
continue
}
root, err := html.Parse(resp.Body)
resp.Body.Close()
if err != nil {
log.Println("parse error")
continue
}
log.Println(root)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment