My solution to Go tour #70 - Web Crawler

go-tour-69.go

package main

import (
	"os"
	"fmt"
)

type FetcherResult struct {
	Urls []string
	body string
	err os.Error
}

type Fetcher interface {
	// Fetch sends, on the given channel, the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string, ch chan FetcherResult)
}

func Crawl(url string, depth int, fetcher Fetcher) {
	fetched_urls := make(map[string]bool)
	ch := make(chan FetcherResult)
	go fetcher.Fetch(url, ch)
	res := <-ch
	fetched_urls[url] = true
	urls := res.Urls
	fmt.Printf("found: %s %q\n", url, res.body)
	for i := depth; i > 0; i-- {
		channels := make([]chan FetcherResult, len(urls))
		for j := 0; j < len(urls); j++ {
			ch := make(chan FetcherResult)
			channels[j] = ch
			go fetcher.Fetch(urls[j], ch)
			fetched_urls[urls[j]] = true
		}
		new_urls := make([]string, 0)
		for j := 0; j < len(urls); j++ {
			res := <-channels[j]
			if res.err != nil {
				fmt.Println(res.err)
				continue
			}
			fmt.Printf("found: %s %q\n", urls[j], res.body)
			for _, u := range res.Urls {
				if _, seen := fetched_urls[u]; seen {
					continue
				}
				new_urls = append(new_urls, u)
			}
		}
		urls = new_urls
	}
}

func main() {
	Crawl("http://golang.org/", 4, fetcher)
}


// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls     []string
}

func (f *fakeFetcher) Fetch(url string, ch chan FetcherResult) {
	if res, ok := (*f)[url]; ok {
		ch <- FetcherResult{res.urls, res.body, nil}
	}
	ch <- FetcherResult{nil, "", fmt.Errorf("not found: %s", url)}
}

// fetcher is a populated fakeFetcher.
var fetcher = &fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}