Solution to golang tour exercise 73: Web Crawler

webcrawler.go

package main

import (
    "fmt"
)

type Fetcher interface {
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to crawl pages starting with url,
// to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
    
    type Req struct{
        urls []string
        depth int
    }
    
    singleFetch := func (fetcher Fetcher, url string, depth int, crawlCh chan Req) {
        body, urls, err := fetcher.Fetch(url)
        
        switch {
        case err != nil:
            fmt.Println(err)
        default:
            fmt.Printf("found: %s %q\n", url, body)
            crawlCh <- Req{urls, depth-1}
        }
    }
    
    var seen = make(map[string]bool)
    
    discardSeen := func(urls []string) []string {
        var notseen []string
        for _, u := range urls {
            if !seen[u] {
                seen[u] = true
                notseen = append(notseen, u)
            }
        }
        return notseen
    }
    
    var crawlCh = make(chan Req)
    var quitCh  = make(chan bool)
    
    // crawlCh: channel to crawl urls
    go func() {
        for {
            req := <-crawlCh

            // replace req.urls with only unseen urls
            req.urls = discardSeen(req.urls)
            
            if req.depth <= 0 || len(req.urls) == 0 {
                quitCh <- false
                return
            }
            
            // process urls in parallel
            for _, url := range req.urls {
                go singleFetch(fetcher, url, req.depth, crawlCh)
            }
        }
    }()
    
    crawlCh <- Req{[]string{url}, depth}
    <- quitCh

    return
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

zoutput.txt

found: http://golang.org/ "The Go Programming Language"
found: http://golang.org/pkg/ "Packages"
not found: http://golang.org/cmd/
found: http://golang.org/pkg/fmt/ "Package fmt"
found: http://golang.org/pkg/os/ "Package os"