go tour #69

sample.go

func Crawl(url string, depth, workerCnt int, fetcher Fetcher) {
    reqch := make(chan int)
    seen := make(map[string]bool)
    working := make(chan bool, workerCnt)

    var crawl func(string, int)
    crawl = func(url string, depth int) {
        defer func(){reqch <- -1}()
        if depth <= 0 {
            return
        }
        if _,ok := seen[url] ; ok {
            return
        }
        seen[url] = true
        working <- true
        defer func(){ <-working}()
        body, urls, err := fetcher.Fetch(url)
        if err != nil {
            fmt.Println(err)
            return
        }
        fmt.Printf("found: %s %q\n", url, body)
        reqch <- len(urls)
        for _, u := range urls {
            go crawl(u, depth-1)
        }
    }
    go crawl(url, depth)
    actsum := 1
    for diff :=range reqch {
        actsum += diff
        if actsum == 0 {
            break
        }
    }
}