My solutions to https://go.dev/tour/concurrency/10

csp_crawler.go

package main

import (
	"fmt"
	"os"
)

// Fetcher implementations must be concurrency-safe
type Fetcher interface {
	// Fetch returns the body of URL and a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"https://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"https://golang.org/pkg/",
			"https://golang.org/cmd/",
		},
	},
	"https://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"https://golang.org/",
			"https://golang.org/cmd/",
			"https://golang.org/pkg/fmt/",
			"https://golang.org/pkg/os/",
		},
	},
	"https://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"https://golang.org/",
			"https://golang.org/pkg/",
		},
	},
	"https://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"https://golang.org/",
			"https://golang.org/pkg/",
		},
	},
}

// maxDepth is exclusive.
func Crawl(rootUrl string, maxDepth int, fetcher Fetcher) map[string]bool {
	urlsSeen := make(map[string]bool) // url -> exists

	type UrlToFetch struct {
		url   string
		depth int
	}

	runningFetchers := 0
	fetcherDoneCh := make(chan any)
	urlExistsCh := make(chan string)
	urlsToFetchCh := make(chan UrlToFetch, 1)

	urlsToFetchCh <- UrlToFetch{rootUrl, 0}

	for {
		select {
		case urlToFetch := <-urlsToFetchCh:
			// If the requested depth is too high, ignore.
			if urlToFetch.depth > maxDepth {
				continue
			}

			// See if we've already seen the url before, record it otherwise.
			_, seen := urlsSeen[urlToFetch.url]
			if seen {
				continue
			}
			urlsSeen[urlToFetch.url] = false

			// Run a fetcher for the unseen url.
			runningFetchers++
			go func(urlToFetch UrlToFetch) {
				defer func() { fetcherDoneCh <- nil }()
				_, newUrls, err := fetcher.Fetch(urlToFetch.url)
				if err != nil {
					fmt.Fprintln(os.Stderr, err)
					return
				}
				urlExistsCh <- urlToFetch.url
				for _, newUrl := range newUrls {
					urlsToFetchCh <- UrlToFetch{newUrl, urlToFetch.depth + 1}
				}
			}(urlToFetch)
		case legitUrl := <-urlExistsCh:
			urlsSeen[legitUrl] = true
		case <-fetcherDoneCh:
			runningFetchers--
		default:
			if runningFetchers == 0 {
				return urlsSeen
			}
		}
	}
}

func main() {
	urls := Crawl("https://golang.org/", 2, fetcher)
	fmt.Println("links gathered: ")
	for url, exists := range urls {
		var existanceString string
		if exists {
			existanceString = "exists"
		} else {
			existanceString = "does not exist"
		}
		fmt.Printf("  %s (%s)\n", url, existanceString)
	}
}

traditional_crawler.go

package main

import (
	"fmt"
	"os"
	"sync"
)

// Fetcher implementations must be concurrency-safe
type Fetcher interface {
	// Fetch returns the body of URL and a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"https://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"https://golang.org/pkg/",
			"https://golang.org/cmd/",
		},
	},
	"https://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"https://golang.org/",
			"https://golang.org/cmd/",
			"https://golang.org/pkg/fmt/",
			"https://golang.org/pkg/os/",
		},
	},
	"https://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"https://golang.org/",
			"https://golang.org/pkg/",
		},
	},
	"https://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"https://golang.org/",
			"https://golang.org/pkg/",
		},
	},
}

// maxDepth is exclusive.
func Crawl(rootUrl string, maxDepth int, fetcher Fetcher) map[string]bool {
	wg := &sync.WaitGroup{}
	urlsSeenMonitor := &sync.Mutex{}
	urlsSeen := make(map[string]bool) // url -> exists

	wg.Add(1)
	go crawlStep(
		rootUrl,
		0,
		maxDepth,
		wg,
		urlsSeen,
		urlsSeenMonitor)
	wg.Wait()

	return urlsSeen
}

func crawlStep(
	url string,
	depth int,
	maxDepth int,
	wg *sync.WaitGroup,
	urlsSeen map[string]bool,
	urlsSeenMonitor *sync.Mutex) {

	defer wg.Done()

	if depth >= maxDepth {
		return
	}
	_, newUrls, err := fetcher.Fetch(url)

	urlsSeenMonitor.Lock()
	defer urlsSeenMonitor.Unlock()

	if err != nil {
		urlsSeen[url] = false
		fmt.Fprintln(os.Stderr, err)
		return
	}

	urlsSeen[url] = true
	for _, newUrl := range newUrls {
		_, seen := urlsSeen[newUrl]
		if seen {
			continue
		}
		wg.Add(1)
		urlsSeen[newUrl] = false
		// ^ This will prevent a race condition where we
		// might kick off a duplicate goroutine before the
		// one running for newUrl has marked it as seen.
		go crawlStep(
			newUrl,
			depth+1,
			maxDepth,
			wg,
			urlsSeen,
			urlsSeenMonitor)
	}
}

func main() {
	urls := Crawl("https://golang.org/", 10, fetcher)
	fmt.Println("links gathered: ")
	for url, exists := range urls {
		var existanceString string
		if exists {
			existanceString = "exists"
		} else {
			existanceString = "does not exist"
		}
		fmt.Printf("  %s (%s)\n", url, existanceString)
	}
}