crawl.go

type linkCollector func(page crawler.Page) []string

func crawl(client crawler.Client, startPageURL string, collect linkCollector, maxConcurrency int) {
	concurrencyLimiter := make(chan struct{}, maxConcurrency)
	var wg sync.WaitGroup
	for i := 0; i < maxConcurrency; i++ {
		concurrencyLimiter <- struct{}{}
	}

	wg.Add(1)
	go recursiveCrawl(client, startPageURL, collect, &wg, concurrencyLimiter)

	wg.Wait()
}

func recursiveCrawl(client crawler.Client, pageURL string, collect linkCollector, wg *sync.WaitGroup, concurrencyLimiter chan struct{}) {
	<-concurrencyLimiter

	page := client.Get(pageURL)
	links := collect(page)
	for _, link := range links {
		wg.Add(1)
		go recursiveCrawl(client, link, collect, wg, concurrencyLimiter)
	}

	concurrencyLimiter <- struct{}{}
	wg.Done()
}