serps_urls.go

func getSERPsURLs(baseURL *url.URL, depth int) (pages []*url.URL, err error) {
	doc, err := getDoc(baseURL)
	i := 0
	doc.Find("#nav").Each(func(_ int, table *goquery.Selection) {
		table.Find("tbody").Each(func(_ int, trs *goquery.Selection) {
			trs.Find("tr").Each(func(_ int, tds *goquery.Selection) {
				tds.Find("td").Each(func(_ int, srg *goquery.Selection) {
					srg.Find("a").Each(func(_ int, s *goquery.Selection) {
						if i >= depth-1 {
							return
						}
						href, exists := s.Attr("href")
						if exists {
							nextURL, _ := baseURL.Parse(href)
							pages = append(pages, nextURL)
							i++
						} else {
							err = errors.New("failed to retrieve the search result page")
							return
						}
					})
				})
			})
		})
	})
	return
}