basic usage of colly go library

colly.go

package main

import (
	"fmt"

	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector(
		// MaxDepth is 2, so only the links on the scraped page
		// and links on those pages are visited
		colly.MaxDepth(2),
	)
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		e.Request.Visit(link)
	})
	c.OnRequest(func(r *colly.Request) {
		r.Ctx.Put("url", r.URL.String())
	})
	c.OnResponse(func(r *colly.Response) {
		url := r.Ctx.Get("url")
		fmt.Println((colly.SanitizeFileName(url)))
		r.Save(colly.SanitizeFileName(url))
	})
	c.Visit("https://caleblemoine.dev/")
}