parser.go

parser.go

package main

import (
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/goquery"
)

func getTitle(url string) (title string, timeDiff int64) {

	t := time.Now()

	res, err := http.Get(url)

	if err != nil {
		fmt.Println(err)
	}

	defer res.Body.Close()

	doc, err := goquery.NewDocumentFromReader(res.Body)

	if err != nil {
		log.Fatalln(err)
	}

	return doc.Find("title").Text(), time.Since(t).Milliseconds()
}

func main() {
	res, err := http.Get("https://tproger.ru")

	if err != nil {
		log.Fatalln(err)
	}

	defer res.Body.Close()

	doc, err := goquery.NewDocumentFromReader(res.Body)

	if err != nil {
		log.Fatal(err)
	}

	var urls []string

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		href, _ := s.Attr("href")

		if strings.Contains(href, "tag") {
			urls = append(urls, "https://tproger.ru"+href)
		}
	})

	var wg sync.WaitGroup

	for _, url := range urls {

		wg.Add(1)

		go func(url string) {
			title, timeDiff := getTitle(url)
			fmt.Printf("Title: %s, Time: %dms\n", title, timeDiff)
			wg.Done()
		}(url)
	}

	wg.Wait()
}