http_calls_concurrent

http_calls_concurrent.go

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io"
	"net/http"
	"sync"
	"time"
)

func fetchUrls(url string) (string, error) {
	httpResponse, err := http.Get(url)

	if err != nil {
		return "", err
	}

	if httpResponse.StatusCode != http.StatusOK {
		return "", nil
	}

	defer httpResponse.Body.Close()

	title := getTitle(httpResponse.Body)

	return title, nil
}

func getTitle(htmlResponse io.ReadCloser) string {

	doc, err := goquery.NewDocumentFromReader(htmlResponse)

	if err != nil {
		return "Title cannot be fetched"
	}

	title := doc.Find("title").Text()

	return title
}

func main() {
	urls := []string{"https://github.com", "https://www.enuygun.com", "https://aligoren.com", "https://coderwall.com",
		"https://eksisozluk.com", "https://medium.com", "https://stackoverflow.com", "https://www.youtube.com"}
	urlSize := len(urls)

	mu := &sync.Mutex{}

	responses := make([]string, 0)

	wg := sync.WaitGroup{}
	wg.Add(urlSize)

	start := time.Now()
	for index, url := range urls {
		go func(index int, url string) {
			defer wg.Done()

			response, _ := fetchUrls(url)
			if len(response) > 0 {
				mu.Lock()
				responses = append(responses, response)
				mu.Unlock()
			}

		}(index, url)
	}
	wg.Wait()
	elapsedTime := time.Since(start)

	fmt.Printf("%#v\n", responses)
	fmt.Printf("Elapsed time: %s", elapsedTime.String())
}