Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
http_calls_concurrent
package main
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"io"
"net/http"
"sync"
"time"
)
func fetchUrls(url string) (string, error) {
httpResponse, err := http.Get(url)
if err != nil {
return "", err
}
if httpResponse.StatusCode != http.StatusOK {
return "", nil
}
defer httpResponse.Body.Close()
title := getTitle(httpResponse.Body)
return title, nil
}
func getTitle(htmlResponse io.ReadCloser) string {
doc, err := goquery.NewDocumentFromReader(htmlResponse)
if err != nil {
return "Title cannot be fetched"
}
title := doc.Find("title").Text()
return title
}
func main() {
urls := []string{"https://github.com", "https://www.enuygun.com", "https://aligoren.com", "https://coderwall.com",
"https://eksisozluk.com", "https://medium.com", "https://stackoverflow.com", "https://www.youtube.com"}
urlSize := len(urls)
mu := &sync.Mutex{}
responses := make([]string, 0)
wg := sync.WaitGroup{}
wg.Add(urlSize)
start := time.Now()
for index, url := range urls {
go func(index int, url string) {
defer wg.Done()
response, _ := fetchUrls(url)
if len(response) > 0 {
mu.Lock()
responses = append(responses, response)
mu.Unlock()
}
}(index, url)
}
wg.Wait()
elapsedTime := time.Since(start)
fmt.Printf("%#v\n", responses)
fmt.Printf("Elapsed time: %s", elapsedTime.String())
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment