Bounded Parallel Get Requests in Golang

pget.go

package main

import (
	"fmt"
	"net/http"
	"sort"
	"time"
)

// a struct to hold the result from each request including an index
// which will be used for sorting the results after they come in
type result struct {
	index int
	res   http.Response
	err   error
}

// boundedParallelGet sends requests in parallel but only up to a certain
// limit, and furthermore it's only parallel up to the amount of CPUs but
// is always concurrent up to the concurrency limit
func boundedParallelGet(urls []string, concurrencyLimit int) []result {

	// this buffered channel will block at the concurrency limit
	semaphoreChan := make(chan struct{}, concurrencyLimit)

	// this channel will not block and collect the http request results
	resultsChan := make(chan *result)

	// make sure we close these channels when we're done with them
	defer func() {
		close(semaphoreChan)
		close(resultsChan)
	}()

	// keen an index and loop through every url we will send a request to
	for i, url := range urls {

		// start a go routine with the index and url in a closure
		go func(i int, url string) {

			// this sends an empty struct into the semaphoreChan which
			// is basically saying add one to the limit, but when the
			// limit has been reached block until there is room
			semaphoreChan <- struct{}{}

			// send the request and put the response in a result struct
			// along with the index so we can sort them later along with
			// any error that might have occoured
			res, err := http.Get(url)
			result := &result{i, *res, err}

			// now we can send the result struct through the resultsChan
			resultsChan <- result

			// once we're done it's we read from the semaphoreChan which
			// has the effect of removing one from the limit and allowing
			// another goroutine to start
			<-semaphoreChan

		}(i, url)
	}

	// make a slice to hold the results we're expecting
	var results []result

	// start listening for any results over the resultsChan
	// once we get a result append it to the result slice
	for {
		result := <-resultsChan
		results = append(results, *result)

		// if we've reached the expected amount of urls then stop
		if len(results) == len(urls) {
			break
		}
	}

	// let's sort these results real quick
	sort.Slice(results, func(i, j int) bool {
		return results[i].index < results[j].index
	})

	// now we're done we return the results
	return results
}

// we'll use the init function to set up the benchmark
// by making a slice of 100 URLs to send requets to
var urls []string

func init() {
	for i := 0; i < 100; i++ {
		urls = append(urls, "http://httpbin.org/get")
	}
}

// the main function sets up an anonymous benchmark func
// that will time how long it takes to get all the URLs
// at the specified concurrency level
//
// and you should see something like the following printed
// depending on how fast your computer and internet is
//
// 5 bounded parallel requests: 100/100 in 5.533223255
// 10 bounded parallel requests: 100/100 in 2.5115351219
// 25 bounded parallel requests: 100/100 in 1.189462884
// 50 bounded parallel requests: 100/100 in 1.17430002
// 75 bounded parallel requests: 100/100 in 1.001383863
// 100 bounded parallel requests: 100/100 in 1.3769354
func main() {
	benchmark := func(urls []string, concurrency int) string {
		startTime := time.Now()
		results := boundedParallelGet(urls, concurrency)
		seconds := time.Since(startTime).Seconds()
		tmplate := "%d bounded parallel requests: %d/%d in %v"
		return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds)
	}

	fmt.Println(benchmark(urls, 10))
	fmt.Println(benchmark(urls, 25))
	fmt.Println(benchmark(urls, 50))
	fmt.Println(benchmark(urls, 75))
	fmt.Println(benchmark(urls, 100))
}

how can handle error if i passed a non existing url it says panic memory error can we avoit that and send all 100 request to non existing url

i am using this script to read 100 url from file some of url non existing or time out but it is fail to run complite script

can you help me

sending request from a file but this script stop woking

You saved my life lol

Impressive. I'm totally impressed how you explain stuff with comments. Thanks Man!

You should definitely create a blog of your own :)

great example, only one thing - it will result in deadlock if urls is an empty array

instead infinite loop you can use:

for i := 0; i < len(urls); i++ {
    result := <-resultsChan
}

instead infinite loop you can use:

for i := 0; i < len(urls); i++ {
    result := <-resultsChan
}

or... i later found the community prefering this:

for {
    // infinite loop until stopped
}

Is it possible to stop execution of first error? (not panic)

Is it possible to stop execution of first error? (not panic)

error counter variable maybe?

Is it possible to stop execution of first error? (not panic)

error counter variable maybe?

variable or channel?