Created
September 22, 2019 14:26
-
-
Save Herts/6e24279a8eec98233def3229f557ffae to your computer and use it in GitHub Desktop.
This is an example of using a buffered channel to limit the number of goroutines running at the same time.
It will start a server locally listening port 9988, so if there is a warning from firewall on windows, allow it. This, I think, is very useful when scraping the Internet if you want to limit the concurrent connections.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"html" | |
"io/ioutil" | |
"log" | |
"net/http" | |
"sync" | |
"time" | |
) | |
func GetConcurrently() { | |
// set limit to 10, and make a buffered channel | |
limit := 10 | |
limiterChan := make(chan struct{}, limit) | |
for i := 0; i < limit; i++ { | |
limiterChan <- struct{}{} | |
} | |
// set num of goroutines to 1 << 6 | |
// num = 1 * 2^6 = 64 | |
num := 1 << 6 | |
// use sync.Waitgroup for managing goroutines | |
wg := sync.WaitGroup{} | |
wg.Add(num) | |
log.Println(num, "routines will be started") | |
for i := 0; i < num; i++ { | |
go func(i int, wg *sync.WaitGroup) { | |
t := time.Now() | |
log.Println("goroutine", i, "is waiting for signal from channel") | |
<-limiterChan | |
log.Println("After", time.Now().Sub(t), "goroutine", i, "got the signal") | |
// make a http GET request | |
resp, _ := http.Get(fmt.Sprint("http://localhost:9988/", i)) | |
defer resp.Body.Close() | |
byteResp, err := ioutil.ReadAll(resp.Body) | |
if err != nil { | |
log.Println(err) | |
} | |
log.Println("goroutine", i, "received", string(byteResp)) | |
// give the signal back to the channel | |
limiterChan <- struct{}{} | |
wg.Done() | |
}(i, &wg) | |
} | |
wg.Wait() | |
} | |
func handleGet(w http.ResponseWriter, r *http.Request) { | |
_, err := fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path)) | |
if err != nil { | |
log.Println(err) | |
} | |
} | |
func main() { | |
http.HandleFunc("/", handleGet) | |
go GetConcurrently() | |
log.Fatal(http.ListenAndServe(":9988", nil)) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
When limit = 2, num = 6: