Skip to content

Instantly share code, notes, and snippets.

@orcaman
Created March 8, 2020 08:14
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save orcaman/1539974ac6330229b140f2d922258906 to your computer and use it in GitHub Desktop.
Save orcaman/1539974ac6330229b140f2d922258906 to your computer and use it in GitHub Desktop.
CloudFunction to gather multiple GET requests
package p
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"sync"
"go.uber.org/ratelimit"
)
func Gather(w http.ResponseWriter, r *http.Request) {
rl := ratelimit.New(100)
urlParams := r.URL.Query().Get(("urls"))
if len(urlParams) == 0 {
fmt.Fprintf(w, "missing urls param")
return
}
sURLs := strings.Split(urlParams, ",")
m := sync.Map{}
wg := &sync.WaitGroup{}
for _, s := range sURLs {
rl.Take()
wg.Add(1)
go func(sURL string) {
log.Printf("calling url: %s\n", sURL)
defer func() {
log.Printf("done with url: %s\n", sURL)
wg.Done()
}()
resp, err := http.Get(sURL)
if err != nil {
log.Printf("%v", err)
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("%v", err)
return
}
var asJSON map[string]interface{}
err = json.Unmarshal(body, &asJSON)
if err != nil {
log.Printf("%v", err)
return
}
m.Store(sURL, asJSON)
}(s)
}
wg.Wait()
ret := map[string]interface{}{}
for _, s := range sURLs {
if v, ok := m.Load(s); ok {
data := v.(map[string]interface{})
ret[s] = data
}
}
v, err := json.Marshal(ret)
if err != nil {
log.Printf("%v", err)
return
}
fmt.Fprintf(w, "%s", string(v))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment