Skip to content

Instantly share code, notes, and snippets.

@PotHix
Last active December 28, 2015 08:59
Show Gist options
  • Save PotHix/7475316 to your computer and use it in GitHub Desktop.
Save PotHix/7475316 to your computer and use it in GitHub Desktop.
Go and a lot of http requests
const NumberOfGoroutines = 15
func main() {
var wg sync.WaitGroup
channel := make(chan string, NumberOfGoroutines)
redisch := make(chan *namesp.Metric, 3)
for i := 0; i < NumberOfGoroutines; i++ {
wg.Add(1)
go namesp.CacheMetric(channel, redisch, &wg, i)
}
// Get the value back and store on redis
for i := 0; i < NumberOfGoroutines/5; i++ {
go namesp.StoreValue(redisch)
}
runtime.GOMAXPROCS(2)
for _, server := range audit.CacheMachines() { // The name of some servers
metrics := namesp.GetMetrics() // just a list of metrics to use as part of the url
for _, metric := range metrics {
channel <- fmt.Sprintf(metric, server)
}
}
close(channel)
wg.Wait()
}
type Result struct {
Status int
}
type Metric struct {
Json []byte
Name string
}
var (
rc = redis.New("localhost:6379")
)
func retrieveJson(url string) []byte {
resp, err := http.Get(url)
if err != nil {
panic("Could not get the URL: " + url)
}
body, err := ioutil.ReadAll(resp.Body)
defer resp.Body.Close()
if err != nil {
panic("Could not read the response body")
}
return body
}
func cacheResource(full_metric string, out chan *Metric) {
before, after := formattedTime()
url := fmt.Sprintf(NAMESP_URL, full_metric, before, after)
fmt.Println("processing: " + url)
var metric *Metric
metric = new(Metric)
metric.Json = retrieveJson(url)
metric.Name = full_metric
out <- metric
}
func CacheMetric(server chan string, out chan *Metric, wg *sync.WaitGroup, i int) {
for metric := range server {
starting := int(time.Now().Unix())
cacheResource(metric, out)
ending := int(time.Now().Unix())
fmt.Println(strconv.Itoa(ending-starting) + ": [" + strconv.Itoa(i) + "] done processing " + metric)
}
wg.Done()
}
func StoreValue(out chan *Metric) {
for metric := range out {
result := &Result{}
json.Unmarshal(metric.Json, &result)
if result.Status == 200 {
rc.Set(metric.Name, string(metric.Json))
} else {
fmt.Println("Got the status '" + strconv.Itoa(result.Status) + "' when looking for " + metric.Name)
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment