golang snippet template using structs, goroutines with limited paralellism, http requests, json unmarshall, safe access to variables

template.go

package main

import (
    "os"
    "fmt"
    "time"
    "io"
    "io/ioutil"
    "sync"
    "log"
    "net/http"
    "net/http/httputil"
    "net/url"
    "strings"
    "bytes"
    "encoding/json"
    "sort"
)

// Max number of running goroutines
const MAX_CONCURRENT_JOBS = 10

// Main struct
type RunTasks struct {
    mutex     sync.Mutex
    client    *http.Client
    waitGroup *sync.WaitGroup
    waitChan  chan struct{}
    pages     map[int][]byte
    debug     bool
}

// Struct generator
func NewRunTasks() *RunTasks {
    client := &http.Client{
        Timeout: 10 * time.Second,
    }

    // Check if we launched the program with the debug arg
    debug := false
    if len(os.Args) == 2 {
        if os.Args[1] == "debug" {
            debug = true
        }
    }

    return &RunTasks{
        client:    client,
        waitGroup: &sync.WaitGroup{},
        waitChan:  make(chan struct{}, MAX_CONCURRENT_JOBS),
        pages:     map[int][]byte{},
        debug:     debug,

    }
}

// HTTP fetch
func (rt *RunTasks) fetch(
    method  string,
    uri     string,
    form    map[string]string,
    data    []byte,
    headers map[string]string,
    debug   bool,
    ) (result []byte, err error) {

    // Generate the request body
    var rBody io.Reader = nil

    // Create a request body with a FORM
    if form != nil {
        args := url.Values{}
        for k, v := range form {
            args.Set(k, v)
        }
        rBody = strings.NewReader(args.Encode())
    }

    // Create a request body with RAW data
    if data != nil {
        rBody = bytes.NewBuffer(data)
    }

    // Create the request
    req, err := http.NewRequest(method, uri, rBody)
    if err != nil {
        return nil, err
    }

    // Add headers
    for k, v := range headers {
        req.Header.Add(k, v)
    }

    // Print request headers and body on bebug
    if debug == true {
        reqDump, err := httputil.DumpRequestOut(req, true)
        if err != nil {
            log.Fatal(err)
        }
        log.Printf(">> REQUEST:\n%s\n", string(reqDump))
    }

    // Perform the request
    res, err := rt.client.Do(req)
    if err != nil {
        return nil, err
    }
    defer res.Body.Close()

    // Print response headers and body on bebug
    if debug == true {
        respDump, err := httputil.DumpResponse(res, true)
        if err != nil {
            log.Fatal(err)
        }
        log.Printf("<< RESPONSE:\n%s\n", string(respDump))
    }

    // Retrieve the body
    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        return nil, err
    }

    return body, nil
}

// Update shared variables avoiding race conditions
func (rt *RunTasks) updateMap(key int, value []byte) (err error) {
    // Unlock on exit
    defer rt.mutex.Unlock()
    // While locked only one goroutine can access the map
    rt.mutex.Lock()
    // Now we can safely access the variable
    rt.pages[key] = value

    return nil
}

// Job to run as goroutine
func (rt *RunTasks) job(index int) (err error) {
    // Decrement the counter when the goroutine completes
    defer rt.waitGroup.Done()
    url := fmt.Sprintf("https://pokeapi.co/api/v2/pokemon/%d", index)
    // Fetch the page
    pageBytes, err := rt.fetch("GET", url, nil, nil, nil, rt.debug)
    if err != nil {
        log.Printf("Error %s while fetching %s", err, url)
    }
    // Save the results in a map
    rt.updateMap(index, pageBytes)

    return nil
}

func main() {
    // Create a runtasks struct
    rt := NewRunTasks()

    // Fetch all 151 pokemons from pokeapi
    for count := 1; count <= 151; count++ {
        // Put an empty struct in the fixed size channel
        // This will block and wait when channel is full
        rt.waitChan <- struct{}{}
        // Increment the counter
        rt.waitGroup.Add(1)
        go func(count int) {
            log.Printf(">> Job %v added. Got %v queued jobs.\n", count, len(rt.waitChan))
            rt.job(count)
            // Revome an item from the channel
            <- rt.waitChan
            log.Printf("<< Job %v done. Got %v queued jobs.\n", count, len(rt.waitChan))
        }(count)
    }
    // Wait for the goroutines to finish.
    rt.waitGroup.Wait()

    // Create a slice with the keys of the pages map
    keys := make([]int, 0, len(rt.pages))
    for key := range rt.pages {
        keys = append(keys, key)
    }
    // Sort the list
    sort.Ints(keys)

    // Print our map
    for index, k := range keys {
        page := rt.pages[k]
        // Unmarshall the Json
        var content map[string]interface{}
        json.Unmarshal(page, &content)
        // Print info for each result
        fmt.Println("Index:", index, "Length:", len(page), "Name:", content["name"], "Exp:", content["base_experience"])
    }

    os.Exit(0)
}