peculiar crawler - 3

crawler3.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	noOfId       = 1000000
	noOfworker   = 10000
	userRequired = 350
	minAge       = 10
	maxAge       = 50
	timeLimit    = 10
)

type User struct {
	Name string
	Age  int
}

func worker(wg *sync.WaitGroup, ch chan User, jobs chan int) {
	defer wg.Done()
	for id := range jobs {
		url := fmt.Sprintf("http://localhost:8080/user/%d", id)
		client := http.Client{
			Timeout: 5 * time.Second,
		}
		res, err := client.Get(url)
		if err != nil {
			//log error
			continue
		}
		data, err := ioutil.ReadAll(res.Body)
		// close response body
		res.Body.Close()
		if err != nil {
			//log error
			continue
		}
		var user User
		err = json.Unmarshal(data, &user)
		if err != nil {
			continue
		}
		if user.Age >= minAge && user.Age <= maxAge {
			ch <- user
		}
	}
}

func producer(ctx context.Context, wg *sync.WaitGroup, termination chan int, jobs chan int) {
	timeup := false
produce:
	for i := 1; i <= noOfId; i++ {
		select {
		case <-ctx.Done():
			if ctx.Err().Error() == "context deadline exceeded" {
				log.Println("Time Up")
			}
			timeup = true
			break produce
		default:
			jobs <- i
		}
	}
	close(jobs)
	//we wait for already running gorutines to finish casue
	//"Blocked Forever" goroutines might not get collected by GC (see issue #19702)
	//But this is not a major problem .
	wg.Wait()
	if timeup {
		termination <- 2
	} else {
		termination <- 1
	}
}
func printResult(myusers []User) {
	log.Printf("total users found : %d\n", len(myusers))
	for _, user := range myusers {
		fmt.Printf("user name :%s \nage :%d\n\n", user.Name, user.Age)
	}
}
func main() {
	var myusers []User
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*timeLimit)
	defer cancel()
	ch := make(chan User, noOfworker)
	jobs := make(chan int, noOfworker)
	termination := make(chan int)
	var wg sync.WaitGroup
	for i := 1; i <= noOfworker; i++ {
		wg.Add(1)
		go worker(&wg, ch, jobs)
	}
	go producer(ctx, &wg, termination, jobs)
consume:
	for true {
		select {
		case res := <-ch:
			myusers = append(myusers, res)
			if len(myusers) == userRequired {
				cancel()
				log.Println("Enough Users Found")
				break consume
			}
		case v := <-termination:
			if v == 1 {
				log.Println("All Worker Finished")
			}
			break consume
		}
	}
	printResult(myusers)

}