peculiar crawler - 2

crawler2.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	noOfId       = 1000000
	noOfWorker   = 10000
	userRequired = 350
	minAge       = 10
	maxAge       = 12
)

type User struct {
	Name string
	Age  int
}

func worker(wg *sync.WaitGroup, ch chan User, jobs chan int) {
	defer wg.Done()
	for id := range jobs {
		url := fmt.Sprintf("http://localhost:8080/user/%d", id)
		client := http.Client{
			Timeout: 5 * time.Second,
		}
		res, err := client.Get(url)
		if err != nil {
			//log error
			continue
		}
		data, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			continue
		}
		var user User
		err = json.Unmarshal(data, &user)
		if err != nil {
			continue
		}
		if user.Age >= minAge && user.Age <= maxAge {
			ch <- user
		}
	}
}

func producer(wg *sync.WaitGroup, termination chan bool, jobs chan int) {
	for i := 1; i <= noOfId; i++ {
		jobs <- i
	}
	close(jobs)
	wg.Wait()
	termination <- true
}

func printResult(myusers []User) {
	log.Printf("total users found : %d\n", len(myusers))
	for _, user := range myusers {
		fmt.Printf("user name :%s \nage :%d\n\n", user.Name, user.Age)
	}
}

func main() {
	var myusers []User
	ch := make(chan User, noOfWorker)
	jobs := make(chan int, noOfWorker)
	termination := make(chan bool)
	var wg sync.WaitGroup
	for i := 1; i <= noOfWorker; i++ {
		wg.Add(1)
		go worker(&wg, ch, jobs)
	}
	go producer(&wg, termination, jobs)
consume:
	for {
		select {
		case res := <-ch:
			myusers = append(myusers, res)
			if len(myusers) == userRequired {
				log.Println("Enough Users Found")
				break consume
			}
		case <-termination:
			log.Println("All Worker Finished")
			break consume
		case <-time.After(2 * time.Second):
			log.Println("Time Up")
			break consume
		}
	}
	printResult(myusers)
}