Skip to content

Instantly share code, notes, and snippets.

@nebiros
Created April 9, 2020 20:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nebiros/87392e252ad8b7bdf659b57152570790 to your computer and use it in GitHub Desktop.
Save nebiros/87392e252ad8b7bdf659b57152570790 to your computer and use it in GitHub Desktop.
[SOLVED] Write a minimal thread pool using go routines and channels
package main
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
const (
workers = 3
jobs = 5
)
func main() {
jsonPath := os.Args[1]
ext := filepath.Ext(jsonPath)
if ext != ".json" {
log.Fatalf("file '%s' is not a json file", jsonPath)
}
if _, err := os.Stat("./data"); os.IsNotExist(err) {
if err := os.Mkdir("./data", 0755); err != nil {
log.Fatalf("something went wrong: %s", err.Error())
}
}
jsonFile, err := os.Open(jsonPath)
if err != nil {
log.Fatalf("can't open '%s'", jsonPath)
}
defer jsonFile.Close()
b, err := ioutil.ReadAll(jsonFile)
if err != nil {
log.Fatalf("can't open '%s'", jsonPath)
}
var urls []string
if err := json.Unmarshal(b, &urls); err != nil {
log.Fatalf("can't unmarshal '%s'", jsonPath)
}
httpClient := &http.Client{Timeout: 30 * time.Second}
jobsChan := make(chan string, jobs)
var wg sync.WaitGroup
for w := 1; w <= workers; w++ {
go worker(w, &wg, httpClient, jobsChan)
}
for _, url := range urls {
jobsChan <- url
wg.Add(1)
}
wg.Wait()
}
func worker(workerID int, wg *sync.WaitGroup, httpClient *http.Client, jobs <-chan string) {
for job := range jobs {
log.Printf("#%d - Downloading %s", workerID, job)
downloadImage(job, httpClient)
wg.Done()
log.Printf("#%d - Completed %s", workerID, job)
}
}
func downloadImage(url string, httpClient *http.Client) {
resp, err := httpClient.Get(url)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
imagePath := fmt.Sprintf("./data/%s", filepath.Base(url))
f, err := os.Create(imagePath)
if err != nil {
log.Fatal(err)
}
defer f.Close()
if _, err := io.Copy(f, resp.Body); err != nil {
log.Fatal(err)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment