Skip to content

Instantly share code, notes, and snippets.

@kevinisbest
Created September 17, 2019 06:40
Show Gist options
  • Save kevinisbest/61ce524666f1c4aaa3f6d988eda6861a to your computer and use it in GitHub Desktop.
Save kevinisbest/61ce524666f1c4aaa3f6d988eda6861a to your computer and use it in GitHub Desktop.
Processing all files with same extension in master-worker concurrency way
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
)
var (
// use this to control all worker to wait
wg = &sync.WaitGroup{}
)
// do task
func do(goroutine int, zipPath string) {
// waitgroup -1 after finishing this task
defer wg.Done()
// Here I replace zip with csv
zipPath = strings.Replace(zipPath, "zip", "csv", -1)
fmt.Printf("goroutine %d done zip %s\n", goroutine, zipPath)
}
// assign work to worker
func worker(works <-chan string, goroutine int) {
for work := range works {
do(goroutine, work)
}
}
func startWorkerPool(works <-chan string, size int) {
for i := 0; i < size; i++ {
go worker(works, i)
}
}
func findFiles(folderPath string, works chan string, idx int) int {
err := filepath.Walk(folderPath,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
} else if path[len(path)-3:] == "zip" {
// find one file, waitgroup+1
wg.Add(1)
works <- path
idx++
}
return nil
})
if err != nil {
fmt.Println(err)
}
return idx
}
func main() {
works := make(chan string)
// assign workers
startWorkerPool(works, 2)
folderPath := "/Users/kevin/Desktop/temp"
idx := 0
idx = findFiles(folderPath, works, idx)
// wait untill each worker finishes their task
wg.Wait()
fmt.Println("\nTotal files:", idx)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment