Last active
May 5, 2020 19:44
-
-
Save cnnrznn/2fbb0f4b624d79b7e97cb740a43242d0 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
func worker(line string) []string { | |
// do your phone# parsing | |
return []string{} | |
} | |
func main() { | |
fn := os.Args[1] | |
numWorkers := 10 | |
srcChans := make([]chan interface{}, numWorkers) | |
outChan := make(chan []string, 1024) | |
var wg sync.WaitGroup | |
wg.Add(numWorkers) | |
for i := 0; i < numWorkers; i++ { | |
srcChans[i] = make(chan interface{}) | |
go func(i int) { | |
defer wg.Done() | |
for item := range srcChans[i] { | |
outChan <- worker(item.(string)) | |
} | |
}(i) | |
} | |
go func() { | |
wg.Wait() | |
close(outChan) | |
}() | |
gomr.TextFileParallel(fn, srcChans) | |
for item := range outChan { | |
fmt.Println(item) | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This is a program that uses my mapreduce library's (github.com/cnnrznn/gomr) file reader to read input into multiple Go channels concurrently. This allows for concurrently processing of a file in chunks.
The library provides input split on the newline character.