-
-
Save automaticalldramatic/682363ba868a71ab2038da72169c91c3 to your computer and use it in GitHub Desktop.
Simplification of sample code at https://blog.gopheracademy.com/advent-2015/automi-stream-processing-over-go-channels/ Best practise is 1) to inject channels and 2) avoid concurrency in APIs.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
func ingest(out <-chan []string) { | |
out <- []string{"aaaa", "bbb"} | |
out <- []string{"cccccc", "dddddd"} | |
out <- []string{"e", "fffff", "g"} | |
close(out) | |
} | |
func process(in <-chan []string, out <-chan int) { | |
for data := range in { | |
for _, word := range data { | |
out <- len(word) | |
} | |
} | |
} | |
func store(in <-chan int) { | |
for data := range in { | |
fmt.Println(data) | |
} | |
} | |
func main() { | |
concurrency := 4 | |
// stage 1 ingest data from source | |
in := make(chan []string) | |
go ingest(in) | |
// stage 2 - process data | |
reduced := make(chan int) | |
var wg sync.WaitGroup | |
wg.Add(concurrency) | |
for i := 0; i < concurrency; i++ { | |
go func() { | |
process(in, reduced) | |
wg.Done() | |
}() | |
} | |
go func() { | |
wg.Wait() | |
close(reduced) | |
}() | |
// stage 3 - store | |
store(reduced) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment