Created
June 16, 2021 16:39
-
-
Save mt3o/2e4c052f80d248a0b5e67e2ecdae2ce8 to your computer and use it in GitHub Desktop.
python multiprocessing with intermediate queue
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# stolen from: https://stackoverflow.com/questions/6672525/multiprocessing-queue-in-python | |
import multiprocessing | |
num_procs = 4 | |
def do_work(message): | |
print "work",message ,"completed" | |
def worker(): | |
for item in iter( q.get, None ): | |
do_work(item) | |
q.task_done() | |
q.task_done() | |
q = multiprocessing.JoinableQueue() | |
procs = [] | |
for i in range(num_procs): | |
procs.append( multiprocessing.Process(target=worker) ) | |
procs[-1].daemon = True | |
procs[-1].start() | |
source = ['hi','there','how','are','you','doing'] | |
for item in source: | |
q.put(item) | |
q.join() | |
for p in procs: | |
q.put( None ) | |
q.join() | |
for p in procs: | |
p.join() | |
print "Finished everything...." | |
print "num active children:", multiprocessing.active_children() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment