Skip to content

Instantly share code, notes, and snippets.

@mt3o
Created June 16, 2021 16:39
Show Gist options
  • Save mt3o/2e4c052f80d248a0b5e67e2ecdae2ce8 to your computer and use it in GitHub Desktop.
Save mt3o/2e4c052f80d248a0b5e67e2ecdae2ce8 to your computer and use it in GitHub Desktop.
python multiprocessing with intermediate queue
# stolen from: https://stackoverflow.com/questions/6672525/multiprocessing-queue-in-python
import multiprocessing
num_procs = 4
def do_work(message):
print "work",message ,"completed"
def worker():
for item in iter( q.get, None ):
do_work(item)
q.task_done()
q.task_done()
q = multiprocessing.JoinableQueue()
procs = []
for i in range(num_procs):
procs.append( multiprocessing.Process(target=worker) )
procs[-1].daemon = True
procs[-1].start()
source = ['hi','there','how','are','you','doing']
for item in source:
q.put(item)
q.join()
for p in procs:
q.put( None )
q.join()
for p in procs:
p.join()
print "Finished everything...."
print "num active children:", multiprocessing.active_children()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment