Skip to content

Instantly share code, notes, and snippets.

@lebedov
Created June 11, 2014 15:08
Show Gist options
  • Save lebedov/806642ff739c75aeba53 to your computer and use it in GitHub Desktop.
Save lebedov/806642ff739c75aeba53 to your computer and use it in GitHub Desktop.
How to use multiple queues for passing data to/from a multiprocessing pool.
#!/usr/bin/env python
"""
How to use multiple queues for passing data to/from a multiprocessing pool.
"""
from multiprocessing import Pool, Queue
import shortuuid
def f(q_in, q_out):
"""
Worker process body.
"""
# Create unique ID to clearly distinguish between workers:
u = shortuuid.uuid()
while True:
m = q_in.get()
print '%s: %s' % (u, m)
if m == 'q':
break
else:
q_out.put(-m)
if __name__ == '__main__':
N = 4
q_in = Queue()
q_out = Queue()
p = Pool(N, f, (q_in, q_out))
# Send input:
M = 10
for i in xrange(M):
q_in.put(i)
# Get output:
for i in xrange(M):
m = q_out.get()
print m
# Tell workers to quit:
for i in xrange(N):
q_in.put('q')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment