Skip to content

Instantly share code, notes, and snippets.

@ei-grad
Created October 28, 2016 16:48
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save ei-grad/42028032733df038501018175c20ee61 to your computer and use it in GitHub Desktop.
Save ei-grad/42028032733df038501018175c20ee61 to your computer and use it in GitHub Desktop.
Python multiprocessing.Queue sucks
zmq: 2.904726
queue: 12.812472
from threading import Thread
from time import time
from multiprocessing import Queue
import zmq
def inproc(sock):
while True:
d = sock.recv_pyobj()
if not d:
break
sock.close()
def queue(q):
while True:
d = q.get()
if not d:
break
def test_inproc():
ctx = zmq.Context()
pull = ctx.socket(zmq.PULL)
pull.bind('inproc://worker')
worker = Thread(target=inproc, args=(pull,))
worker.start()
push = ctx.socket(zmq.PUSH)
push.connect('inproc://worker')
t0 = time()
for i in range(1000000):
push.send_pyobj(b'q')
push.send_pyobj(b'')
push.close()
worker.join()
ctx.term()
print("zmq: %f" % (time() - t0))
def test_queue():
q = Queue()
worker = Thread(target=queue, args=(q,))
worker.start()
t0 = time()
for i in range(1000000):
q.put(b'q')
q.put(b'')
worker.join()
print("queue: %f" % (time() - t0))
if __name__ == "__main__":
test_inproc()
test_queue()
@ei-grad
Copy link
Author

ei-grad commented Dec 25, 2017

норм... а zmq и multiprocessing.Queue запусти чтоб на одном железе сравнить?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment