Skip to content

Instantly share code, notes, and snippets.

Last active November 3, 2017 21:59
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
What would you like to do?
import time
from distributed import Worker, Scheduler, Client, Nanny
import toolz
from tornado import gen
from tornado.ioloop import IOLoop
from dask import delayed
def inc(x):
# time.sleep(0.01)
return x + 1
def add(x, y):
return x + y
L = [inc(i) for i in range(2048)]
while len(L) > 1:
L = [add(a, b) for a, b in toolz.partition_all(2, L)]
[total] = L
def f():
yield [w._start(0) for w in workers]
c = yield Client(s.address, loop=loop, asynchronous=True)
start = time.time()
yield c.compute(total)
stop = time.time()
print("Duration: %0.3f seconds" % (stop - start))
if __name__ == '__main__':
loop = IOLoop.current()
s = Scheduler(loop=loop)
workers = [Nanny(s.ip, s.port, loop=loop) for i in range(3)]
# pip install snakeviz
# python -m cProfile -o prof.out
# snakeviz prof.out
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment