Dask task graph handling costs on the client

Dask-task-graph-handling-costs-on-the-client.ipynb

dask-task-graph-handling-costs.py

# Example from "Doing Nothing Poorly: Accelerating the Dask Scheduler" workshop
# Dask Summit 2021

# Task graph handling costs on the client

import pickle
import dask
from dask.datasets import timeseries

# Create dask task graph
%time ddf = timeseries().shuffle("id", shuffle="tasks").head(compute=False)
# Wall time: 4.01 s

# Optimize
%time ddf_opt, = dask.optimize(ddf)
# Wall time: 1.3 s

# Serialize
byte_total = 0
for k, v in ddf_opt.__dask_graph__().items():
    byte_total += len(pickle.dumps(k)) + len(pickle.dumps(v))
# Wall time: 731 ms

# Send to the scheduler
dask.utils.format_bytes(byte_total)
# '15.88 MB' (Assume ~587 ms at 100MB/s)