Reminder for myself on configuring local dask.distributed cluster and URL of corresponding documentations

dask_note.py

# setup local cluster for dask
# 
# API documentation
# * dask.distributed.worker
# https://distributed.dask.org/en/latest/worker.html#distributed.worker.Worker
# * dask.distributed.LocalCluster
# https://distributed.dask.org/en/latest/api.html#distributed.LocalCluster
# * dask.distributed.Client
# https://distributed.dask.org/en/latest/api.html#distributed.Client

from dask.distributed import LocalCluster, Client


lc = LocalCluster(
        processes=True, # use processes instead of threads
        n_workers=2, # 1 scheduler, plus this number of worker
        threads_per_worker=1, # note: some libraries may use multi-threading
        memory_limit="4GB", # memory cap per worker
        )
c = Client(lc)

# then just regular dask code and the settings will take effect automatically