Created
June 19, 2023 08:52
-
-
Save crusaderky/87afa4d9952c9674f79cdffd993179b0 to your computer and use it in GitHub Desktop.
Dask Demo Day: Fine performance metrics and spans
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"id": "070dfc8c-2bf2-45d5-b515-195e0b6f367c", | |
"metadata": {}, | |
"source": [ | |
"# Dask Demo Day 2023-06-16\n", | |
"## Fine Performance Metrics and Spans\n", | |
"#### Presenter: Guido Imperiale (Coiled)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "b0adab3a-c0d8-4bbb-af95-5c22baa55631", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"import time\n", | |
"\n", | |
"import dask\n", | |
"import dask.array as da\n", | |
"from dask.datasets import timeseries\n", | |
"from distributed import Client\n", | |
"from distributed.metrics import context_meter\n", | |
"from distributed.spans import span\n", | |
"\n", | |
"# Spans are built on top of annotations. In order to make \n", | |
"# annotations work in dask.array, we need to disable task fusion.\n", | |
"dask.config.set({\"optimization.fuse.active\": False})" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "138e53af-357f-45e7-9065-1de997b90acb", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"client = Client()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "ffdcc199-ea1d-40ed-9426-dece70d88a83", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"ddf = timeseries(\"2000\", \"2020\", partition_freq=\"1w\")\n", | |
"ddf.groupby(\"id\").x.mean().compute()" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"id": "e8555424-22e9-4049-a1cf-666bafeb8cc0", | |
"metadata": {}, | |
"source": [ | |
"(please restart notebook)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "0b353ae4-37ce-4034-8a8b-2dbac7dcf8a7", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"client = Client(n_workers=4, threads_per_worker=1, memory_limit=\"2 GiB\")\n", | |
"rng = da.random.RandomState(0)\n", | |
"a = rng.random((14_000, 14_000))\n", | |
"b = (a @ a.T).sum()\n", | |
"b.compute()" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"id": "b6df7dca-307d-4c76-9499-e130d8db96fb", | |
"metadata": {}, | |
"source": [ | |
"(please restart notebook)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "884e37ee-d5e7-42c4-b407-064891488a4a", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"client = Client(n_workers=4, threads_per_worker=1, memory_limit=\"2 GiB\")\n", | |
"\n", | |
"@span(\"dot_product_self\")\n", | |
"def dot_product_self(a):\n", | |
" return a @ a.T\n", | |
"\n", | |
"with span(\"my workload\"):\n", | |
" with span(\"data load\"):\n", | |
" rng = da.random.RandomState(0)\n", | |
" a = rng.random((14_000, 14_000))\n", | |
" b = dot_product_self(a)\n", | |
" c = b.sum()\n", | |
"\n", | |
"c.compute()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "1650a860-9e21-4268-ba14-15045ad24577", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"with span(\"timeseries\"):\n", | |
" ddf = timeseries(\"2000\", \"2020\", partition_freq=\"1w\")\n", | |
" ddf.groupby(\"id\").x.mean().compute()" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"id": "079fbc72-e653-45cb-bffa-ee4869eb180e", | |
"metadata": {}, | |
"source": [ | |
"(please restart notebook)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "071e1d13-7a7c-442d-a0fc-25d0c0a45902", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"client = Client()\n", | |
"\n", | |
"def io_task():\n", | |
" time.sleep(0.8)\n", | |
" with context_meter.meter(\"I/O\"):\n", | |
" time.sleep(0.5)\n", | |
" context_meter.digest_metric(\"metered I/O\", 2**30, \"bytes\")\n", | |
"\n", | |
"def gpu_task():\n", | |
" with context_meter.meter(\"GPU\"):\n", | |
" time.sleep(0.7)\n", | |
" context_meter.digest_metric(\"task-gpu\", 123, \"gpu_seconds\")\n", | |
" \n", | |
"client.gather([\n", | |
" client.submit(io_task),\n", | |
" client.submit(gpu_task),\n", | |
"]) " | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "30ffb3cc-7af0-4fb9-a8ba-51f83448cdd8", | |
"metadata": {}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.11.4" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment