Created
September 18, 2019 23:08
-
-
Save mrocklin/148da2076cac9ba183071241b3d8e18a to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Accelerate real-time Python workload\n", | |
"\n", | |
"We start with a simple signals processing workload, and then accelerate it by several orders magnitude using the following libraries:\n", | |
"\n", | |
"1. Numpy\n", | |
"2. Numba\n", | |
"3. Dask\n", | |
"4. CuPy\n", | |
"5. Numba CUDA\n", | |
"\n", | |
"We eventually run a real-time streaming system on multi-GPU hardware." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"import numpy as np\n", | |
"\n", | |
"def gather_from_detector():\n", | |
" return np.random.random((1000, 1000))\n", | |
"\n", | |
"@numba.jit\n", | |
"def smooth(x):\n", | |
" out = np.empty_like(x)\n", | |
" for i in range(1, x.shape[0] - 1):\n", | |
" for j in range(1, x.shape[1] - 1):\n", | |
" out[i, j] = (x[i + -1, j + -1] + x[i + -1, j + 0] + x[i + -1, j + 1] +\n", | |
" x[i + 0, j + -1] + x[i + 0, j + 0] + x[i + 0, j + 1] +\n", | |
" x[i + 1, j + -1] + x[i + 1, j + 0] + x[i + 1, j + 1]) // 9\n", | |
"\n", | |
" return out\n", | |
"\n", | |
"def save(x, filename):\n", | |
" pass" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"%%time\n", | |
"\n", | |
"for i in range(1000):\n", | |
" img = client.submit(gather_from_detector, pure=False)\n", | |
" img = client.submit(smooth, img)\n", | |
" img = client.submit(np.fft.fft2, img)\n", | |
" future = client.submit(save, img, \"file-\" + str(i) + \"-.dat\")\n", | |
" fire_and_forget(future)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from dask.distributed import Client, LocalCluster, fire_and_forget\n", | |
"\n", | |
"cluster = LocalCluster()\n", | |
"client = Client(cluster)\n", | |
"client" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"import asyncio\n", | |
"\n", | |
"async def f():\n", | |
" while True:\n", | |
" img = client.submit(gather_from_detector, pure=False)\n", | |
" img = client.submit(smooth, img)\n", | |
" img = client.submit(np.fft.fft2, img)\n", | |
" future = client.submit(save, img, \"file-\" + str(i) + \"-.dat\")\n", | |
" fire_and_forget(future)\n", | |
" await asyncio.sleep(0.01)\n", | |
" \n", | |
"asyncio.ensure_future(f())" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"cluster.scale(8)" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.7.4" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 4 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment