Skip to content

Instantly share code, notes, and snippets.

@tonghuikang

tonghuikang/multiprocessing.ipynb

Created January 16, 2021 11:44
Show Gist options
  • Save tonghuikang/92b49510b51a26aba1cbc890dde3b99a to your computer and use it in GitHub Desktop.
Save tonghuikang/92b49510b51a26aba1cbc890dde3b99a to your computer and use it in GitHub Desktop.
Download ZIP
Multiprocessing Example
Display the source blob
Display the rendered blob
Raw
multiprocessing.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"def intensive_function(iterations=10**8, name=\"unnamed\"):\n",
" print(\"starting {}\".format(name))\n",
" for i in range(iterations):\n",
" pass\n",
" print(\"ending {}\".format(name))\n",
" return"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"starting unnamed\n",
"ending unnamed\n",
"CPU times: user 1.92 s, sys: 15.6 ms, total: 1.94 s\n",
"Wall time: 1.93 s\n"
]
}
],
"source": [
"%%time\n",
"intensive_function()"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"starting 0\n",
"starting 1\n",
"starting 2\n",
"starting 3\n",
"starting 4\n",
"starting 5\n",
"starting 6\n",
"starting 7\n",
"starting 8\n",
"starting 9\n",
"starting 10\n",
"starting 11\n",
"starting 12\n",
"starting 13\n",
"starting 14\n",
"starting 15\n",
"starting 16\n",
"starting 17\n",
"starting 18\n",
"starting 19\n",
"starting 20\n",
"starting 21\n",
"starting 22\n",
"starting 23\n",
"starting 24\n",
"starting 25\n",
"starting 26\n",
"starting 27\n",
"starting 28\n",
"starting 29\n",
"starting 30\n",
"starting 31\n",
"starting 32\n",
"starting 33\n",
"starting 34\n",
"starting 35\n",
"starting 36\n",
"starting 37\n",
"starting 38\n",
"starting 39\n",
"starting 40\n",
"starting 41\n",
"starting 42\n",
"starting 43\n",
"starting 44\n",
"starting 45\n",
"starting 46\n",
"starting 47\n",
"starting 48\n",
"starting 49\n",
"starting 50\n",
"starting 51\n",
"starting 52\n",
"starting 53\n",
"starting 54\n",
"starting 55\n",
"starting 56\n",
"starting 57\n",
"starting 58\n",
"starting 59\n",
"starting 60\n",
"starting 61\n",
"starting 62\n",
"starting 63\n",
"starting 64\n",
"starting 65\n",
"starting 66\n",
"starting 67\n",
"starting 68\n",
"starting 69\n",
"starting 70\n",
"starting 71\n",
"starting 72\n",
"starting 73\n",
"starting 74\n",
"starting 75\n",
"starting 76\n",
"starting 77\n",
"starting 78\n",
"starting 79\n",
"starting 80\n",
"starting 81\n",
"starting 82\n",
"starting 83\n",
"starting 84\n",
"starting 85\n",
"starting 86\n",
"starting 87\n",
"starting 89\n",
"starting 88\n",
"starting 90\n",
"starting 91\n",
"starting 92\n",
"starting 93\n",
"starting 94\n",
"starting 95\n",
"starting 96\n",
"starting 97\n",
"starting 98\n",
"starting 99\n",
"starting 100\n",
"starting 101\n",
"starting 102\n",
"starting 103\n",
"starting 104\n",
"starting 105\n",
"starting 106\n",
"starting 107\n",
"starting 108\n",
"starting 109\n",
"starting 110\n",
"starting 111\n",
"starting 112\n",
"starting 113\n",
"starting 114\n",
"starting 115\n",
"starting 116\n",
"starting 117\n",
"starting 118\n",
"starting 119\n",
"starting 120\n",
"starting 121\n",
"starting 122\n",
"starting 123\n",
"starting 124\n",
"starting 125\n",
"starting 126\n",
"starting 127\n",
"starting 128\n",
"starting 129\n",
"starting 130\n",
"starting 131\n",
"starting 132\n",
"starting 133\n",
"starting 134\n",
"starting 135\n",
"starting 136\n",
"starting 137\n",
"starting 138\n",
"starting 139\n",
"starting 140\n",
"starting 141\n",
"starting 142\n",
"starting 143\n",
"starting 144\n",
"starting 145\n",
"starting 146\n",
"starting 147\n",
"starting 148\n",
"starting 149\n",
"starting 150\n",
"starting 151\n",
"starting 152\n",
"starting 153\n",
"starting 154\n",
"starting 155\n",
"starting 156\n",
"starting 157\n",
"starting 158\n",
"starting 160\n",
"starting 159\n",
"starting 161\n",
"starting 162\n",
"starting 164\n",
"starting 163\n",
"starting 165\n",
"starting 166\n",
"starting 168\n",
"starting 167\n",
"starting 169\n",
"starting 170\n",
"starting 171\n",
"starting 173\n",
"starting 172\n",
"starting 174\n",
"starting 175\n",
"starting 176\n",
"starting 177\n",
"starting 179\n",
"starting 178\n",
"starting 180\n",
"starting 181\n",
"starting 182\n",
"starting 183\n",
"starting 184\n",
"starting 186\n",
"starting 185\n",
"starting 187\n",
"starting 188\n",
"starting 189\n",
"starting 190\n",
"starting 191\n",
"starting 192\n",
"starting 193\n",
"starting 195\n",
"starting 194\n",
"starting 196\n",
"starting 197\n",
"starting 198\n",
"starting 199\n",
"ending 49\n",
"ending 64\n",
"ending 61\n",
"ending 62\n",
"ending 59\n",
"ending 53\n",
"ending 63\n",
"ending 86\n",
"ending 85\n",
"ending 78\n",
"ending 75\n",
"ending 68\n",
"ending 60\n",
"ending 83\n",
"ending 79\n",
"ending 80\n",
"ending 67\n",
"ending 70\n",
"ending 65\n",
"ending 71\n",
"ending 66\n",
"ending 77\n",
"ending 69\n",
"ending 74\n",
"ending 73\n",
"ending 81\n",
"ending 82\n",
"ending 76\n",
"ending 84\n",
"ending 96\n",
"ending 107\n",
"ending 72\n",
"ending 121\n",
"ending 0\n",
"ending 110\n",
"ending 89\n",
"ending 88\n",
"ending 94\n",
"ending 36\n",
"ending 87\n",
"ending 7\n",
"ending 21\n",
"ending 111\n",
"ending 32\n",
"ending 15\n",
"ending 13\n",
"ending 16\n",
"ending 26\n",
"ending 8\n",
"ending 93\n",
"ending 20\n",
"ending 109\n",
"ending 9\n",
"ending 112\n",
"ending 58\n",
"ending 40\n",
"ending 139\n",
"ending 105\n",
"ending 114\n",
"ending 98\n",
"ending 149\n",
"ending 99\n",
"ending 181\n",
"ending 55\n",
"ending 138\n",
"ending 90\n",
"ending 104\n",
"ending 101\n",
"ending 18\n",
"ending 103\n",
"ending 100\n",
"ending 91\n",
"ending 33\n",
"ending 97\n",
"ending 14\n",
"ending 5\n",
"ending 118\n",
"ending 4\n",
"ending 1\n",
"ending 56\n",
"ending 23\n",
"ending 95\n",
"ending 17\n",
"ending 6\n",
"ending 173\n",
"ending 10\n",
"ending 186\n",
"ending 3\n",
"ending 108\n",
"ending 43\n",
"ending 22\n",
"ending 189\n",
"ending 34\n",
"ending 187\n",
"ending 106\n",
"ending 29\n",
"ending 12\n",
"ending 51\n",
"ending 27\n",
"ending 45\n",
"ending 31\n",
"ending 92\n",
"ending 19\n",
"ending 116\n",
"ending 182\n",
"ending 42\n",
"ending 54\n",
"ending 24\n",
"ending 183\n",
"ending 25\n",
"ending 113\n",
"ending 11\n",
"ending 192\n",
"ending 2\n",
"ending 39\n",
"ending 35\n",
"ending 28\n",
"ending 179\n",
"ending 188\n",
"ending 47\n",
"ending 158\n",
"ending 102\n",
"ending 30\n",
"ending 57\n",
"ending 145\n",
"ending 155\n",
"ending 46\n",
"ending 180\n",
"ending 117\n",
"ending 50\n",
"ending 44\n",
"ending 37\n",
"ending 164\n",
"ending 38\n",
"ending 131\n",
"ending 120\n",
"ending 172\n",
"ending 196\n",
"ending 184\n",
"ending 167\n",
"ending 126\n",
"ending 133\n",
"ending 52\n",
"ending 141\n",
"ending 41\n",
"ending 170\n",
"ending 193\n",
"ending 171\n",
"ending 142\n",
"ending 148\n",
"ending 135\n",
"ending 134\n",
"ending 115\n",
"ending 119\n",
"ending 153\n",
"ending 146\n",
"ending 154\n",
"ending 124\n",
"ending 178\n",
"ending 130\n",
"ending 156\n",
"ending 129\n",
"ending 176\n",
"ending 48\n",
"ending 128\n",
"ending 144\n",
"ending 191\n",
"ending 123\n",
"ending 122\n",
"ending 140\n",
"ending 169\n",
"ending 161\n",
"ending 163\n",
"ending 190\n",
"ending 194\n",
"ending 125\n",
"ending 162\n",
"ending 127\n",
"ending 160\n",
"ending 137\n",
"ending 143\n",
"ending 177\n",
"ending 147\n",
"ending 136\n",
"ending 175\n",
"ending 195\n",
"ending 198\n",
"ending 159\n",
"ending 199\n",
"ending 168\n",
"ending 165\n",
"ending 166\n",
"ending 151\n",
"ending 132\n",
"ending 174\n",
"ending 150\n",
"ending 152\n",
"ending 185\n",
"ending 157\n",
"ending 197\n",
"CPU times: user 734 ms, sys: 2.63 s, total: 3.37 s\n",
"Wall time: 11.8 s\n"
]
}
],
"source": [
"%%time\n",
"processes = []\n",
"for i in range(200):\n",
" p = Process(target=intensive_function, args=(10**8, i))\n",
" p.start()\n",
" processes.append(p)\n",
"for p in processes:\n",
" p.join()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment