Skip to content

Instantly share code, notes, and snippets.

@crusaderky

crusaderky/rebalance_demo.ipynb

Created April 30, 2021 10:01
Show Gist options
  • Save crusaderky/c1ccf5fd0107b13c8d24bbed5197d5f6 to your computer and use it in GitHub Desktop.
Save crusaderky/c1ccf5fd0107b13c8d24bbed5197d5f6 to your computer and use it in GitHub Desktop.
Download ZIP
O(1) rebalance
Display the source blob
Display the rendered blob
Raw
rebalance_demo.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "limiting-regression",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table style=\"border: 2px solid white;\">\n",
"<tr>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Client</h3>\n",
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
" <li><b>Scheduler: </b>tcp://127.0.0.1:42153</li>\n",
" <li><b>Dashboard: </b><a href='http://127.0.0.1:8787/status' target='_blank'>http://127.0.0.1:8787/status</a></li>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
" <li><b>Workers: </b>32</li>\n",
" <li><b>Cores: </b>128</li>\n",
" <li><b>Memory: </b>64.00 GiB</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: 'tcp://127.0.0.1:42153' processes=32 threads=128, memory=64.00 GiB>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import threading\n",
"import time\n",
"\n",
"import distributed\n",
"import numpy\n",
"import pandas\n",
"\n",
"# This requires 64 GiB RAM on the host\n",
"n_workers = 32\n",
"client = distributed.Client(\n",
" n_workers=n_workers, threads_per_worker=4, memory_limit=\"2 GiB\"\n",
")\n",
"client.wait_for_workers(n_workers)\n",
"scheduler = client.cluster.scheduler\n",
"workers = scheduler._workers_dv.values()\n",
"assert len(workers) == n_workers\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "unavailable-skiing",
"metadata": {},
"outputs": [],
"source": [
"# Load the cluster with N keys, S bytes per key, triangular distribution\n",
"N = 10_000\n",
"S = 2 * 2 ** 20\n",
"# N = 100_000\n",
"# S = 160 * 1024\n",
"\n",
"# Don't have more than this many running futures to avoid crashing the scheduler\n",
"sem = threading.Semaphore(4000)\n",
"\n",
"tasks_to_do = {w: i * N // sum(range(n_workers)) for i, w in enumerate(workers)}\n",
"\n",
"futures = []\n",
"while tasks_to_do:\n",
" for w, n in list(tasks_to_do.items()):\n",
" if not n:\n",
" del tasks_to_do[w]\n",
" continue\n",
"\n",
" sem.acquire()\n",
" f = client.submit(numpy.random.random, S // 8, workers=[w.address], pure=False)\n",
" f.add_done_callback(lambda _: sem.release())\n",
" futures.append(f)\n",
" tasks_to_do[w] -= 1\n",
" \n",
"time.sleep(4)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "contained-oxide",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1800 messages defined in 5105us\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sending</th>\n",
" <th>receiving</th>\n",
" <th>has</th>\n",
" <th>will have</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:32909</th>\n",
" <td>0</td>\n",
" <td>256</td>\n",
" <td>0</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:33525</th>\n",
" <td>0</td>\n",
" <td>236</td>\n",
" <td>20</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:33777</th>\n",
" <td>0</td>\n",
" <td>216</td>\n",
" <td>40</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:34323</th>\n",
" <td>0</td>\n",
" <td>197</td>\n",
" <td>60</td>\n",
" <td>257</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:34719</th>\n",
" <td>0</td>\n",
" <td>176</td>\n",
" <td>80</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:34827</th>\n",
" <td>0</td>\n",
" <td>159</td>\n",
" <td>100</td>\n",
" <td>259</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:35315</th>\n",
" <td>0</td>\n",
" <td>136</td>\n",
" <td>120</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:35501</th>\n",
" <td>0</td>\n",
" <td>132</td>\n",
" <td>141</td>\n",
" <td>273</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37059</th>\n",
" <td>0</td>\n",
" <td>108</td>\n",
" <td>161</td>\n",
" <td>269</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37129</th>\n",
" <td>0</td>\n",
" <td>75</td>\n",
" <td>181</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37235</th>\n",
" <td>0</td>\n",
" <td>55</td>\n",
" <td>201</td>\n",
" <td>256</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37585</th>\n",
" <td>0</td>\n",
" <td>37</td>\n",
" <td>221</td>\n",
" <td>258</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37809</th>\n",
" <td>0</td>\n",
" <td>17</td>\n",
" <td>241</td>\n",
" <td>258</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:38253</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>262</td>\n",
" <td>262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:38735</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>282</td>\n",
" <td>282</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:38837</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>302</td>\n",
" <td>302</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:39569</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>322</td>\n",
" <td>322</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:39821</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>342</td>\n",
" <td>342</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:39899</th>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>362</td>\n",
" <td>361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:40983</th>\n",
" <td>25</td>\n",
" <td>0</td>\n",
" <td>383</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:41113</th>\n",
" <td>32</td>\n",
" <td>0</td>\n",
" <td>403</td>\n",
" <td>371</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:41239</th>\n",
" <td>65</td>\n",
" <td>0</td>\n",
" <td>423</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:42831</th>\n",
" <td>79</td>\n",
" <td>0</td>\n",
" <td>443</td>\n",
" <td>364</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:42833</th>\n",
" <td>101</td>\n",
" <td>0</td>\n",
" <td>463</td>\n",
" <td>362</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:42937</th>\n",
" <td>112</td>\n",
" <td>0</td>\n",
" <td>483</td>\n",
" <td>371</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:44579</th>\n",
" <td>146</td>\n",
" <td>0</td>\n",
" <td>504</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45365</th>\n",
" <td>148</td>\n",
" <td>0</td>\n",
" <td>524</td>\n",
" <td>376</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45413</th>\n",
" <td>177</td>\n",
" <td>0</td>\n",
" <td>544</td>\n",
" <td>367</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45763</th>\n",
" <td>204</td>\n",
" <td>0</td>\n",
" <td>564</td>\n",
" <td>360</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45941</th>\n",
" <td>226</td>\n",
" <td>0</td>\n",
" <td>584</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:46389</th>\n",
" <td>242</td>\n",
" <td>0</td>\n",
" <td>604</td>\n",
" <td>362</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:46509</th>\n",
" <td>242</td>\n",
" <td>0</td>\n",
" <td>625</td>\n",
" <td>383</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sending receiving has will have\n",
"tcp://127.0.0.1:32909 0 256 0 256\n",
"tcp://127.0.0.1:33525 0 236 20 256\n",
"tcp://127.0.0.1:33777 0 216 40 256\n",
"tcp://127.0.0.1:34323 0 197 60 257\n",
"tcp://127.0.0.1:34719 0 176 80 256\n",
"tcp://127.0.0.1:34827 0 159 100 259\n",
"tcp://127.0.0.1:35315 0 136 120 256\n",
"tcp://127.0.0.1:35501 0 132 141 273\n",
"tcp://127.0.0.1:37059 0 108 161 269\n",
"tcp://127.0.0.1:37129 0 75 181 256\n",
"tcp://127.0.0.1:37235 0 55 201 256\n",
"tcp://127.0.0.1:37585 0 37 221 258\n",
"tcp://127.0.0.1:37809 0 17 241 258\n",
"tcp://127.0.0.1:38253 0 0 262 262\n",
"tcp://127.0.0.1:38735 0 0 282 282\n",
"tcp://127.0.0.1:38837 0 0 302 302\n",
"tcp://127.0.0.1:39569 0 0 322 322\n",
"tcp://127.0.0.1:39821 0 0 342 342\n",
"tcp://127.0.0.1:39899 1 0 362 361\n",
"tcp://127.0.0.1:40983 25 0 383 358\n",
"tcp://127.0.0.1:41113 32 0 403 371\n",
"tcp://127.0.0.1:41239 65 0 423 358\n",
"tcp://127.0.0.1:42831 79 0 443 364\n",
"tcp://127.0.0.1:42833 101 0 463 362\n",
"tcp://127.0.0.1:42937 112 0 483 371\n",
"tcp://127.0.0.1:44579 146 0 504 358\n",
"tcp://127.0.0.1:45365 148 0 524 376\n",
"tcp://127.0.0.1:45413 177 0 544 367\n",
"tcp://127.0.0.1:45763 204 0 564 360\n",
"tcp://127.0.0.1:45941 226 0 584 358\n",
"tcp://127.0.0.1:46389 242 0 604 362\n",
"tcp://127.0.0.1:46509 242 0 625 383"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def rebalance_report():\n",
" t0 = time.time()\n",
" msgs = scheduler._rebalance_find_msgs(None, workers)\n",
" t1 = time.time()\n",
" print(f\"{len(msgs)} messages defined in {(t1 - t0) * 1e6:.0f}us\")\n",
" cnt = {w: [0, 0] for w in workers}\n",
" for s, r, _ in msgs:\n",
" cnt[s][0] += 1\n",
" cnt[r][1] += 1\n",
" df = pandas.DataFrame(cnt, index=[\"sending\", \"receiving\"]).T\n",
" df.index = [w.address for w in df.index]\n",
" df[\"has\"] = [len(w._has_what) for w in workers]\n",
" df[\"will have\"] = df[\"has\"] - df[\"sending\"] + df[\"receiving\"]\n",
" return df\n",
"\n",
"rebalance_report()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "brave-austin",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 571 ms, sys: 90 ms, total: 661 ms\n",
"Wall time: 1.71 s\n",
"CPU times: user 1.03 ms, sys: 129 µs, total: 1.16 ms\n",
"Wall time: 1.11 ms\n",
"CPU times: user 1.39 ms, sys: 0 ns, total: 1.39 ms\n",
"Wall time: 1.38 ms\n",
"CPU times: user 933 µs, sys: 127 µs, total: 1.06 ms\n",
"Wall time: 1.05 ms\n",
"CPU times: user 1.17 ms, sys: 168 µs, total: 1.34 ms\n",
"Wall time: 1.32 ms\n",
"CPU times: user 10.5 ms, sys: 5.2 ms, total: 15.7 ms\n",
"Wall time: 43.4 ms\n",
"CPU times: user 7.05 ms, sys: 0 ns, total: 7.05 ms\n",
"Wall time: 17.4 ms\n",
"CPU times: user 7.4 ms, sys: 370 µs, total: 7.77 ms\n",
"Wall time: 13 ms\n",
"CPU times: user 1.17 ms, sys: 0 ns, total: 1.17 ms\n",
"Wall time: 1.16 ms\n",
"CPU times: user 0 ns, sys: 1.09 ms, total: 1.09 ms\n",
"Wall time: 1.09 ms\n"
]
}
],
"source": [
"for _ in range(10):\n",
" %time client.rebalance()\n",
" time.sleep(4)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "thousand-implementation",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 messages defined in 188us\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sending</th>\n",
" <th>receiving</th>\n",
" <th>has</th>\n",
" <th>will have</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:32909</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>264</td>\n",
" <td>264</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:33525</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>264</td>\n",
" <td>264</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:33777</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>262</td>\n",
" <td>262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:34323</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>263</td>\n",
" <td>263</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:34719</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>262</td>\n",
" <td>262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:34827</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>262</td>\n",
" <td>262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:35315</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>263</td>\n",
" <td>263</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:35501</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>273</td>\n",
" <td>273</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37059</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>274</td>\n",
" <td>274</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37129</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>261</td>\n",
" <td>261</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37235</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>262</td>\n",
" <td>262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37585</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>263</td>\n",
" <td>263</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:37809</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>261</td>\n",
" <td>261</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:38253</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>262</td>\n",
" <td>262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:38735</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>282</td>\n",
" <td>282</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:38837</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>302</td>\n",
" <td>302</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:39569</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>322</td>\n",
" <td>322</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:39821</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>342</td>\n",
" <td>342</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:39899</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>361</td>\n",
" <td>361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:40983</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>358</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:41113</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>362</td>\n",
" <td>362</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:41239</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>358</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:42831</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>361</td>\n",
" <td>361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:42833</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>362</td>\n",
" <td>362</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:42937</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>362</td>\n",
" <td>362</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:44579</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>358</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45365</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>361</td>\n",
" <td>361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45413</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>362</td>\n",
" <td>362</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45763</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>360</td>\n",
" <td>360</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:45941</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>358</td>\n",
" <td>358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:46389</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>361</td>\n",
" <td>361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tcp://127.0.0.1:46509</th>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>357</td>\n",
" <td>357</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sending receiving has will have\n",
"tcp://127.0.0.1:32909 0 0 264 264\n",
"tcp://127.0.0.1:33525 0 0 264 264\n",
"tcp://127.0.0.1:33777 0 0 262 262\n",
"tcp://127.0.0.1:34323 0 0 263 263\n",
"tcp://127.0.0.1:34719 0 0 262 262\n",
"tcp://127.0.0.1:34827 0 0 262 262\n",
"tcp://127.0.0.1:35315 0 0 263 263\n",
"tcp://127.0.0.1:35501 0 0 273 273\n",
"tcp://127.0.0.1:37059 0 0 274 274\n",
"tcp://127.0.0.1:37129 0 0 261 261\n",
"tcp://127.0.0.1:37235 0 0 262 262\n",
"tcp://127.0.0.1:37585 0 0 263 263\n",
"tcp://127.0.0.1:37809 0 0 261 261\n",
"tcp://127.0.0.1:38253 0 0 262 262\n",
"tcp://127.0.0.1:38735 0 0 282 282\n",
"tcp://127.0.0.1:38837 0 0 302 302\n",
"tcp://127.0.0.1:39569 0 0 322 322\n",
"tcp://127.0.0.1:39821 0 0 342 342\n",
"tcp://127.0.0.1:39899 0 0 361 361\n",
"tcp://127.0.0.1:40983 0 0 358 358\n",
"tcp://127.0.0.1:41113 0 0 362 362\n",
"tcp://127.0.0.1:41239 0 0 358 358\n",
"tcp://127.0.0.1:42831 0 0 361 361\n",
"tcp://127.0.0.1:42833 0 0 362 362\n",
"tcp://127.0.0.1:42937 0 0 362 362\n",
"tcp://127.0.0.1:44579 0 0 358 358\n",
"tcp://127.0.0.1:45365 0 0 361 361\n",
"tcp://127.0.0.1:45413 0 0 362 362\n",
"tcp://127.0.0.1:45763 0 0 360 360\n",
"tcp://127.0.0.1:45941 0 0 358 358\n",
"tcp://127.0.0.1:46389 0 0 361 361\n",
"tcp://127.0.0.1:46509 0 0 357 357"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rebalance_report()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "wicked-chemistry",
"metadata": {},
"outputs": [],
"source": [
"del futures"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "saving-semiconductor",
"metadata": {},
"outputs": [],
"source": [
"import gc\n",
"\n",
"_ = client.run(gc.collect)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "sunset-obligation",
"metadata": {},
"outputs": [],
"source": [
"import ctypes\n",
"\n",
"def trim_memory() -> int:\n",
" libc = ctypes.CDLL(\"libc.so.6\")\n",
" return libc.malloc_trim(0)\n",
"\n",
"_ = client.run(trim_memory)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "sunset-compiler",
"metadata": {},
"outputs": [],
"source": [
"2*.3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "narrow-quebec",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment