Skip to content

Instantly share code, notes, and snippets.

@aaronspring

aaronspring/MWE_parallelize_resample_metric.ipynb

Created February 26, 2020 05:48
Show Gist options
  • Save aaronspring/b8c413916ad98d666d40627d16b0907e to your computer and use it in GitHub Desktop.
Save aaronspring/b8c413916ad98d666d40627d16b0907e to your computer and use it in GitHub Desktop.
Download ZIP
MWE_parallelize_resample_metric
Display the source blob
Display the rendered blob
Raw
MWE_parallelize_resample_metric.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Bootstrap metric in parallel\n",
"\n",
"This notebooks tries to apply a metric on resampled with replacement inputs in parallel."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import xarray as xr\n",
"from dask.distributed import Client\n",
"import multiprocessing\n",
"#import warnings\n",
"#warnings.filterwarnings(\"ignore\")\n",
"# number of logical cpus: https://www.dkrz.de/up/systems/mistral/configuration\n",
"ncpu = multiprocessing.cpu_count() \n",
"import timeit\n",
"import matplotlib.pyplot as plt\n",
"import xskillscore as xs\n",
"import dask"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# resources from one node"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of CPUs: 48, number of threads: 8, number of workers: 6, processes: True\n"
]
}
],
"source": [
"processes = True\n",
"nworker = 6\n",
"threads = ncpu // nworker\n",
"print(\n",
" f\"Number of CPUs: {ncpu}, number of threads: {threads}, number of workers: {nworker}, processes: {processes}\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table style=\"border: 2px solid white;\">\n",
"<tr>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Client</h3>\n",
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
" <li><b>Scheduler: </b>tcp://127.0.0.1:40397</li>\n",
" <li><b>Dashboard: </b><a href='http://localhost:8888/proxy/8787/status' target='_blank'>http://localhost:8888/proxy/8787/status</a>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
" <li><b>Workers: </b>6</li>\n",
" <li><b>Cores: </b>48</li>\n",
" <li><b>Memory: </b>16.11 GB</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: 'tcp://127.0.0.1:40397' processes=6 threads=48, memory=16.11 GB>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client = Client(\n",
" processes=processes,\n",
" threads_per_worker=threads,\n",
" n_workers=nworker,\n",
" memory_limit=\"128GB\",\n",
")\n",
"\n",
"client"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# resources across many nodes"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from dask_jobqueue import SLURMCluster\n",
"n=48\n",
"cluster = SLURMCluster(cores=n,\n",
" n_workers=ncpu//6,\n",
" memory=\"64GB\",\n",
" project=\"mh0727\",\n",
" interface=\"ib0\")\n",
"\n",
"cluster.scale_up(cores=n*4)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table style=\"border: 2px solid white;\">\n",
"<tr>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Client</h3>\n",
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
" <li><b>Scheduler: </b>tcp://10.50.40.142:44012</li>\n",
" <li><b>Dashboard: </b><a href='http://localhost:8888/proxy/8787/status' target='_blank'>http://localhost:8888/proxy/8787/status</a>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
" <li><b>Workers: </b>0</li>\n",
" <li><b>Cores: </b>0</li>\n",
" <li><b>Memory: </b>0 B</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: 'tcp://10.50.40.142:44012' processes=0 threads=0, memory=0 B>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client = Client(cluster)\n",
"client"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## small data"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def gen_data(nlead=10,res=1, persist=True,chunksize=-1,nmember=10, ninit=50):\n",
" nlat = 180/res\n",
" nlon = 360/res\n",
" lead = np.arange(1900,1900+nlead)\n",
" lat = np.linspace(-89.5, 89.5, nlat)\n",
" lon = np.linspace(0.5, 359.5, nlon)\n",
" member=np.arange(nmember)\n",
" init=np.arange(ninit)\n",
"\n",
"\n",
" a = xr.DataArray(dask.array.random.random((len(lead), len(lat), len(lon),len(member),len(init))),\n",
" dims=[\"lead\", \"lat\", \"lon\",'member','init'],\n",
" coords=[lead, lat, lon, member, init])\n",
"\n",
" chunks = {'lead':chunksize}\n",
" a = a.chunk(chunks)\n",
" a = a.chunk(chunks)\n",
"\n",
" if persist:\n",
" a = a.persist()\n",
" return a\n",
"\n",
"xa=gen_data(res=5, persist=True)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('lead', 'lat', 'lon', 'member', 'init')"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"xa.dims"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 103.68 MB </td> <td> 103.68 MB </td></tr>\n",
" <tr><th> Shape </th><td> (10, 36, 72, 10, 50) </td> <td> (10, 36, 72, 10, 50) </td></tr>\n",
" <tr><th> Count </th><td> 1 Tasks </td><td> 1 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"463\" height=\"160\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"60\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"40\" x2=\"60\" y2=\"40\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"60\" y1=\"0\" x2=\"60\" y2=\"40\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 60.000000,0.000000 60.000000,40.328956 0.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"30.000000\" y=\"60.328956\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >36</text>\n",
" <text x=\"80.000000\" y=\"20.164478\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,80.000000,20.164478)\">10</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"130\" y1=\"40\" x2=\"200\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"130\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"130.000000,0.000000 200.588235,70.588235 200.588235,110.917191 130.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"213\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"283\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"213\" y1=\"0\" x2=\"283\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"130.000000,0.000000 213.333333,0.000000 283.921569,70.588235 200.588235,70.588235\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"200\" y1=\"70\" x2=\"283\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"200\" y1=\"110\" x2=\"283\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"110\" style=\"stroke-width:2\" />\n",
" <line x1=\"283\" y1=\"70\" x2=\"283\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"200.588235,70.588235 283.921569,70.588235 283.921569,110.917191 200.588235,110.917191\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"242.254902\" y=\"130.917191\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >50</text>\n",
" <text x=\"303.921569\" y=\"90.752713\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,303.921569,90.752713)\">10</text>\n",
" <text x=\"155.294118\" y=\"95.623073\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,155.294118,95.623073)\">72</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<random_sample, shape=(10, 36, 72, 10, 50), dtype=float64, chunksize=(10, 36, 72, 10, 50), chunktype=numpy.ndarray>"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"xa.data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## metric functions"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"resample_dim='member'\n",
"dim='init'"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"def mse_xarray(xa,xb,dim=dim):\n",
" return ((xa-xb)**2).mean(dim)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 77 ms, sys: 13 ms, total: 90 ms\n",
"Wall time: 327 ms\n"
]
}
],
"source": [
"%%time\n",
"r = mse_xarray(xa,xa,dim)\n",
"_ = r.compute()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## resampling functions"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"def resample_xarray_isel(hind, shuffle_dim=resample_dim):\n",
" \"\"\"Resample with replacement in dimension `shuffle_dim` from values of\n",
" `to_be_shuffled`\n",
"\n",
" Args:\n",
" hind (xr.object): input xr.objext to be shuffled.\n",
" shuffle_dim (str): dimension to shuffle along.\n",
"\n",
" Returns:\n",
" xr.object: shuffled along `shuffle_dim`.\n",
"\n",
" \"\"\"\n",
" to_be_shuffled = hind[shuffle_dim]\n",
" smp = np.random.randint(0, len(to_be_shuffled),len(to_be_shuffled))\n",
" smp_hind = hind.isel({shuffle_dim: smp})\n",
" smp_hind[shuffle_dim] = hind[shuffle_dim]\n",
" return smp_hind"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Build graph ...\n",
"CPU times: user 9 ms, sys: 1e+03 µs, total: 10 ms\n",
"Wall time: 8.52 ms\n",
"Compute ...\n",
"CPU times: user 167 ms, sys: 328 ms, total: 495 ms\n",
"Wall time: 620 ms\n"
]
}
],
"source": [
"print(f'Build graph ...')\n",
"%time ra = resample_xarray_isel(xa, resample_dim)\n",
"print(f'Compute ...')\n",
"%time rac = ra.compute()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## resample and metric once"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Build graph ...\n",
"CPU times: user 5 ms, sys: 0 ns, total: 5 ms\n",
"Wall time: 5.79 ms\n",
"Compute ...\n",
"CPU times: user 93 ms, sys: 25 ms, total: 118 ms\n",
"Wall time: 390 ms\n"
]
}
],
"source": [
"print(f'Build graph ...')\n",
"%time ra = mse_xarray(resample_xarray_isel(xa, resample_dim), xa, dim)\n",
"print(f'Compute ...')\n",
"%time rac = ra.compute()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## resample metric several times"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"from climpred.bootstrap import my_quantile"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 103.68 MB </td> <td> 103.68 MB </td></tr>\n",
" <tr><th> Shape </th><td> (10, 36, 72, 10, 50) </td> <td> (10, 36, 72, 10, 50) </td></tr>\n",
" <tr><th> Count </th><td> 1 Tasks </td><td> 1 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"463\" height=\"160\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"60\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"40\" x2=\"60\" y2=\"40\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"60\" y1=\"0\" x2=\"60\" y2=\"40\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 60.000000,0.000000 60.000000,40.328956 0.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"30.000000\" y=\"60.328956\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >36</text>\n",
" <text x=\"80.000000\" y=\"20.164478\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,80.000000,20.164478)\">10</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"130\" y1=\"40\" x2=\"200\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"130\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"130.000000,0.000000 200.588235,70.588235 200.588235,110.917191 130.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"213\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"283\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"213\" y1=\"0\" x2=\"283\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"130.000000,0.000000 213.333333,0.000000 283.921569,70.588235 200.588235,70.588235\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"200\" y1=\"70\" x2=\"283\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"200\" y1=\"110\" x2=\"283\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"110\" style=\"stroke-width:2\" />\n",
" <line x1=\"283\" y1=\"70\" x2=\"283\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"200.588235,70.588235 283.921569,70.588235 283.921569,110.917191 200.588235,110.917191\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"242.254902\" y=\"130.917191\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >50</text>\n",
" <text x=\"303.921569\" y=\"90.752713\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,303.921569,90.752713)\">10</text>\n",
" <text x=\"155.294118\" y=\"95.623073\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,155.294118,95.623073)\">72</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<random_sample, shape=(10, 36, 72, 10, 50), dtype=float64, chunksize=(10, 36, 72, 10, 50), chunktype=numpy.ndarray>"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"q=.95\n",
"xa=gen_data(res=5, persist=True)\n",
"xa.data"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"bootstrap = 8*4"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Build graph resampling ...\n",
"CPU times: user 92 ms, sys: 6 ms, total: 98 ms\n",
"Wall time: 95.3 ms\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 3.32 GB </td> <td> 103.68 MB </td></tr>\n",
" <tr><th> Shape </th><td> (32, 10, 36, 72, 10, 50) </td> <td> (1, 10, 36, 72, 10, 50) </td></tr>\n",
" <tr><th> Count </th><td> 97 Tasks </td><td> 32 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"545\" height=\"160\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"41\" y2=\"31\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"40\" x2=\"41\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"41\" />\n",
" <line x1=\"11\" y1=\"1\" x2=\"11\" y2=\"42\" />\n",
" <line x1=\"12\" y1=\"2\" x2=\"12\" y2=\"43\" />\n",
" <line x1=\"13\" y1=\"3\" x2=\"13\" y2=\"44\" />\n",
" <line x1=\"14\" y1=\"4\" x2=\"14\" y2=\"45\" />\n",
" <line x1=\"15\" y1=\"5\" x2=\"15\" y2=\"46\" />\n",
" <line x1=\"16\" y1=\"6\" x2=\"16\" y2=\"47\" />\n",
" <line x1=\"17\" y1=\"7\" x2=\"17\" y2=\"48\" />\n",
" <line x1=\"18\" y1=\"8\" x2=\"18\" y2=\"49\" />\n",
" <line x1=\"19\" y1=\"9\" x2=\"19\" y2=\"50\" />\n",
" <line x1=\"20\" y1=\"10\" x2=\"20\" y2=\"51\" />\n",
" <line x1=\"21\" y1=\"11\" x2=\"21\" y2=\"52\" />\n",
" <line x1=\"22\" y1=\"12\" x2=\"22\" y2=\"53\" />\n",
" <line x1=\"23\" y1=\"13\" x2=\"23\" y2=\"54\" />\n",
" <line x1=\"24\" y1=\"14\" x2=\"24\" y2=\"55\" />\n",
" <line x1=\"25\" y1=\"15\" x2=\"25\" y2=\"56\" />\n",
" <line x1=\"26\" y1=\"16\" x2=\"26\" y2=\"56\" />\n",
" <line x1=\"27\" y1=\"17\" x2=\"27\" y2=\"57\" />\n",
" <line x1=\"28\" y1=\"18\" x2=\"28\" y2=\"58\" />\n",
" <line x1=\"29\" y1=\"19\" x2=\"29\" y2=\"59\" />\n",
" <line x1=\"30\" y1=\"20\" x2=\"30\" y2=\"60\" />\n",
" <line x1=\"31\" y1=\"21\" x2=\"31\" y2=\"61\" />\n",
" <line x1=\"32\" y1=\"22\" x2=\"32\" y2=\"62\" />\n",
" <line x1=\"33\" y1=\"23\" x2=\"33\" y2=\"63\" />\n",
" <line x1=\"34\" y1=\"24\" x2=\"34\" y2=\"64\" />\n",
" <line x1=\"35\" y1=\"25\" x2=\"35\" y2=\"65\" />\n",
" <line x1=\"36\" y1=\"26\" x2=\"36\" y2=\"66\" />\n",
" <line x1=\"37\" y1=\"27\" x2=\"37\" y2=\"67\" />\n",
" <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"68\" />\n",
" <line x1=\"39\" y1=\"29\" x2=\"39\" y2=\"69\" />\n",
" <line x1=\"40\" y1=\"30\" x2=\"40\" y2=\"70\" />\n",
" <line x1=\"41\" y1=\"31\" x2=\"41\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.000000,0.000000 41.372549,31.372549 41.372549,71.701505 10.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"70\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"0\" x2=\"70\" y2=\"0\" />\n",
" <line x1=\"11\" y1=\"1\" x2=\"71\" y2=\"1\" />\n",
" <line x1=\"12\" y1=\"2\" x2=\"72\" y2=\"2\" />\n",
" <line x1=\"13\" y1=\"3\" x2=\"73\" y2=\"3\" />\n",
" <line x1=\"14\" y1=\"4\" x2=\"74\" y2=\"4\" />\n",
" <line x1=\"15\" y1=\"5\" x2=\"75\" y2=\"5\" />\n",
" <line x1=\"16\" y1=\"6\" x2=\"76\" y2=\"6\" />\n",
" <line x1=\"17\" y1=\"7\" x2=\"77\" y2=\"7\" />\n",
" <line x1=\"18\" y1=\"8\" x2=\"78\" y2=\"8\" />\n",
" <line x1=\"19\" y1=\"9\" x2=\"79\" y2=\"9\" />\n",
" <line x1=\"20\" y1=\"10\" x2=\"80\" y2=\"10\" />\n",
" <line x1=\"21\" y1=\"11\" x2=\"81\" y2=\"11\" />\n",
" <line x1=\"22\" y1=\"12\" x2=\"82\" y2=\"12\" />\n",
" <line x1=\"23\" y1=\"13\" x2=\"83\" y2=\"13\" />\n",
" <line x1=\"24\" y1=\"14\" x2=\"84\" y2=\"14\" />\n",
" <line x1=\"25\" y1=\"15\" x2=\"85\" y2=\"15\" />\n",
" <line x1=\"26\" y1=\"16\" x2=\"86\" y2=\"16\" />\n",
" <line x1=\"27\" y1=\"17\" x2=\"87\" y2=\"17\" />\n",
" <line x1=\"28\" y1=\"18\" x2=\"88\" y2=\"18\" />\n",
" <line x1=\"29\" y1=\"19\" x2=\"89\" y2=\"19\" />\n",
" <line x1=\"30\" y1=\"20\" x2=\"90\" y2=\"20\" />\n",
" <line x1=\"31\" y1=\"21\" x2=\"91\" y2=\"21\" />\n",
" <line x1=\"32\" y1=\"22\" x2=\"92\" y2=\"22\" />\n",
" <line x1=\"33\" y1=\"23\" x2=\"93\" y2=\"23\" />\n",
" <line x1=\"34\" y1=\"24\" x2=\"94\" y2=\"24\" />\n",
" <line x1=\"35\" y1=\"25\" x2=\"95\" y2=\"25\" />\n",
" <line x1=\"36\" y1=\"26\" x2=\"96\" y2=\"26\" />\n",
" <line x1=\"37\" y1=\"27\" x2=\"97\" y2=\"27\" />\n",
" <line x1=\"38\" y1=\"28\" x2=\"98\" y2=\"28\" />\n",
" <line x1=\"39\" y1=\"29\" x2=\"99\" y2=\"29\" />\n",
" <line x1=\"40\" y1=\"30\" x2=\"100\" y2=\"30\" />\n",
" <line x1=\"41\" y1=\"31\" x2=\"101\" y2=\"31\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"41\" y2=\"31\" style=\"stroke-width:2\" />\n",
" <line x1=\"70\" y1=\"0\" x2=\"101\" y2=\"31\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.000000,0.000000 70.000000,0.000000 101.372549,31.372549 41.372549,31.372549\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"41\" y1=\"31\" x2=\"101\" y2=\"31\" style=\"stroke-width:2\" />\n",
" <line x1=\"41\" y1=\"71\" x2=\"101\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"41\" y1=\"31\" x2=\"41\" y2=\"71\" style=\"stroke-width:2\" />\n",
" <line x1=\"101\" y1=\"31\" x2=\"101\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"41.372549,31.372549 101.372549,31.372549 101.372549,71.701505 41.372549,71.701505\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"71.372549\" y=\"91.701505\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >36</text>\n",
" <text x=\"121.372549\" y=\"51.537027\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,121.372549,51.537027)\">10</text>\n",
" <text x=\"15.686275\" y=\"76.015230\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,15.686275,76.015230)\">32</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"241\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"171\" y1=\"40\" x2=\"241\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"171\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"241\" y1=\"70\" x2=\"241\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"171.000000,0.000000 241.588235,70.588235 241.588235,110.917191 171.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"254\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"241\" y1=\"70\" x2=\"324\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"241\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"254\" y1=\"0\" x2=\"324\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"171.000000,0.000000 254.333333,0.000000 324.921569,70.588235 241.588235,70.588235\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"241\" y1=\"70\" x2=\"324\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"241\" y1=\"110\" x2=\"324\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"241\" y1=\"70\" x2=\"241\" y2=\"110\" style=\"stroke-width:2\" />\n",
" <line x1=\"324\" y1=\"70\" x2=\"324\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"241.588235,70.588235 324.921569,70.588235 324.921569,110.917191 241.588235,110.917191\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"283.254902\" y=\"130.917191\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >50</text>\n",
" <text x=\"344.921569\" y=\"90.752713\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,344.921569,90.752713)\">10</text>\n",
" <text x=\"196.294118\" y=\"95.623073\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,196.294118,95.623073)\">72</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<concatenate, shape=(32, 10, 36, 72, 10, 50), dtype=float64, chunksize=(1, 10, 36, 72, 10, 50), chunktype=numpy.ndarray>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Metric ...\n",
"CPU times: user 7 ms, sys: 1 ms, total: 8 ms\n",
"Wall time: 7.21 ms\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 66.36 MB </td> <td> 2.07 MB </td></tr>\n",
" <tr><th> Shape </th><td> (32, 10, 36, 72, 10) </td> <td> (1, 10, 36, 72, 10) </td></tr>\n",
" <tr><th> Count </th><td> 226 Tasks </td><td> 32 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"345\" height=\"205\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"40\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"1\" x2=\"40\" y2=\"1\" />\n",
" <line x1=\"0\" y1=\"3\" x2=\"40\" y2=\"3\" />\n",
" <line x1=\"0\" y1=\"5\" x2=\"40\" y2=\"5\" />\n",
" <line x1=\"0\" y1=\"6\" x2=\"40\" y2=\"6\" />\n",
" <line x1=\"0\" y1=\"8\" x2=\"40\" y2=\"8\" />\n",
" <line x1=\"0\" y1=\"10\" x2=\"40\" y2=\"10\" />\n",
" <line x1=\"0\" y1=\"11\" x2=\"40\" y2=\"11\" />\n",
" <line x1=\"0\" y1=\"13\" x2=\"40\" y2=\"13\" />\n",
" <line x1=\"0\" y1=\"15\" x2=\"40\" y2=\"15\" />\n",
" <line x1=\"0\" y1=\"16\" x2=\"40\" y2=\"16\" />\n",
" <line x1=\"0\" y1=\"18\" x2=\"40\" y2=\"18\" />\n",
" <line x1=\"0\" y1=\"20\" x2=\"40\" y2=\"20\" />\n",
" <line x1=\"0\" y1=\"21\" x2=\"40\" y2=\"21\" />\n",
" <line x1=\"0\" y1=\"23\" x2=\"40\" y2=\"23\" />\n",
" <line x1=\"0\" y1=\"25\" x2=\"40\" y2=\"25\" />\n",
" <line x1=\"0\" y1=\"26\" x2=\"40\" y2=\"26\" />\n",
" <line x1=\"0\" y1=\"28\" x2=\"40\" y2=\"28\" />\n",
" <line x1=\"0\" y1=\"30\" x2=\"40\" y2=\"30\" />\n",
" <line x1=\"0\" y1=\"31\" x2=\"40\" y2=\"31\" />\n",
" <line x1=\"0\" y1=\"33\" x2=\"40\" y2=\"33\" />\n",
" <line x1=\"0\" y1=\"35\" x2=\"40\" y2=\"35\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"40\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"38\" x2=\"40\" y2=\"38\" />\n",
" <line x1=\"0\" y1=\"40\" x2=\"40\" y2=\"40\" />\n",
" <line x1=\"0\" y1=\"41\" x2=\"40\" y2=\"41\" />\n",
" <line x1=\"0\" y1=\"43\" x2=\"40\" y2=\"43\" />\n",
" <line x1=\"0\" y1=\"45\" x2=\"40\" y2=\"45\" />\n",
" <line x1=\"0\" y1=\"46\" x2=\"40\" y2=\"46\" />\n",
" <line x1=\"0\" y1=\"48\" x2=\"40\" y2=\"48\" />\n",
" <line x1=\"0\" y1=\"50\" x2=\"40\" y2=\"50\" />\n",
" <line x1=\"0\" y1=\"51\" x2=\"40\" y2=\"51\" />\n",
" <line x1=\"0\" y1=\"53\" x2=\"40\" y2=\"53\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"53\" style=\"stroke-width:2\" />\n",
" <line x1=\"40\" y1=\"0\" x2=\"40\" y2=\"53\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 40.328956,0.000000 40.328956,53.333333 0.000000,53.333333\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"20.164478\" y=\"73.333333\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >10</text>\n",
" <text x=\"60.328956\" y=\"26.666667\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,60.328956,26.666667)\">32</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"145\" y2=\"35\" style=\"stroke-width:2\" />\n",
" <line x1=\"110\" y1=\"120\" x2=\"145\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"110\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"145\" y1=\"35\" x2=\"145\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"110.000000,0.000000 145.294118,35.294118 145.294118,155.294118 110.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"150\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"145\" y1=\"35\" x2=\"185\" y2=\"35\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"145\" y2=\"35\" style=\"stroke-width:2\" />\n",
" <line x1=\"150\" y1=\"0\" x2=\"185\" y2=\"35\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"110.000000,0.000000 150.328956,0.000000 185.623073,35.294118 145.294118,35.294118\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"145\" y1=\"35\" x2=\"185\" y2=\"35\" style=\"stroke-width:2\" />\n",
" <line x1=\"145\" y1=\"155\" x2=\"185\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"145\" y1=\"35\" x2=\"145\" y2=\"155\" style=\"stroke-width:2\" />\n",
" <line x1=\"185\" y1=\"35\" x2=\"185\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"145.294118,35.294118 185.623073,35.294118 185.623073,155.294118 145.294118,155.294118\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"165.458595\" y=\"175.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >10</text>\n",
" <text x=\"205.623073\" y=\"95.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,205.623073,95.294118)\">72</text>\n",
" <text x=\"117.647059\" y=\"157.647059\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,117.647059,157.647059)\">36</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<mean_agg-aggregate, shape=(32, 10, 36, 72, 10), dtype=float64, chunksize=(1, 10, 36, 72, 10), chunktype=numpy.ndarray>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Quantile ...\n",
"CPU times: user 7 ms, sys: 0 ns, total: 7 ms\n",
"Wall time: 7.45 ms\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 2.07 MB </td> <td> 2.07 MB </td></tr>\n",
" <tr><th> Shape </th><td> (10, 36, 72, 10) </td> <td> (10, 36, 72, 10) </td></tr>\n",
" <tr><th> Count </th><td> 230 Tasks </td><td> 1 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"345\" height=\"205\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"40\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"27\" x2=\"40\" y2=\"27\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"27\" style=\"stroke-width:2\" />\n",
" <line x1=\"40\" y1=\"0\" x2=\"40\" y2=\"27\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 40.328956,0.000000 40.328956,27.054036 0.000000,27.054036\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"20.164478\" y=\"47.054036\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >10</text>\n",
" <text x=\"60.328956\" y=\"13.527018\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,60.328956,13.527018)\">1</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"145\" y2=\"35\" style=\"stroke-width:2\" />\n",
" <line x1=\"110\" y1=\"120\" x2=\"145\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"110\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"145\" y1=\"35\" x2=\"145\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"110.000000,0.000000 145.294118,35.294118 145.294118,155.294118 110.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"150\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"145\" y1=\"35\" x2=\"185\" y2=\"35\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"145\" y2=\"35\" style=\"stroke-width:2\" />\n",
" <line x1=\"150\" y1=\"0\" x2=\"185\" y2=\"35\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"110.000000,0.000000 150.328956,0.000000 185.623073,35.294118 145.294118,35.294118\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"145\" y1=\"35\" x2=\"185\" y2=\"35\" style=\"stroke-width:2\" />\n",
" <line x1=\"145\" y1=\"155\" x2=\"185\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"145\" y1=\"35\" x2=\"145\" y2=\"155\" style=\"stroke-width:2\" />\n",
" <line x1=\"185\" y1=\"35\" x2=\"185\" y2=\"155\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"145.294118,35.294118 185.623073,35.294118 185.623073,155.294118 145.294118,155.294118\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"165.458595\" y=\"175.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >10</text>\n",
" <text x=\"205.623073\" y=\"95.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,205.623073,95.294118)\">72</text>\n",
" <text x=\"117.647059\" y=\"157.647059\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,117.647059,157.647059)\">36</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<getitem, shape=(10, 36, 72, 10), dtype=float64, chunksize=(10, 36, 72, 10), chunktype=numpy.ndarray>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 1.06 s, sys: 60 ms, total: 1.12 s\n",
"Wall time: 2.45 s\n"
]
}
],
"source": [
"print(f'Build graph resampling ...')\n",
"%time ra = xr.concat([resample_xarray_isel(xa) for _ in range(bootstrap)],'bootstrap')\n",
"display(ra.data)\n",
"print(f'Metric ...')\n",
"%time ra = mse_xarray(ra, xa)\n",
"display(ra.data)\n",
"print(f'Quantile ...')\n",
"%time ra = my_quantile(ra,q=q,dim='bootstrap')\n",
"display(ra.data)\n",
"#%time rac_pure_true = client.compute(ra, pure=False).result()\n",
"%time rac = ra.compute()"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(10, 36, 72, 10)"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rac.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### serial eager computation"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 7.46 s, sys: 7.85 s, total: 15.3 s\n",
"Wall time: 13.9 s\n"
]
}
],
"source": [
"xa = xa.load()\n",
"#%time ra = mse_xarray(xr.concat([resample_xarray_isel(xa) for _ in range(bootstrap)],'bootstrap'),xa,dim)\n",
"%time ra = my_quantile(mse_xarray(xr.concat([resample_xarray_isel(xa) for _ in range(bootstrap)],'bootstrap'),xa,dim),q=q,dim=dim)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- eager is faster 😞 on a single node\n",
"- lazy is faster 🤗 on multiple nodes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## client.map"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"xa=gen_data(res=5, persist=True)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"def resample(dummy):\n",
" return resample_xarray_isel(xa)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 3.32 GB </td> <td> 103.68 MB </td></tr>\n",
" <tr><th> Shape </th><td> (32, 10, 36, 72, 10, 50) </td> <td> (1, 10, 36, 72, 10, 50) </td></tr>\n",
" <tr><th> Count </th><td> 97 Tasks </td><td> 32 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"545\" height=\"160\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"41\" y2=\"31\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"40\" x2=\"41\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"41\" />\n",
" <line x1=\"11\" y1=\"1\" x2=\"11\" y2=\"42\" />\n",
" <line x1=\"12\" y1=\"2\" x2=\"12\" y2=\"43\" />\n",
" <line x1=\"13\" y1=\"3\" x2=\"13\" y2=\"44\" />\n",
" <line x1=\"14\" y1=\"4\" x2=\"14\" y2=\"45\" />\n",
" <line x1=\"15\" y1=\"5\" x2=\"15\" y2=\"46\" />\n",
" <line x1=\"16\" y1=\"6\" x2=\"16\" y2=\"47\" />\n",
" <line x1=\"17\" y1=\"7\" x2=\"17\" y2=\"48\" />\n",
" <line x1=\"18\" y1=\"8\" x2=\"18\" y2=\"49\" />\n",
" <line x1=\"19\" y1=\"9\" x2=\"19\" y2=\"50\" />\n",
" <line x1=\"20\" y1=\"10\" x2=\"20\" y2=\"51\" />\n",
" <line x1=\"21\" y1=\"11\" x2=\"21\" y2=\"52\" />\n",
" <line x1=\"22\" y1=\"12\" x2=\"22\" y2=\"53\" />\n",
" <line x1=\"23\" y1=\"13\" x2=\"23\" y2=\"54\" />\n",
" <line x1=\"24\" y1=\"14\" x2=\"24\" y2=\"55\" />\n",
" <line x1=\"25\" y1=\"15\" x2=\"25\" y2=\"56\" />\n",
" <line x1=\"26\" y1=\"16\" x2=\"26\" y2=\"56\" />\n",
" <line x1=\"27\" y1=\"17\" x2=\"27\" y2=\"57\" />\n",
" <line x1=\"28\" y1=\"18\" x2=\"28\" y2=\"58\" />\n",
" <line x1=\"29\" y1=\"19\" x2=\"29\" y2=\"59\" />\n",
" <line x1=\"30\" y1=\"20\" x2=\"30\" y2=\"60\" />\n",
" <line x1=\"31\" y1=\"21\" x2=\"31\" y2=\"61\" />\n",
" <line x1=\"32\" y1=\"22\" x2=\"32\" y2=\"62\" />\n",
" <line x1=\"33\" y1=\"23\" x2=\"33\" y2=\"63\" />\n",
" <line x1=\"34\" y1=\"24\" x2=\"34\" y2=\"64\" />\n",
" <line x1=\"35\" y1=\"25\" x2=\"35\" y2=\"65\" />\n",
" <line x1=\"36\" y1=\"26\" x2=\"36\" y2=\"66\" />\n",
" <line x1=\"37\" y1=\"27\" x2=\"37\" y2=\"67\" />\n",
" <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"68\" />\n",
" <line x1=\"39\" y1=\"29\" x2=\"39\" y2=\"69\" />\n",
" <line x1=\"40\" y1=\"30\" x2=\"40\" y2=\"70\" />\n",
" <line x1=\"41\" y1=\"31\" x2=\"41\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.000000,0.000000 41.372549,31.372549 41.372549,71.701505 10.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"70\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"0\" x2=\"70\" y2=\"0\" />\n",
" <line x1=\"11\" y1=\"1\" x2=\"71\" y2=\"1\" />\n",
" <line x1=\"12\" y1=\"2\" x2=\"72\" y2=\"2\" />\n",
" <line x1=\"13\" y1=\"3\" x2=\"73\" y2=\"3\" />\n",
" <line x1=\"14\" y1=\"4\" x2=\"74\" y2=\"4\" />\n",
" <line x1=\"15\" y1=\"5\" x2=\"75\" y2=\"5\" />\n",
" <line x1=\"16\" y1=\"6\" x2=\"76\" y2=\"6\" />\n",
" <line x1=\"17\" y1=\"7\" x2=\"77\" y2=\"7\" />\n",
" <line x1=\"18\" y1=\"8\" x2=\"78\" y2=\"8\" />\n",
" <line x1=\"19\" y1=\"9\" x2=\"79\" y2=\"9\" />\n",
" <line x1=\"20\" y1=\"10\" x2=\"80\" y2=\"10\" />\n",
" <line x1=\"21\" y1=\"11\" x2=\"81\" y2=\"11\" />\n",
" <line x1=\"22\" y1=\"12\" x2=\"82\" y2=\"12\" />\n",
" <line x1=\"23\" y1=\"13\" x2=\"83\" y2=\"13\" />\n",
" <line x1=\"24\" y1=\"14\" x2=\"84\" y2=\"14\" />\n",
" <line x1=\"25\" y1=\"15\" x2=\"85\" y2=\"15\" />\n",
" <line x1=\"26\" y1=\"16\" x2=\"86\" y2=\"16\" />\n",
" <line x1=\"27\" y1=\"17\" x2=\"87\" y2=\"17\" />\n",
" <line x1=\"28\" y1=\"18\" x2=\"88\" y2=\"18\" />\n",
" <line x1=\"29\" y1=\"19\" x2=\"89\" y2=\"19\" />\n",
" <line x1=\"30\" y1=\"20\" x2=\"90\" y2=\"20\" />\n",
" <line x1=\"31\" y1=\"21\" x2=\"91\" y2=\"21\" />\n",
" <line x1=\"32\" y1=\"22\" x2=\"92\" y2=\"22\" />\n",
" <line x1=\"33\" y1=\"23\" x2=\"93\" y2=\"23\" />\n",
" <line x1=\"34\" y1=\"24\" x2=\"94\" y2=\"24\" />\n",
" <line x1=\"35\" y1=\"25\" x2=\"95\" y2=\"25\" />\n",
" <line x1=\"36\" y1=\"26\" x2=\"96\" y2=\"26\" />\n",
" <line x1=\"37\" y1=\"27\" x2=\"97\" y2=\"27\" />\n",
" <line x1=\"38\" y1=\"28\" x2=\"98\" y2=\"28\" />\n",
" <line x1=\"39\" y1=\"29\" x2=\"99\" y2=\"29\" />\n",
" <line x1=\"40\" y1=\"30\" x2=\"100\" y2=\"30\" />\n",
" <line x1=\"41\" y1=\"31\" x2=\"101\" y2=\"31\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"41\" y2=\"31\" style=\"stroke-width:2\" />\n",
" <line x1=\"70\" y1=\"0\" x2=\"101\" y2=\"31\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.000000,0.000000 70.000000,0.000000 101.372549,31.372549 41.372549,31.372549\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"41\" y1=\"31\" x2=\"101\" y2=\"31\" style=\"stroke-width:2\" />\n",
" <line x1=\"41\" y1=\"71\" x2=\"101\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"41\" y1=\"31\" x2=\"41\" y2=\"71\" style=\"stroke-width:2\" />\n",
" <line x1=\"101\" y1=\"31\" x2=\"101\" y2=\"71\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"41.372549,31.372549 101.372549,31.372549 101.372549,71.701505 41.372549,71.701505\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"71.372549\" y=\"91.701505\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >36</text>\n",
" <text x=\"121.372549\" y=\"51.537027\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,121.372549,51.537027)\">10</text>\n",
" <text x=\"15.686275\" y=\"76.015230\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,15.686275,76.015230)\">32</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"241\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"171\" y1=\"40\" x2=\"241\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"171\" y2=\"40\" style=\"stroke-width:2\" />\n",
" <line x1=\"241\" y1=\"70\" x2=\"241\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"171.000000,0.000000 241.588235,70.588235 241.588235,110.917191 171.000000,40.328956\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"254\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"241\" y1=\"70\" x2=\"324\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"171\" y1=\"0\" x2=\"241\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"254\" y1=\"0\" x2=\"324\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"171.000000,0.000000 254.333333,0.000000 324.921569,70.588235 241.588235,70.588235\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"241\" y1=\"70\" x2=\"324\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"241\" y1=\"110\" x2=\"324\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"241\" y1=\"70\" x2=\"241\" y2=\"110\" style=\"stroke-width:2\" />\n",
" <line x1=\"324\" y1=\"70\" x2=\"324\" y2=\"110\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"241.588235,70.588235 324.921569,70.588235 324.921569,110.917191 241.588235,110.917191\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"283.254902\" y=\"130.917191\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >50</text>\n",
" <text x=\"344.921569\" y=\"90.752713\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,344.921569,90.752713)\">10</text>\n",
" <text x=\"196.294118\" y=\"95.623073\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,196.294118,95.623073)\">72</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<concatenate, shape=(32, 10, 36, 72, 10, 50), dtype=float64, chunksize=(1, 10, 36, 72, 10, 50), chunktype=numpy.ndarray>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 5.67 s, sys: 11.3 s, total: 16.9 s\n",
"Wall time: 18.6 s\n"
]
}
],
"source": [
"%%time\n",
"x = client.map(resample, list(range(bootstrap)))\n",
"y = client.map(mse_xarray, x, xa, dim)\n",
"r = xr.concat(client.gather(x),'bootstrap')\n",
"display(r.data)\n",
"#r=my_quantile(r,q=q,dim='bootstrap')\n",
"r=r.compute()"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(8, 10, 36, 72, 10, 50)"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"r.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"client.close()\n",
"cluster.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:pymistral] *",
"language": "python",
"name": "conda-env-pymistral-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment