Skip to content

Instantly share code, notes, and snippets.

@delgadom

delgadom/diagnose_xarray_fut_to_dask_performance.ipynb

Created August 26, 2020 00:40
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save delgadom/f86e0716fb6ba0111fd53ee8da527fe2 to your computer and use it in GitHub Desktop.
Save delgadom/f86e0716fb6ba0111fd53ee8da527fe2 to your computer and use it in GitHub Desktop.
Download ZIP
Debugging performance bottleneck in xarray futures to dask-backed array
Display the source blob
Display the rendered blob
Raw
diagnose_xarray_fut_to_dask_performance.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import functools\n",
"\n",
"import dask.array\n",
"import numpy as np\n",
"import xarray as xr\n",
"from dask import distributed as dd\n",
"\n",
"\n",
"def dataarrays_from_delayed(futures, client=None):\n",
" \"\"\"\n",
" Returns a list of xarray dataarrays from a list of futures of dataarrays\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.DataArray` objects.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" arrays : list\n",
" list of :py:class:`xarray.DataArray` objects with\n",
" :py:class:`dask.array.Array` backends.\n",
" Examples\n",
" --------\n",
" Given a mapped xarray DataArray, pull the metadata into memory while\n",
" leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np\n",
" >>> def build_arr(multiplier):\n",
" ... return multiplier * xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_arr, range(3))\n",
" >>> arrs = dataarrays_from_delayed(fut)\n",
" >>> arrs[-1] # doctest: +ELLIPSIS\n",
" <xarray.DataArray ...(x: 2)>\n",
" dask.array<...shape=(2,), dtype=int64, chunksize=(2,), chunktype=numpy.ndarray>\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" This list of arrays can now be manipulated using normal xarray tools:\n",
" .. code-block:: python\n",
" >>> xr.concat(arrs, dim='simulation') # doctest: +ELLIPSIS\n",
" <xarray.DataArray ...(simulation: 3, x: 2)>\n",
" dask.array<...shape=(3, 2), dtype=int64, chunksize=(1, 2), chunktype=numpy.ndarray>\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" Dimensions without coordinates: simulation\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" if client is None:\n",
" client = dd.get_client()\n",
"\n",
" delayed_arrays = client.map(lambda x: x.data, futures)\n",
"\n",
" dask_array_metadata = client.gather(\n",
" client.map(lambda x: (x.data.shape, x.data.dtype), futures)\n",
" )\n",
"\n",
" dask_arrays = [\n",
" dask.array.from_delayed(delayed_arrays[i], *dask_array_metadata[i])\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" # using dict(x.coords) b/c gathering coords can blow up memory for some reason\n",
" array_metadata = client.gather(\n",
" client.map(\n",
" lambda x: {\n",
" \"dims\": x.dims,\n",
" \"coords\": dict(x.coords),\n",
" \"attrs\": x.attrs,\n",
" \"name\": x.name,\n",
" },\n",
" futures,\n",
" )\n",
" )\n",
"\n",
" data_arrays = [\n",
" xr.DataArray(dask_arrays[i], **array_metadata[i]) for i in range(len(futures))\n",
" ]\n",
"\n",
" return data_arrays\n",
"\n",
"\n",
"def dataarray_from_delayed(futures, dim=None, client=None):\n",
" \"\"\"\n",
" Returns a DataArray from a list of futures\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.DataArray` objects.\n",
" dim : str, optional\n",
" dimension along which to concat :py:class:`xarray.DataArray`.\n",
" Inferred by default.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" array : object\n",
" :py:class:`xarray.DataArray` concatenated along ``dim`` with\n",
" a :py:class:`dask.array.Array` backend.\n",
" Examples\n",
" --------\n",
" Given a mapped xarray DataArray, pull the metadata into memory while\n",
" leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np, pandas as pd\n",
" >>> def build_arr(multiplier):\n",
" ... return multiplier * xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_arr, range(3))\n",
" >>> da = dataarray_from_delayed(\n",
" ... fut,\n",
" ... dim=pd.Index(range(3), name='simulation'))\n",
" ...\n",
" >>> da # doctest: +ELLIPSIS\n",
" <xarray.DataArray ...(simulation: 3, x: 2)>\n",
" dask.array<...shape=(3, 2), dtype=int64, chunksize=(1, 2), chunktype=numpy.ndarray>\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" * simulation (simulation) int64 0 1 2\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" data_arrays = dataarrays_from_delayed(futures, client=client)\n",
" da = xr.concat(data_arrays, dim=dim)\n",
"\n",
" return da\n",
"\n",
"\n",
"def datasets_from_delayed(futures, client=None):\n",
" \"\"\"\n",
" Returns a list of xarray datasets from a list of futures of datasets\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.Dataset` objects.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" datasets : list\n",
" list of :py:class:`xarray.Dataset` objects with\n",
" :py:class:`dask.array.Array` backends for each variable.\n",
" Examples\n",
" --------\n",
" Given a mapped :py:class:`xarray.Dataset`, pull the metadata into memory\n",
" while leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np\n",
" >>> def build_ds(multiplier):\n",
" ... return multiplier * xr.Dataset({\n",
" ... 'var1': xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])})\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_ds, range(3))\n",
" >>> arrs = datasets_from_delayed(fut)\n",
" >>> arrs[-1] # doctest: +ELLIPSIS\n",
" <xarray.Dataset>\n",
" Dimensions: (x: 2)\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" Data variables:\n",
" var1 (x) int64 dask.array<chunksize=(2,), meta=np.ndarray>\n",
" This list of arrays can now be manipulated using normal xarray tools:\n",
" .. code-block:: python\n",
" >>> xr.concat(arrs, dim='y') # doctest: +ELLIPSIS\n",
" <xarray.Dataset>\n",
" Dimensions: (x: 2, y: 3)\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" Dimensions without coordinates: y\n",
" Data variables:\n",
" var1 (y, x) int64 dask.array<chunksize=(1, 2), meta=np.ndarray>\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" if client is None:\n",
" client = dd.get_client()\n",
"\n",
" data_var_keys = client.gather(\n",
" client.map(lambda x: list(x.data_vars.keys()), futures)\n",
" )\n",
"\n",
" delayed_arrays = [\n",
" {k: (client.submit(lambda x: x[k].data, futures[i])) for k in data_var_keys[i]}\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" dask_array_metadata = [\n",
" {\n",
" k: (\n",
" client.submit(\n",
" lambda x: (x[k].data.shape, x[k].data.dtype), futures[i]\n",
" ).result()\n",
" )\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" dask_data_arrays = [\n",
" {\n",
" k: (\n",
" dask.array.from_delayed(\n",
" delayed_arrays[i][k], *dask_array_metadata[i][k]\n",
" )\n",
" )\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" # using dict(x.coords) b/c gathering coords can blow up memory for some reason\n",
" array_metadata = [\n",
" {\n",
" k: client.submit(\n",
" lambda x: {\n",
" \"dims\": x[k].dims,\n",
" \"coords\": dict(x[k].coords),\n",
" \"attrs\": x[k].attrs,\n",
" },\n",
" futures[i],\n",
" ).result()\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" data_arrays = [\n",
" {\n",
" k: (xr.DataArray(dask_data_arrays[i][k], **array_metadata[i][k]))\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" datasets = [xr.Dataset(arr) for arr in data_arrays]\n",
"\n",
" dataset_metadata = client.gather(client.map(lambda x: x.attrs, futures))\n",
"\n",
" for i in range(len(futures)):\n",
" datasets[i].attrs.update(dataset_metadata[i])\n",
"\n",
" return datasets\n",
"\n",
"\n",
"def dataset_from_delayed(futures, dim=None, client=None):\n",
" \"\"\"\n",
" Returns an :py:class:`xarray.Dataset` from a list of futures\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.Dataset` objects.\n",
" dim : str, optional\n",
" dimension along which to concat :py:class:`xarray.Dataset`.\n",
" Inferred by default.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" dataset : object\n",
" :py:class:`xarray.Dataset` concatenated along ``dim`` with\n",
" :py:class:`dask.array.Array` backends for each variable.\n",
" Examples\n",
" --------\n",
" Given a mapped :py:class:`xarray.Dataset`, pull the metadata into memory\n",
" while leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np, pandas as pd\n",
" >>> def build_ds(multiplier):\n",
" ... return multiplier * xr.Dataset({\n",
" ... 'var1': xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])})\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_ds, range(3))\n",
" >>> ds = dataset_from_delayed(fut, dim=pd.Index(range(3), name='y'))\n",
" >>> ds\n",
" <xarray.Dataset>\n",
" Dimensions: (x: 2, y: 3)\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" * y (y) int64 0 1 2\n",
" Data variables:\n",
" var1 (y, x) int64 dask.array<chunksize=(1, 2), meta=np.ndarray>\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" datasets = datasets_from_delayed(futures, client=client)\n",
" ds = xr.concat(datasets, dim=dim)\n",
"\n",
" return ds\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import functools\n",
"\n",
"import dask.array\n",
"import numpy as np\n",
"import xarray as xr\n",
"from dask import distributed as dd\n",
"\n",
"\n",
"def dataarrays_from_delayed_old(futures, client=None):\n",
" \"\"\"\n",
" Returns a list of xarray dataarrays from a list of futures of dataarrays\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.DataArray` objects.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" arrays : list\n",
" list of :py:class:`xarray.DataArray` objects with\n",
" :py:class:`dask.array.Array` backends.\n",
" Examples\n",
" --------\n",
" Given a mapped xarray DataArray, pull the metadata into memory while\n",
" leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np\n",
" >>> def build_arr(multiplier):\n",
" ... return multiplier * xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_arr, range(3))\n",
" >>> arrs = dataarrays_from_delayed(fut)\n",
" >>> arrs[-1] # doctest: +ELLIPSIS\n",
" <xarray.DataArray ...(x: 2)>\n",
" dask.array<...shape=(2,), dtype=int64, chunksize=(2,), chunktype=numpy.ndarray>\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" This list of arrays can now be manipulated using normal xarray tools:\n",
" .. code-block:: python\n",
" >>> xr.concat(arrs, dim='simulation') # doctest: +ELLIPSIS\n",
" <xarray.DataArray ...(simulation: 3, x: 2)>\n",
" dask.array<...shape=(3, 2), dtype=int64, chunksize=(1, 2), chunktype=numpy.ndarray>\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" Dimensions without coordinates: simulation\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" if client is None:\n",
" client = dd.get_client()\n",
"\n",
" delayed_arrays = client.map(lambda x: x.data, futures)\n",
"\n",
" dask_array_metadata = client.gather(\n",
" client.map(lambda x: (x.data.shape, x.data.dtype), futures)\n",
" )\n",
"\n",
" dask_arrays = [\n",
" dask.array.from_delayed(delayed_arrays[i], *dask_array_metadata[i])\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" # using dict(x.coords) b/c gathering coords can blow up memory for some reason\n",
" array_metadata = client.gather(\n",
" client.map(\n",
" lambda x: {\n",
" \"dims\": x.dims,\n",
" \"coords\": x.coords,\n",
" \"attrs\": x.attrs,\n",
" \"name\": x.name,\n",
" },\n",
" futures,\n",
" )\n",
" )\n",
"\n",
" data_arrays = [\n",
" xr.DataArray(dask_arrays[i], **array_metadata[i]) for i in range(len(futures))\n",
" ]\n",
"\n",
" return data_arrays\n",
"\n",
"\n",
"def dataarray_from_delayed_old(futures, dim=None, client=None):\n",
" \"\"\"\n",
" Returns a DataArray from a list of futures\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.DataArray` objects.\n",
" dim : str, optional\n",
" dimension along which to concat :py:class:`xarray.DataArray`.\n",
" Inferred by default.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" array : object\n",
" :py:class:`xarray.DataArray` concatenated along ``dim`` with\n",
" a :py:class:`dask.array.Array` backend.\n",
" Examples\n",
" --------\n",
" Given a mapped xarray DataArray, pull the metadata into memory while\n",
" leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np, pandas as pd\n",
" >>> def build_arr(multiplier):\n",
" ... return multiplier * xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_arr, range(3))\n",
" >>> da = dataarray_from_delayed(\n",
" ... fut,\n",
" ... dim=pd.Index(range(3), name='simulation'))\n",
" ...\n",
" >>> da # doctest: +ELLIPSIS\n",
" <xarray.DataArray ...(simulation: 3, x: 2)>\n",
" dask.array<...shape=(3, 2), dtype=int64, chunksize=(1, 2), chunktype=numpy.ndarray>\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" * simulation (simulation) int64 0 1 2\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" data_arrays = dataarrays_from_delayed_old(futures, client=client)\n",
" da = xr.concat(data_arrays, dim=dim)\n",
"\n",
" return da\n",
"\n",
"\n",
"def datasets_from_delayed_old(futures, client=None):\n",
" \"\"\"\n",
" Returns a list of xarray datasets from a list of futures of datasets\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.Dataset` objects.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" datasets : list\n",
" list of :py:class:`xarray.Dataset` objects with\n",
" :py:class:`dask.array.Array` backends for each variable.\n",
" Examples\n",
" --------\n",
" Given a mapped :py:class:`xarray.Dataset`, pull the metadata into memory\n",
" while leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np\n",
" >>> def build_ds(multiplier):\n",
" ... return multiplier * xr.Dataset({\n",
" ... 'var1': xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])})\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_ds, range(3))\n",
" >>> arrs = datasets_from_delayed(fut)\n",
" >>> arrs[-1] # doctest: +ELLIPSIS\n",
" <xarray.Dataset>\n",
" Dimensions: (x: 2)\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" Data variables:\n",
" var1 (x) int64 dask.array<chunksize=(2,), meta=np.ndarray>\n",
" This list of arrays can now be manipulated using normal xarray tools:\n",
" .. code-block:: python\n",
" >>> xr.concat(arrs, dim='y') # doctest: +ELLIPSIS\n",
" <xarray.Dataset>\n",
" Dimensions: (x: 2, y: 3)\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" Dimensions without coordinates: y\n",
" Data variables:\n",
" var1 (y, x) int64 dask.array<chunksize=(1, 2), meta=np.ndarray>\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" if client is None:\n",
" client = dd.get_client()\n",
"\n",
" data_var_keys = client.gather(\n",
" client.map(lambda x: list(x.data_vars.keys()), futures)\n",
" )\n",
"\n",
" delayed_arrays = [\n",
" {k: (client.submit(lambda x: x[k].data, futures[i])) for k in data_var_keys[i]}\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" dask_array_metadata = [\n",
" {\n",
" k: (\n",
" client.submit(\n",
" lambda x: (x[k].data.shape, x[k].data.dtype), futures[i]\n",
" ).result()\n",
" )\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" dask_data_arrays = [\n",
" {\n",
" k: (\n",
" dask.array.from_delayed(\n",
" delayed_arrays[i][k], *dask_array_metadata[i][k]\n",
" )\n",
" )\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" # using dict(x.coords) b/c gathering coords can blow up memory for some reason\n",
" array_metadata = [\n",
" {\n",
" k: client.submit(\n",
" lambda x: {\n",
" \"dims\": x[k].dims,\n",
" \"coords\": x[k].coords,\n",
" \"attrs\": x[k].attrs,\n",
" },\n",
" futures[i],\n",
" ).result()\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" data_arrays = [\n",
" {\n",
" k: (xr.DataArray(dask_data_arrays[i][k], **array_metadata[i][k]))\n",
" for k in data_var_keys[i]\n",
" }\n",
" for i in range(len(futures))\n",
" ]\n",
"\n",
" datasets = [xr.Dataset(arr) for arr in data_arrays]\n",
"\n",
" dataset_metadata = client.gather(client.map(lambda x: x.attrs, futures))\n",
"\n",
" for i in range(len(futures)):\n",
" datasets[i].attrs.update(dataset_metadata[i])\n",
"\n",
" return datasets\n",
"\n",
"\n",
"def dataset_from_delayed_old(futures, dim=None, client=None):\n",
" \"\"\"\n",
" Returns an :py:class:`xarray.Dataset` from a list of futures\n",
" Parameters\n",
" ----------\n",
" futures : list\n",
" list of :py:class:`dask.delayed.Future` objects holding\n",
" :py:class:`xarray.Dataset` objects.\n",
" dim : str, optional\n",
" dimension along which to concat :py:class:`xarray.Dataset`.\n",
" Inferred by default.\n",
" client : object, optional\n",
" :py:class:`dask.distributed.Client` to use in gathering\n",
" metadata on futures. If not provided, client is inferred\n",
" from context.\n",
" Returns\n",
" -------\n",
" dataset : object\n",
" :py:class:`xarray.Dataset` concatenated along ``dim`` with\n",
" :py:class:`dask.array.Array` backends for each variable.\n",
" Examples\n",
" --------\n",
" Given a mapped :py:class:`xarray.Dataset`, pull the metadata into memory\n",
" while leaving the data on the workers:\n",
" .. code-block:: python\n",
" >>> import numpy as np, pandas as pd\n",
" >>> def build_ds(multiplier):\n",
" ... return multiplier * xr.Dataset({\n",
" ... 'var1': xr.DataArray(\n",
" ... np.arange(2), dims=['x'], coords=[['a', 'b']])})\n",
" ...\n",
" >>> client = dd.Client()\n",
" >>> fut = client.map(build_ds, range(3))\n",
" >>> ds = dataset_from_delayed(fut, dim=pd.Index(range(3), name='y'))\n",
" >>> ds\n",
" <xarray.Dataset>\n",
" Dimensions: (x: 2, y: 3)\n",
" Coordinates:\n",
" * x (x) <U1 'a' 'b'\n",
" * y (y) int64 0 1 2\n",
" Data variables:\n",
" var1 (y, x) int64 dask.array<chunksize=(1, 2), meta=np.ndarray>\n",
" >>> client.close()\n",
" \"\"\"\n",
"\n",
" datasets = datasets_from_delayed_old(futures, client=client)\n",
" ds = xr.concat(datasets, dim=dim)\n",
"\n",
" return ds\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Performance tests"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: memory_profiler in /opt/conda/lib/python3.7/site-packages (0.57.0)\n",
"Requirement already satisfied: psutil in /opt/conda/lib/python3.7/site-packages (from memory_profiler) (5.7.2)\n"
]
}
],
"source": [
"! pip install memory_profiler"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"%load_ext memory_profiler"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"import rhg_compute_tools.kubernetes as rhgk"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"client, cluster = rhgk.get_micro_cluster()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"cluster.scale(12)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8ec142eaa4a443f0abafc06dbecf2442",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HTML(value='<h2>KubeCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n <style scoped>\\n .…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"cluster"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"import xarray as xr\n",
"import pandas as pd\n",
"import numpy as np\n",
"import dask.distributed as dd"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def generate_partition(i, part_size, other_dims=[], other_coords=[]):\n",
" return xr.DataArray(\n",
" np.random.random([part_size] + list(map(len, other_coords))),\n",
" dims=(['ind'] + other_dims),\n",
" coords=([np.arange(i*part_size, (i+1) * part_size)] + other_coords),\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"futs = client.map(\n",
" generate_partition,\n",
" range(1000),\n",
" part_size=1000,\n",
" other_dims=['abc', 'xyz', '123'],\n",
" other_coords=[list('abcdef'), list('uvwxyz'), list(range(6))])"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "574a7db00dad4834aa9dd6e9743b949c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox()"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"dd.progress(futs)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"dd.wait(futs);"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"peak memory: 328.64 MiB, increment: 61.14 MiB\n"
]
}
],
"source": [
"%memit dda = dataarray_from_delayed(futs, dim='ind')"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1.6093254089355469"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dda.nbytes / 1024**3"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"peak memory: 3621.26 MiB, increment: 3293.57 MiB\n"
]
}
],
"source": [
"%memit dda_old = dataarray_from_delayed_old(futs, dim='ind')"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1.6093254089355469"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dda.nbytes / 1024**3"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Digging in"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Coordinates:\n",
" * ind (ind) int64 0 1 2 3 4 5 6 7 8 ... 992 993 994 995 996 997 998 999\n",
" * abc (abc) <U1 'a' 'b' 'c' 'd' 'e' 'f'\n",
" * xyz (xyz) <U1 'u' 'v' 'w' 'x' 'y' 'z'\n",
" * 123 (123) int64 0 1 2 3 4 5"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"futs[0].result().coords"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'ind': <xarray.DataArray 'ind' (ind: 1000)>\n",
" array([ 0, 1, 2, ..., 997, 998, 999])\n",
" Coordinates:\n",
" * ind (ind) int64 0 1 2 3 4 5 6 7 8 ... 992 993 994 995 996 997 998 999,\n",
" 'abc': <xarray.DataArray 'abc' (abc: 6)>\n",
" array(['a', 'b', 'c', 'd', 'e', 'f'], dtype='<U1')\n",
" Coordinates:\n",
" * abc (abc) <U1 'a' 'b' 'c' 'd' 'e' 'f',\n",
" 'xyz': <xarray.DataArray 'xyz' (xyz: 6)>\n",
" array(['u', 'v', 'w', 'x', 'y', 'z'], dtype='<U1')\n",
" Coordinates:\n",
" * xyz (xyz) <U1 'u' 'v' 'w' 'x' 'y' 'z',\n",
" '123': <xarray.DataArray '123' (123: 6)>\n",
" array([0, 1, 2, 3, 4, 5])\n",
" Coordinates:\n",
" * 123 (123) int64 0 1 2 3 4 5}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dict(futs[0].result().coords)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"c = futs[0].result().coords"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt, dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2 {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray (ind: 1000, abc: 6, xyz: 6, 123: 6)&gt;\n",
"array([[[[5.66951380e-01, 5.27003417e-01, 6.61142381e-02,\n",
" 6.65614259e-01, 7.31881990e-02, 8.15425697e-01],\n",
" [6.76711973e-01, 2.36968496e-01, 9.99796409e-01,\n",
" 6.69944196e-01, 6.45082955e-01, 5.01547439e-01],\n",
" [7.67192468e-01, 1.35409187e-01, 3.51349937e-01,\n",
" 3.08533230e-01, 3.34389030e-01, 6.98403585e-01],\n",
" [9.12699441e-01, 7.60327108e-02, 9.89405806e-01,\n",
" 9.79343966e-01, 9.32329158e-01, 8.37154900e-01],\n",
" [3.41115833e-01, 8.82332476e-01, 6.13685751e-02,\n",
" 4.66004621e-02, 9.85316584e-01, 1.10086052e-01],\n",
" [3.59774047e-01, 2.53553111e-01, 2.19055271e-01,\n",
" 4.31214326e-01, 7.58216491e-01, 6.44113266e-01]],\n",
"\n",
" [[1.36644134e-01, 4.39293635e-01, 2.06717763e-01,\n",
" 1.12492524e-01, 2.37967405e-01, 5.59797391e-01],\n",
" [1.85496520e-02, 5.88813427e-04, 7.60810328e-01,\n",
" 1.37569271e-02, 4.15910064e-01, 4.91066102e-01],\n",
" [1.73083865e-01, 9.44496993e-01, 2.67232521e-01,\n",
" 8.45402827e-01, 6.45676812e-01, 6.80708723e-01],\n",
" [6.51794860e-01, 6.14889448e-01, 3.75751204e-01,\n",
"...\n",
" 4.82580522e-01, 1.69045409e-01, 8.58404564e-01],\n",
" [5.70272270e-01, 6.00159413e-01, 6.59846309e-01,\n",
" 6.97958341e-01, 4.55761553e-01, 8.32505322e-01],\n",
" [2.06968921e-01, 6.17068159e-01, 3.21187655e-02,\n",
" 3.56322287e-01, 2.61541176e-01, 4.57011064e-01],\n",
" [6.89140320e-01, 7.59641927e-01, 9.20890836e-02,\n",
" 3.93229902e-01, 2.10811603e-01, 2.68603347e-01]],\n",
"\n",
" [[8.76213980e-01, 2.81373384e-01, 2.22991517e-01,\n",
" 4.89476054e-01, 2.42277819e-01, 6.11421196e-01],\n",
" [5.94232499e-01, 6.72968261e-01, 7.87422830e-01,\n",
" 1.40168175e-01, 2.78141244e-01, 5.24176212e-01],\n",
" [2.87720462e-01, 7.61707023e-01, 8.60952426e-01,\n",
" 1.61954335e-01, 3.03827179e-01, 1.70220843e-01],\n",
" [6.77073169e-01, 3.33404106e-01, 8.03078164e-01,\n",
" 8.33500034e-01, 7.05865806e-01, 8.67319954e-01],\n",
" [6.84990061e-01, 7.13483606e-01, 5.57883212e-01,\n",
" 7.56246247e-02, 5.53165089e-01, 6.61000627e-01],\n",
" [8.46406254e-01, 3.43584378e-01, 3.11580361e-01,\n",
" 3.31421130e-01, 5.41708232e-01, 6.77078810e-01]]]])\n",
"Coordinates:\n",
" * ind (ind) int64 0 1 2 3 4 5 6 7 8 ... 992 993 994 995 996 997 998 999\n",
" * abc (abc) &lt;U1 &#x27;a&#x27; &#x27;b&#x27; &#x27;c&#x27; &#x27;d&#x27; &#x27;e&#x27; &#x27;f&#x27;\n",
" * xyz (xyz) &lt;U1 &#x27;u&#x27; &#x27;v&#x27; &#x27;w&#x27; &#x27;x&#x27; &#x27;y&#x27; &#x27;z&#x27;\n",
" * 123 (123) int64 0 1 2 3 4 5</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'></div><ul class='xr-dim-list'><li><span class='xr-has-index'>ind</span>: 1000</li><li><span class='xr-has-index'>abc</span>: 6</li><li><span class='xr-has-index'>xyz</span>: 6</li><li><span class='xr-has-index'>123</span>: 6</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-9598d066-8be4-4911-ae3e-a33af3ace8d4' class='xr-array-in' type='checkbox' checked><label for='section-9598d066-8be4-4911-ae3e-a33af3ace8d4' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>0.567 0.527 0.06611 0.6656 0.07319 ... 0.3116 0.3314 0.5417 0.6771</span></div><div class='xr-array-data'><pre>array([[[[5.66951380e-01, 5.27003417e-01, 6.61142381e-02,\n",
" 6.65614259e-01, 7.31881990e-02, 8.15425697e-01],\n",
" [6.76711973e-01, 2.36968496e-01, 9.99796409e-01,\n",
" 6.69944196e-01, 6.45082955e-01, 5.01547439e-01],\n",
" [7.67192468e-01, 1.35409187e-01, 3.51349937e-01,\n",
" 3.08533230e-01, 3.34389030e-01, 6.98403585e-01],\n",
" [9.12699441e-01, 7.60327108e-02, 9.89405806e-01,\n",
" 9.79343966e-01, 9.32329158e-01, 8.37154900e-01],\n",
" [3.41115833e-01, 8.82332476e-01, 6.13685751e-02,\n",
" 4.66004621e-02, 9.85316584e-01, 1.10086052e-01],\n",
" [3.59774047e-01, 2.53553111e-01, 2.19055271e-01,\n",
" 4.31214326e-01, 7.58216491e-01, 6.44113266e-01]],\n",
"\n",
" [[1.36644134e-01, 4.39293635e-01, 2.06717763e-01,\n",
" 1.12492524e-01, 2.37967405e-01, 5.59797391e-01],\n",
" [1.85496520e-02, 5.88813427e-04, 7.60810328e-01,\n",
" 1.37569271e-02, 4.15910064e-01, 4.91066102e-01],\n",
" [1.73083865e-01, 9.44496993e-01, 2.67232521e-01,\n",
" 8.45402827e-01, 6.45676812e-01, 6.80708723e-01],\n",
" [6.51794860e-01, 6.14889448e-01, 3.75751204e-01,\n",
"...\n",
" 4.82580522e-01, 1.69045409e-01, 8.58404564e-01],\n",
" [5.70272270e-01, 6.00159413e-01, 6.59846309e-01,\n",
" 6.97958341e-01, 4.55761553e-01, 8.32505322e-01],\n",
" [2.06968921e-01, 6.17068159e-01, 3.21187655e-02,\n",
" 3.56322287e-01, 2.61541176e-01, 4.57011064e-01],\n",
" [6.89140320e-01, 7.59641927e-01, 9.20890836e-02,\n",
" 3.93229902e-01, 2.10811603e-01, 2.68603347e-01]],\n",
"\n",
" [[8.76213980e-01, 2.81373384e-01, 2.22991517e-01,\n",
" 4.89476054e-01, 2.42277819e-01, 6.11421196e-01],\n",
" [5.94232499e-01, 6.72968261e-01, 7.87422830e-01,\n",
" 1.40168175e-01, 2.78141244e-01, 5.24176212e-01],\n",
" [2.87720462e-01, 7.61707023e-01, 8.60952426e-01,\n",
" 1.61954335e-01, 3.03827179e-01, 1.70220843e-01],\n",
" [6.77073169e-01, 3.33404106e-01, 8.03078164e-01,\n",
" 8.33500034e-01, 7.05865806e-01, 8.67319954e-01],\n",
" [6.84990061e-01, 7.13483606e-01, 5.57883212e-01,\n",
" 7.56246247e-02, 5.53165089e-01, 6.61000627e-01],\n",
" [8.46406254e-01, 3.43584378e-01, 3.11580361e-01,\n",
" 3.31421130e-01, 5.41708232e-01, 6.77078810e-01]]]])</pre></div></div></li><li class='xr-section-item'><input id='section-29c89b46-5594-4b88-9db8-f460e2f896da' class='xr-section-summary-in' type='checkbox' checked><label for='section-29c89b46-5594-4b88-9db8-f460e2f896da' class='xr-section-summary' >Coordinates: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>ind</span></div><div class='xr-var-dims'>(ind)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 5 ... 995 996 997 998 999</div><input id='attrs-c1c7be0a-bbc4-4176-8fa0-c68b0fe83d67' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-c1c7be0a-bbc4-4176-8fa0-c68b0fe83d67' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-919084a0-b91d-4be0-9a8a-7ced213f3006' class='xr-var-data-in' type='checkbox'><label for='data-919084a0-b91d-4be0-9a8a-7ced213f3006' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 997, 998, 999])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>abc</span></div><div class='xr-var-dims'>(abc)</div><div class='xr-var-dtype'>&lt;U1</div><div class='xr-var-preview xr-preview'>&#x27;a&#x27; &#x27;b&#x27; &#x27;c&#x27; &#x27;d&#x27; &#x27;e&#x27; &#x27;f&#x27;</div><input id='attrs-165ce89e-5907-48ea-8ffc-091ae0f0176e' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-165ce89e-5907-48ea-8ffc-091ae0f0176e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-26b82947-6ecf-457a-86aa-8dfc4e56dbe7' class='xr-var-data-in' type='checkbox'><label for='data-26b82947-6ecf-457a-86aa-8dfc4e56dbe7' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([&#x27;a&#x27;, &#x27;b&#x27;, &#x27;c&#x27;, &#x27;d&#x27;, &#x27;e&#x27;, &#x27;f&#x27;], dtype=&#x27;&lt;U1&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>xyz</span></div><div class='xr-var-dims'>(xyz)</div><div class='xr-var-dtype'>&lt;U1</div><div class='xr-var-preview xr-preview'>&#x27;u&#x27; &#x27;v&#x27; &#x27;w&#x27; &#x27;x&#x27; &#x27;y&#x27; &#x27;z&#x27;</div><input id='attrs-64b839ec-7b42-4f83-aa7d-6c072116e846' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-64b839ec-7b42-4f83-aa7d-6c072116e846' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-662bb785-be4a-4c1a-b002-65fea6a8b39a' class='xr-var-data-in' type='checkbox'><label for='data-662bb785-be4a-4c1a-b002-65fea6a8b39a' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([&#x27;u&#x27;, &#x27;v&#x27;, &#x27;w&#x27;, &#x27;x&#x27;, &#x27;y&#x27;, &#x27;z&#x27;], dtype=&#x27;&lt;U1&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>123</span></div><div class='xr-var-dims'>(123)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 5</div><input id='attrs-8eaf1d08-b2d6-47c0-a2ec-6924c12dc8fe' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-8eaf1d08-b2d6-47c0-a2ec-6924c12dc8fe' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-dc0bb642-f737-4327-959f-76ce7c68d7fa' class='xr-var-data-in' type='checkbox'><label for='data-dc0bb642-f737-4327-959f-76ce7c68d7fa' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([0, 1, 2, 3, 4, 5])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-9ccde28e-2649-4834-8e68-5b5f17a94b8f' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-9ccde28e-2649-4834-8e68-5b5f17a94b8f' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray (ind: 1000, abc: 6, xyz: 6, 123: 6)>\n",
"array([[[[5.66951380e-01, 5.27003417e-01, 6.61142381e-02,\n",
" 6.65614259e-01, 7.31881990e-02, 8.15425697e-01],\n",
" [6.76711973e-01, 2.36968496e-01, 9.99796409e-01,\n",
" 6.69944196e-01, 6.45082955e-01, 5.01547439e-01],\n",
" [7.67192468e-01, 1.35409187e-01, 3.51349937e-01,\n",
" 3.08533230e-01, 3.34389030e-01, 6.98403585e-01],\n",
" [9.12699441e-01, 7.60327108e-02, 9.89405806e-01,\n",
" 9.79343966e-01, 9.32329158e-01, 8.37154900e-01],\n",
" [3.41115833e-01, 8.82332476e-01, 6.13685751e-02,\n",
" 4.66004621e-02, 9.85316584e-01, 1.10086052e-01],\n",
" [3.59774047e-01, 2.53553111e-01, 2.19055271e-01,\n",
" 4.31214326e-01, 7.58216491e-01, 6.44113266e-01]],\n",
"\n",
" [[1.36644134e-01, 4.39293635e-01, 2.06717763e-01,\n",
" 1.12492524e-01, 2.37967405e-01, 5.59797391e-01],\n",
" [1.85496520e-02, 5.88813427e-04, 7.60810328e-01,\n",
" 1.37569271e-02, 4.15910064e-01, 4.91066102e-01],\n",
" [1.73083865e-01, 9.44496993e-01, 2.67232521e-01,\n",
" 8.45402827e-01, 6.45676812e-01, 6.80708723e-01],\n",
" [6.51794860e-01, 6.14889448e-01, 3.75751204e-01,\n",
"...\n",
" 4.82580522e-01, 1.69045409e-01, 8.58404564e-01],\n",
" [5.70272270e-01, 6.00159413e-01, 6.59846309e-01,\n",
" 6.97958341e-01, 4.55761553e-01, 8.32505322e-01],\n",
" [2.06968921e-01, 6.17068159e-01, 3.21187655e-02,\n",
" 3.56322287e-01, 2.61541176e-01, 4.57011064e-01],\n",
" [6.89140320e-01, 7.59641927e-01, 9.20890836e-02,\n",
" 3.93229902e-01, 2.10811603e-01, 2.68603347e-01]],\n",
"\n",
" [[8.76213980e-01, 2.81373384e-01, 2.22991517e-01,\n",
" 4.89476054e-01, 2.42277819e-01, 6.11421196e-01],\n",
" [5.94232499e-01, 6.72968261e-01, 7.87422830e-01,\n",
" 1.40168175e-01, 2.78141244e-01, 5.24176212e-01],\n",
" [2.87720462e-01, 7.61707023e-01, 8.60952426e-01,\n",
" 1.61954335e-01, 3.03827179e-01, 1.70220843e-01],\n",
" [6.77073169e-01, 3.33404106e-01, 8.03078164e-01,\n",
" 8.33500034e-01, 7.05865806e-01, 8.67319954e-01],\n",
" [6.84990061e-01, 7.13483606e-01, 5.57883212e-01,\n",
" 7.56246247e-02, 5.53165089e-01, 6.61000627e-01],\n",
" [8.46406254e-01, 3.43584378e-01, 3.11580361e-01,\n",
" 3.31421130e-01, 5.41708232e-01, 6.77078810e-01]]]])\n",
"Coordinates:\n",
" * ind (ind) int64 0 1 2 3 4 5 6 7 8 ... 992 993 994 995 996 997 998 999\n",
" * abc (abc) <U1 'a' 'b' 'c' 'd' 'e' 'f'\n",
" * xyz (xyz) <U1 'u' 'v' 'w' 'x' 'y' 'z'\n",
" * 123 (123) int64 0 1 2 3 4 5"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"c._data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This seems like it could be the culprit!!!"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment