Skip to content

Instantly share code, notes, and snippets.

@GenevieveBuckley

GenevieveBuckley/map-overlap-length-without-materializing.ipynb

Created June 28, 2021 13:27
Show Gist options
  • Save GenevieveBuckley/12ae179940e78c10d616dc7fbc93ed8c to your computer and use it in GitHub Desktop.
Save GenevieveBuckley/12ae179940e78c10d616dc7fbc93ed8c to your computer and use it in GitHub Desktop.
Download ZIP
map-overlap-length-without-materializing-the-task-graph
Display the source blob
Display the rendered blob
Raw
map-overlap-length-without-materializing.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "841bf3e4",
"metadata": {},
"source": [
"# map_overlap `len` without materializing the task graph"
]
},
{
"cell_type": "markdown",
"id": "9b86a87b",
"metadata": {},
"source": [
"## Imports"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "eab1af69",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'2021.06.1'"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import dask\n",
"dask.__version__"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "be9af70d",
"metadata": {},
"outputs": [],
"source": [
"from pprint import pprint\n",
"import numpy as np\n",
"import dask.array as da\n",
"from dask.threaded import get"
]
},
{
"cell_type": "markdown",
"id": "b0d4ed68",
"metadata": {},
"source": [
"## Processing function"
]
},
{
"cell_type": "code",
"execution_count": 67,
"id": "fddbb6c4",
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"from collections import Counter, OrderedDict\n",
"from pprint import pprint\n",
"import dask.array as da\n",
"\n",
"def myfunc(bb):\n",
" tasknames = [keyname for keyname, *_part in bb.dask]\n",
" counts = Counter(tasknames)\n",
" # Topological sort of output counts\n",
" layernames = bb.dask._toposort_layers()\n",
" result = OrderedDict()\n",
" ntasks = 0\n",
" for layerkey in layernames:\n",
" for key, val in counts.items():\n",
" if key == layerkey:\n",
" result[key] = val\n",
" ntasks += val\n",
" # print(\"ntasks:\", ntasks)\n",
" # pprint(result)\n",
" \n",
" full_dict = dict(bb.dask)\n",
" all_keys = []\n",
" for name in result:\n",
" for key in full_dict:\n",
" keyname, *_parts = key\n",
" if name == keyname:\n",
" # print(key)\n",
" all_keys.append(key)\n",
" \n",
" dsk = bb.dask\n",
" for key in all_keys:\n",
" out = get(dsk, key)\n",
" print(key, out)"
]
},
{
"cell_type": "markdown",
"id": "7effb0e6",
"metadata": {},
"source": [
"## 1D examples"
]
},
{
"cell_type": "code",
"execution_count": 72,
"id": "66b18385",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('array-de932becc43e72c010bc91ffefe42af1', 0) [1 2]\n",
"('full-81227754a53dbffb4d3c9111b7c25c50', 0) [0]\n",
"('concatenate-ed002b4f6f4ebd1bc56e99b34f5af0cb', 0) [0]\n",
"('concatenate-ed002b4f6f4ebd1bc56e99b34f5af0cb', 1) [1 2]\n",
"('concatenate-ed002b4f6f4ebd1bc56e99b34f5af0cb', 2) [0]\n",
"('overlap-d4cd0ab05a7c640609b1e766b7f3c107', 0) [0 1]\n",
"('overlap-d4cd0ab05a7c640609b1e766b7f3c107', 1) [0 1 2 0]\n",
"('overlap-d4cd0ab05a7c640609b1e766b7f3c107', 2) [2 0]\n",
"('getitem-914a48ec13f954f26224e63cb50186bf', 0) [0 1 2 0]\n",
"('lambda-0d91816a9763f22ff71823cf629d75cb', 0) [0 1 2 0]\n",
"('block-info-_trim-4350a722500aa00c76c31040b2cdaa9c', 0) {0: {'shape': (4,), 'num-chunks': (1,), 'array-location': [(0, 4)], 'chunk-location': (0,)}, None: {'shape': (2,), 'num-chunks': (1,), 'array-location': [(0, 2)], 'chunk-location': (0,), 'chunk-shape': (2,), 'dtype': dtype('int64')}}\n",
"('_trim-4350a722500aa00c76c31040b2cdaa9c', 0) [1 2]\n"
]
}
],
"source": [
"aa = da.from_array([1, 2]) # 1D, single chunk\n",
"bb = aa.map_overlap(lambda x: x, depth=1, boundary=0)\n",
"myfunc(bb)"
]
},
{
"cell_type": "code",
"execution_count": 73,
"id": "950d3d03",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('array-88f8aab41a55470b48065636a28c211a', 0) [1]\n",
"('array-88f8aab41a55470b48065636a28c211a', 1) [2]\n",
"('full-81227754a53dbffb4d3c9111b7c25c50', 0) [0]\n",
"('concatenate-e1f2ad4ac0e5e9b5405412e20ab2a259', 0) [0]\n",
"('concatenate-e1f2ad4ac0e5e9b5405412e20ab2a259', 1) [1]\n",
"('concatenate-e1f2ad4ac0e5e9b5405412e20ab2a259', 2) [2]\n",
"('concatenate-e1f2ad4ac0e5e9b5405412e20ab2a259', 3) [0]\n",
"('overlap-c880896cf2369104f9e88135124c95aa', 0) [0 1]\n",
"('overlap-c880896cf2369104f9e88135124c95aa', 1) [0 1 2]\n",
"('overlap-c880896cf2369104f9e88135124c95aa', 2) [1 2 0]\n",
"('overlap-c880896cf2369104f9e88135124c95aa', 3) [2 0]\n",
"('getitem-a82b4c3f22f761be51068ca43d10c763', 0) [0 1 2]\n",
"('getitem-a82b4c3f22f761be51068ca43d10c763', 1) [1 2 0]\n",
"('lambda-846738265249ae4375dda5f70bd3e183', 0) [0 1 2]\n",
"('lambda-846738265249ae4375dda5f70bd3e183', 1) [1 2 0]\n",
"('block-info-_trim-ac5cf63284633ea0663b86e175bd2807', 0) {0: {'shape': (6,), 'num-chunks': (2,), 'array-location': [(0, 3)], 'chunk-location': (0,)}, None: {'shape': (2,), 'num-chunks': (2,), 'array-location': [(0, 1)], 'chunk-location': (0,), 'chunk-shape': (1,), 'dtype': dtype('int64')}}\n",
"('block-info-_trim-ac5cf63284633ea0663b86e175bd2807', 1) {0: {'shape': (6,), 'num-chunks': (2,), 'array-location': [(3, 6)], 'chunk-location': (1,)}, None: {'shape': (2,), 'num-chunks': (2,), 'array-location': [(1, 2)], 'chunk-location': (1,), 'chunk-shape': (1,), 'dtype': dtype('int64')}}\n",
"('_trim-ac5cf63284633ea0663b86e175bd2807', 0) [1]\n",
"('_trim-ac5cf63284633ea0663b86e175bd2807', 1) [2]\n"
]
}
],
"source": [
"aa = da.from_array([1, 2], chunks=1) # 1D, 1 pixel chunks\n",
"bb = aa.map_overlap(lambda x: x, depth=1, boundary=0)\n",
"myfunc(bb)"
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "a8e923e2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('array-0f7be6dccc1065cdd29a05c38108d02c', 0) [1 2 3]\n",
"('array-0f7be6dccc1065cdd29a05c38108d02c', 1) [4 5 6]\n",
"('full-81227754a53dbffb4d3c9111b7c25c50', 0) [0]\n",
"('concatenate-ef5efc1a2f7b10afc0e37f542ff594c1', 0) [0]\n",
"('concatenate-ef5efc1a2f7b10afc0e37f542ff594c1', 1) [1 2 3]\n",
"('concatenate-ef5efc1a2f7b10afc0e37f542ff594c1', 2) [4 5 6]\n",
"('concatenate-ef5efc1a2f7b10afc0e37f542ff594c1', 3) [0]\n",
"('overlap-3dc7ea0224f3c629323438b83fa9a4ec', 0) [0 1]\n",
"('overlap-3dc7ea0224f3c629323438b83fa9a4ec', 1) [0 1 2 3 4]\n",
"('overlap-3dc7ea0224f3c629323438b83fa9a4ec', 2) [3 4 5 6 0]\n",
"('overlap-3dc7ea0224f3c629323438b83fa9a4ec', 3) [6 0]\n",
"('getitem-a63c4504c4f00e33befb9bf620d6f0ac', 0) [0 1 2 3 4]\n",
"('getitem-a63c4504c4f00e33befb9bf620d6f0ac', 1) [3 4 5 6 0]\n",
"('lambda-83568ac250100f58007250477b69d546', 0) [0 1 2 3 4]\n",
"('lambda-83568ac250100f58007250477b69d546', 1) [3 4 5 6 0]\n",
"('block-info-_trim-51ad988d01edd4a26494f228f94ed025', 0) {0: {'shape': (10,), 'num-chunks': (2,), 'array-location': [(0, 5)], 'chunk-location': (0,)}, None: {'shape': (6,), 'num-chunks': (2,), 'array-location': [(0, 3)], 'chunk-location': (0,), 'chunk-shape': (3,), 'dtype': dtype('int64')}}\n",
"('block-info-_trim-51ad988d01edd4a26494f228f94ed025', 1) {0: {'shape': (10,), 'num-chunks': (2,), 'array-location': [(5, 10)], 'chunk-location': (1,)}, None: {'shape': (6,), 'num-chunks': (2,), 'array-location': [(3, 6)], 'chunk-location': (1,), 'chunk-shape': (3,), 'dtype': dtype('int64')}}\n",
"('_trim-51ad988d01edd4a26494f228f94ed025', 0) [1 2 3]\n",
"('_trim-51ad988d01edd4a26494f228f94ed025', 1) [4 5 6]\n"
]
}
],
"source": [
"aa = da.from_array([1, 2, 3, 4, 5, 6], chunks=3)\n",
"bb = aa.map_overlap(lambda x: x, depth=1, boundary=0)\n",
"myfunc(bb)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "db79870c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "97e3fadf",
"metadata": {},
"source": [
"## 2D examples"
]
},
{
"cell_type": "code",
"execution_count": 74,
"id": "8193c056",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('array-fca6b42e913425bc80444cf3ce8e9301', 0, 0) [[1 2]\n",
" [3 4]]\n",
"('full-3ed42237ec90c020adb799545ae8c701', 0, 0) [[0 0]]\n",
"('concatenate-7b227199262d6b9b258e486f8eba79d7', 0, 0) [[0 0]]\n",
"('concatenate-7b227199262d6b9b258e486f8eba79d7', 1, 0) [[1 2]\n",
" [3 4]]\n",
"('concatenate-7b227199262d6b9b258e486f8eba79d7', 2, 0) [[0 0]]\n",
"('full-7a3dd7bceb509ed34f7a283342a23e7d', 0, 0) [[0]]\n",
"('full-7a3dd7bceb509ed34f7a283342a23e7d', 1, 0) [[0]\n",
" [0]]\n",
"('full-7a3dd7bceb509ed34f7a283342a23e7d', 2, 0) [[0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 0, 0) [[0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 0, 1) [[0 0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 0, 2) [[0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 1, 0) [[0]\n",
" [0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 1, 1) [[1 2]\n",
" [3 4]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 1, 2) [[0]\n",
" [0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 2, 0) [[0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 2, 1) [[0 0]]\n",
"('concatenate-fa9d876fe966a59772a49fbddb458857', 2, 2) [[0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 0, 0) [[0 0]\n",
" [0 1]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 0, 1) [[0 0 0 0]\n",
" [0 1 2 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 0, 2) [[0 0]\n",
" [2 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 1, 0) [[0 0]\n",
" [0 1]\n",
" [0 3]\n",
" [0 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 1, 1) [[0 0 0 0]\n",
" [0 1 2 0]\n",
" [0 3 4 0]\n",
" [0 0 0 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 1, 2) [[0 0]\n",
" [2 0]\n",
" [4 0]\n",
" [0 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 2, 0) [[0 3]\n",
" [0 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 2, 1) [[0 3 4 0]\n",
" [0 0 0 0]]\n",
"('overlap-ef2ae5a5426d7c0611932dccbdbc0905', 2, 2) [[4 0]\n",
" [0 0]]\n",
"('getitem-cbbd1695fcf414142f2506deb14f5a8a', 0, 0) [[0 0 0 0]\n",
" [0 1 2 0]\n",
" [0 3 4 0]\n",
" [0 0 0 0]]\n",
"('lambda-06b6c0a2c4e759f2b823afbdc9a7dc8c', 0, 0) [[0 0 0 0]\n",
" [0 1 2 0]\n",
" [0 3 4 0]\n",
" [0 0 0 0]]\n",
"('block-info-_trim-7c5ce23856e737e3eb2c749598b7f1bb', 0, 0) {0: {'shape': (4, 4), 'num-chunks': (1, 1), 'array-location': [(0, 4), (0, 4)], 'chunk-location': (0, 0)}, None: {'shape': (2, 2), 'num-chunks': (1, 1), 'array-location': [(0, 2), (0, 2)], 'chunk-location': (0, 0), 'chunk-shape': (2, 2), 'dtype': dtype('int64')}}\n",
"('_trim-7c5ce23856e737e3eb2c749598b7f1bb', 0, 0) [[1 2]\n",
" [3 4]]\n"
]
}
],
"source": [
"aa = da.from_array([[1, 2],[3, 4]]) # 2D, single chunk\n",
"bb = aa.map_overlap(lambda x: x, depth=1, boundary=0)\n",
"myfunc(bb)"
]
},
{
"cell_type": "code",
"execution_count": 75,
"id": "201690c6",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('array-5cfcb28ffab26b45ef53c89982db3aa2', 0, 0) [[1]]\n",
"('array-5cfcb28ffab26b45ef53c89982db3aa2', 0, 1) [[2]]\n",
"('array-5cfcb28ffab26b45ef53c89982db3aa2', 1, 0) [[4]]\n",
"('array-5cfcb28ffab26b45ef53c89982db3aa2', 1, 1) [[5]]\n",
"('full-19315621ac0da38333b206d056640538', 0, 0) [[0]]\n",
"('full-19315621ac0da38333b206d056640538', 0, 1) [[0]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 0, 0) [[0]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 0, 1) [[0]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 1, 0) [[1]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 1, 1) [[2]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 2, 0) [[4]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 2, 1) [[5]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 3, 0) [[0]]\n",
"('concatenate-2740eeb10362af688d1188a82a65e98d', 3, 1) [[0]]\n",
"('full-3832c89d06add3bbf3be8a8c626b8c7d', 0, 0) [[0]]\n",
"('full-3832c89d06add3bbf3be8a8c626b8c7d', 1, 0) [[0]]\n",
"('full-3832c89d06add3bbf3be8a8c626b8c7d', 2, 0) [[0]]\n",
"('full-3832c89d06add3bbf3be8a8c626b8c7d', 3, 0) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 0, 0) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 0, 1) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 0, 2) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 0, 3) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 1, 0) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 1, 1) [[1]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 1, 2) [[2]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 1, 3) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 2, 0) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 2, 1) [[4]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 2, 2) [[5]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 2, 3) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 3, 0) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 3, 1) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 3, 2) [[0]]\n",
"('concatenate-5e24e94b5e5911f8a0354c5b0be89019', 3, 3) [[0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 0, 0) [[0 0]\n",
" [0 1]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 0, 1) [[0 0 0]\n",
" [0 1 2]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 0, 2) [[0 0 0]\n",
" [1 2 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 0, 3) [[0 0]\n",
" [2 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 1, 0) [[0 0]\n",
" [0 1]\n",
" [0 4]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 1, 1) [[0 0 0]\n",
" [0 1 2]\n",
" [0 4 5]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 1, 2) [[0 0 0]\n",
" [1 2 0]\n",
" [4 5 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 1, 3) [[0 0]\n",
" [2 0]\n",
" [5 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 2, 0) [[0 1]\n",
" [0 4]\n",
" [0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 2, 1) [[0 1 2]\n",
" [0 4 5]\n",
" [0 0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 2, 2) [[1 2 0]\n",
" [4 5 0]\n",
" [0 0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 2, 3) [[2 0]\n",
" [5 0]\n",
" [0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 3, 0) [[0 4]\n",
" [0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 3, 1) [[0 4 5]\n",
" [0 0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 3, 2) [[4 5 0]\n",
" [0 0 0]]\n",
"('overlap-14d577c20d2c7ea168a01f5ff3ff4b9f', 3, 3) [[5 0]\n",
" [0 0]]\n",
"('getitem-4755aaf030ebf71d24ecc1841e590b0d', 0, 0) [[0 0 0]\n",
" [0 1 2]\n",
" [0 4 5]]\n",
"('getitem-4755aaf030ebf71d24ecc1841e590b0d', 0, 1) [[0 0 0]\n",
" [1 2 0]\n",
" [4 5 0]]\n",
"('getitem-4755aaf030ebf71d24ecc1841e590b0d', 1, 0) [[0 1 2]\n",
" [0 4 5]\n",
" [0 0 0]]\n",
"('getitem-4755aaf030ebf71d24ecc1841e590b0d', 1, 1) [[1 2 0]\n",
" [4 5 0]\n",
" [0 0 0]]\n",
"('lambda-9f36fdc9192a9af4469c33bd3d212fc7', 0, 0) [[0 0 0]\n",
" [0 1 2]\n",
" [0 4 5]]\n",
"('lambda-9f36fdc9192a9af4469c33bd3d212fc7', 0, 1) [[0 0 0]\n",
" [1 2 0]\n",
" [4 5 0]]\n",
"('lambda-9f36fdc9192a9af4469c33bd3d212fc7', 1, 0) [[0 1 2]\n",
" [0 4 5]\n",
" [0 0 0]]\n",
"('lambda-9f36fdc9192a9af4469c33bd3d212fc7', 1, 1) [[1 2 0]\n",
" [4 5 0]\n",
" [0 0 0]]\n",
"('block-info-_trim-2b75dea93416963c04120aebbd7536a3', 0, 0) {0: {'shape': (6, 6), 'num-chunks': (2, 2), 'array-location': [(0, 3), (0, 3)], 'chunk-location': (0, 0)}, None: {'shape': (2, 2), 'num-chunks': (2, 2), 'array-location': [(0, 1), (0, 1)], 'chunk-location': (0, 0), 'chunk-shape': (1, 1), 'dtype': dtype('int64')}}\n",
"('block-info-_trim-2b75dea93416963c04120aebbd7536a3', 0, 1) {0: {'shape': (6, 6), 'num-chunks': (2, 2), 'array-location': [(0, 3), (3, 6)], 'chunk-location': (0, 1)}, None: {'shape': (2, 2), 'num-chunks': (2, 2), 'array-location': [(0, 1), (1, 2)], 'chunk-location': (0, 1), 'chunk-shape': (1, 1), 'dtype': dtype('int64')}}\n",
"('block-info-_trim-2b75dea93416963c04120aebbd7536a3', 1, 0) {0: {'shape': (6, 6), 'num-chunks': (2, 2), 'array-location': [(3, 6), (0, 3)], 'chunk-location': (1, 0)}, None: {'shape': (2, 2), 'num-chunks': (2, 2), 'array-location': [(1, 2), (0, 1)], 'chunk-location': (1, 0), 'chunk-shape': (1, 1), 'dtype': dtype('int64')}}\n",
"('block-info-_trim-2b75dea93416963c04120aebbd7536a3', 1, 1) {0: {'shape': (6, 6), 'num-chunks': (2, 2), 'array-location': [(3, 6), (3, 6)], 'chunk-location': (1, 1)}, None: {'shape': (2, 2), 'num-chunks': (2, 2), 'array-location': [(1, 2), (1, 2)], 'chunk-location': (1, 1), 'chunk-shape': (1, 1), 'dtype': dtype('int64')}}\n",
"('_trim-2b75dea93416963c04120aebbd7536a3', 0, 0) [[1]]\n",
"('_trim-2b75dea93416963c04120aebbd7536a3', 0, 1) [[2]]\n",
"('_trim-2b75dea93416963c04120aebbd7536a3', 1, 0) [[4]]\n",
"('_trim-2b75dea93416963c04120aebbd7536a3', 1, 1) [[5]]\n"
]
}
],
"source": [
"aa = da.from_array([[1, 2], [4, 5]], chunks=1) # 2D, 1 pixel chunks\n",
"bb = aa.map_overlap(lambda x: x, depth=1, boundary=0)\n",
"myfunc(bb)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e2d06ee",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 30,
"id": "c700ffda",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0])"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"get(bb.dask, ('_trim-34ee733cf60b11a59ee93fda12ad670c', 0))"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "e4ecc050",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([1])"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"get(bb.dask, ('_trim-34ee733cf60b11a59ee93fda12ad670c', 1))"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "d98d07d6",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{('_trim-34ee733cf60b11a59ee93fda12ad670c', 0): (subgraph_callable-61730f12-ea3e-4588-8c7b-16d04549bba5,\n",
" functools.partial(<function _trim at 0x7f959881e550>, axes={0: 1}, boundary={0: 'reflect'}),\n",
" (<class 'tuple'>,\n",
" ['block_info']),\n",
" ('block-info-_trim-34ee733cf60b11a59ee93fda12ad670c',\n",
" 0),\n",
" ('lambda-f9fc0ccea52495eb82f414cd021b25d1',\n",
" 0)),\n",
" ('_trim-34ee733cf60b11a59ee93fda12ad670c', 1): (subgraph_callable-61730f12-ea3e-4588-8c7b-16d04549bba5,\n",
" functools.partial(<function _trim at 0x7f959881e550>, axes={0: 1}, boundary={0: 'reflect'}),\n",
" (<class 'tuple'>,\n",
" ['block_info']),\n",
" ('block-info-_trim-34ee733cf60b11a59ee93fda12ad670c',\n",
" 1),\n",
" ('lambda-f9fc0ccea52495eb82f414cd021b25d1',\n",
" 1)),\n",
" ('arange-690b95eab87b800825077f22e660d393', 0): (functools.partial(<function arange at 0x7f9598844700>, like=None),\n",
" 0,\n",
" 1,\n",
" 1,\n",
" 1,\n",
" dtype('int64')),\n",
" ('arange-690b95eab87b800825077f22e660d393', 1): (functools.partial(<function arange at 0x7f9598844700>, like=None),\n",
" 1,\n",
" 2,\n",
" 1,\n",
" 1,\n",
" dtype('int64')),\n",
" ('block-info-_trim-34ee733cf60b11a59ee93fda12ad670c', 0): {None: {'array-location': [(0,\n",
" 1)],\n",
" 'chunk-location': (0,),\n",
" 'chunk-shape': (1,),\n",
" 'dtype': dtype('int64'),\n",
" 'num-chunks': (2,),\n",
" 'shape': (2,)},\n",
" 0: {'array-location': [(0,\n",
" 3)],\n",
" 'chunk-location': (0,),\n",
" 'num-chunks': (2,),\n",
" 'shape': (6,)}},\n",
" ('block-info-_trim-34ee733cf60b11a59ee93fda12ad670c', 1): {None: {'array-location': [(1,\n",
" 2)],\n",
" 'chunk-location': (1,),\n",
" 'chunk-shape': (1,),\n",
" 'dtype': dtype('int64'),\n",
" 'num-chunks': (2,),\n",
" 'shape': (2,)},\n",
" 0: {'array-location': [(3,\n",
" 6)],\n",
" 'chunk-location': (1,),\n",
" 'num-chunks': (2,),\n",
" 'shape': (6,)}},\n",
" ('concatenate-78b27f68e705e10f38419def4321629a', 0): ('getitem-28f0bcca50fbb7eafde78524fc8ec8e6',\n",
" 0),\n",
" ('concatenate-78b27f68e705e10f38419def4321629a', 1): ('arange-690b95eab87b800825077f22e660d393',\n",
" 0),\n",
" ('concatenate-78b27f68e705e10f38419def4321629a', 2): ('arange-690b95eab87b800825077f22e660d393',\n",
" 1),\n",
" ('concatenate-78b27f68e705e10f38419def4321629a', 3): ('getitem-c3e3a02bf3fdd008122271129958abf7',\n",
" 0),\n",
" ('getitem-28f0bcca50fbb7eafde78524fc8ec8e6', 0): (<built-in function getitem>,\n",
" ('arange-690b95eab87b800825077f22e660d393',\n",
" 0),\n",
" (slice(None, None, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 0): ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 0),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 0.09999999999999998): (<built-in function getitem>,\n",
" ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 0),\n",
" (slice(-1, None, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 0.9): (<built-in function getitem>,\n",
" ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 1),\n",
" (slice(0, 1, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 1): ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 1),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 1.1): (<built-in function getitem>,\n",
" ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 1),\n",
" (slice(-1, None, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 1.9): (<built-in function getitem>,\n",
" ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 2),\n",
" (slice(0, 1, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 2): ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 2),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 2.1): (<built-in function getitem>,\n",
" ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 2),\n",
" (slice(-1, None, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 2.9): (<built-in function getitem>,\n",
" ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 3),\n",
" (slice(0, 1, None),)),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1', 3): ('concatenate-78b27f68e705e10f38419def4321629a',\n",
" 3),\n",
" ('getitem-729612624ce61ce35edf66170887d232', 0): (<built-in function getitem>,\n",
" ('overlap-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 1),\n",
" (slice(None, None, None),)),\n",
" ('getitem-729612624ce61ce35edf66170887d232', 1): (<built-in function getitem>,\n",
" ('overlap-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 2),\n",
" (slice(None, None, None),)),\n",
" ('getitem-c3e3a02bf3fdd008122271129958abf7', 0): (<built-in function getitem>,\n",
" ('arange-690b95eab87b800825077f22e660d393',\n",
" 1),\n",
" (slice(-1, -2, -1),)),\n",
" ('lambda-f9fc0ccea52495eb82f414cd021b25d1', 0): (subgraph_callable-3b8ade8e-869e-4087-a0e2-fa0db9edca69,\n",
" ('getitem-729612624ce61ce35edf66170887d232',\n",
" 0)),\n",
" ('lambda-f9fc0ccea52495eb82f414cd021b25d1', 1): (subgraph_callable-3b8ade8e-869e-4087-a0e2-fa0db9edca69,\n",
" ('getitem-729612624ce61ce35edf66170887d232',\n",
" 1)),\n",
" ('overlap-48b8a3d1e03462391c5a32aa8fd122a1', 0): (<function concatenate3 at 0x7f959886bee0>,\n",
" (<function concrete at 0x7f96141874c0>,\n",
" [('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 0),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 0.9)])),\n",
" ('overlap-48b8a3d1e03462391c5a32aa8fd122a1', 1): (<function concatenate3 at 0x7f959886bee0>,\n",
" (<function concrete at 0x7f96141874c0>,\n",
" [('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 0.09999999999999998),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 1),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 1.9)])),\n",
" ('overlap-48b8a3d1e03462391c5a32aa8fd122a1', 2): (<function concatenate3 at 0x7f959886bee0>,\n",
" (<function concrete at 0x7f96141874c0>,\n",
" [('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 1.1),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 2),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 2.9)])),\n",
" ('overlap-48b8a3d1e03462391c5a32aa8fd122a1', 3): (<function concatenate3 at 0x7f959886bee0>,\n",
" (<function concrete at 0x7f96141874c0>,\n",
" [('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 2.1),\n",
" ('getitem-48b8a3d1e03462391c5a32aa8fd122a1',\n",
" 3)]))}\n"
]
}
],
"source": [
"pprint(dict(bb.dask))"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "f4e13193",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([1])"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"get(bb.dask, ('getitem-c3e3a02bf3fdd008122271129958abf7', 0))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "57c5d063",
"metadata": {},
"outputs": [
{
"ename": "KeyError",
"evalue": "('arange-690b95eab87b800825077f22e660d393', 2)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-20-cb817eb37411>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mdsk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/anaconda3/envs/dask/lib/python3.8/site-packages/dask/threaded.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(dsk, result, cache, num_workers, pool, **kwargs)\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0mpool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultiprocessingPoolExecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m results = get_async(\n\u001b[0m\u001b[1;32m 80\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubmit\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_max_workers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/dask/lib/python3.8/site-packages/dask/local.py\u001b[0m in \u001b[0;36mget_async\u001b[0;34m(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[0mfinish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msucceeded\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 528\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnested_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"cache\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 530\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/dask/lib/python3.8/site-packages/dask/local.py\u001b[0m in \u001b[0;36mnested_get\u001b[0;34m(ind, coll)\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnested_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoll\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 310\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcoll\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: ('arange-690b95eab87b800825077f22e660d393', 2)"
]
}
],
"source": [
"key = ('arange-690b95eab87b800825077f22e660d393', 2)\n",
"dsk = dict(bb.dask)\n",
"\n",
"get(dsk, key)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "60e02867",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <div>\n",
" <div>\n",
" <div style=\"width: 52px; height: 52px; position: absolute;\">\n",
" \n",
" <svg width=\"76\" height=\"71\" viewBox=\"0 0 76 71\" fill=\"none\" xmlns=\"http://www.w3.org/2000/svg\">\n",
" <circle cx=\"61.5\" cy=\"36.5\" r=\"13.5\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D); fill: var(--jp-layout-color1, #F2F2F2);\"\n",
" stroke-width=\"2\"/>\n",
" <circle cx=\"14.5\" cy=\"14.5\" r=\"13.5\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D); fill: var(--jp-layout-color1, #F2F2F2);\"\n",
" stroke-width=\"2\"/>\n",
" <circle cx=\"14.5\" cy=\"56.5\" r=\"13.5\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D); fill: var(--jp-layout-color1, #F2F2F2);\"\n",
" stroke-width=\"2\"/>\n",
" <path d=\"M28 16L30.5 16C33.2614 16 35.5 18.2386 35.5 21L35.5 32.0001C35.5 34.7615 37.7386\n",
" 37.0001 40.5 37.0001L43 37.0001\" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\"\n",
" stroke-width=\"1.5\"/>\n",
" <path d=\"M40.5 37L40.5 37.75L40.5 37.75L40.5 37ZM35.5 42L36.25 42L35.5 42ZM35.5 52L34.75\n",
" 52L35.5 52ZM30.5 57L30.5 57.75L30.5 57ZM41.5001 36.25L40.5 36.25L40.5 37.75L41.5001\n",
" 37.75L41.5001 36.25ZM34.75 42L34.75 52L36.25 52L36.25 42L34.75 42ZM30.5 56.25L28.0001\n",
" 56.25L28.0001 57.75L30.5 57.75L30.5 56.25ZM34.75 52C34.75 54.3472 32.8472 56.25 30.5\n",
" 56.25L30.5 57.75C33.6756 57.75 36.25 55.1756 36.25 52L34.75 52ZM40.5 36.25C37.3244 36.25\n",
" 34.75 38.8243 34.75 42L36.25 42C36.25 39.6528 38.1528 37.75 40.5 37.75L40.5 36.25Z\"\n",
" style=\"fill: var(--jp-ui-font-color2, #1D1D1D);\"/>\n",
" <circle cx=\"28\" cy=\"16\" r=\"2.25\" fill=\"#E5E5E5\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"1.5\"/>\n",
" <circle cx=\"28\" cy=\"57\" r=\"2.25\" fill=\"#E5E5E5\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"1.5\"/>\n",
" <path d=\"M45.25 36.567C45.5833 36.7594 45.5833 37.2406 45.25 37.433L42.25 39.1651C41.9167\n",
" 39.3575 41.5 39.117 41.5 38.7321V35.2679C41.5 34.883 41.9167 34.6425 42.25 34.8349L45.25\n",
" 36.567Z\" style=\"fill: var(--jp-ui-font-color2, #1D1D1D);\"/>\n",
" </svg>\n",
" \n",
" </div>\n",
" <div style=\"margin-left: 64px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">HighLevelGraph</h3>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin-bottom:0px;\">\n",
" HighLevelGraph with 9 layers.\n",
" </p>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 0: arange</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" arange-690b95eab87b800825077f22e660d393\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">MaterializedLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(2,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 1: getitem</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" getitem-c3e3a02bf3fdd008122271129958abf7\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">MaterializedLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 2: getitem</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" getitem-28f0bcca50fbb7eafde78524fc8ec8e6\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">MaterializedLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 3: concatenate</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" concatenate-78b27f68e705e10f38419def4321629a\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">MaterializedLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(4,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 4: overlap</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" overlap-48b8a3d1e03462391c5a32aa8fd122a1\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">ArrayOverlapLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(10,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(3,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 5: getitem</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" getitem-729612624ce61ce35edf66170887d232\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">MaterializedLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(6,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(3,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 6: lambda</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" lambda-f9fc0ccea52495eb82f414cd021b25d1\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">Blockwise</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(6,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(3,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 7: block-info</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" block-info-_trim-34ee733cf60b11a59ee93fda12ad670c\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">MaterializedLayer</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(2,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">object</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" <div style=\"\">\n",
" \n",
" <svg width=\"24\" height=\"24\" viewBox=\"0 0 32 32\" fill=\"none\"\n",
" xmlns=\"http://www.w3.org/2000/svg\" style=\"position: absolute;\">\n",
" <circle cx=\"16\" cy=\"16\" r=\"14\" fill=\"#8F8F8F\"\n",
" style=\"stroke: var(--jp-ui-font-color2, #1D1D1D);\" stroke-width=\"2\"/>\n",
" </svg>\n",
" \n",
" <details style=\"margin-left: 32px;\">\n",
" <summary style=\"margin-bottom: 10px; margin-top: 10px;\">\n",
" <h4 style=\"display: inline;\">Layer 8: trim</h4>\n",
" </summary>\n",
" <p style=\"color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;\">\n",
" _trim-34ee733cf60b11a59ee93fda12ad670c\n",
" </p>\n",
" <table style=\"width: 100%;\">\n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">layer_type</th>\n",
" <td style=\"text-align: left;\">Blockwise</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">is_materialized</th>\n",
" <td style=\"text-align: left;\">True</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">shape</th>\n",
" <td style=\"text-align: left;\">(2,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">dtype</th>\n",
" <td style=\"text-align: left;\">int64</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunksize</th>\n",
" <td style=\"text-align: left;\">(1,)</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">type</th>\n",
" <td style=\"text-align: left;\">dask.array.core.Array</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th style=\"text-align: left; width: 150px;\">chunk_type</th>\n",
" <td style=\"text-align: left;\">numpy.ndarray</td>\n",
" </tr>\n",
" </table>\n",
" </details>\n",
" </div>\n",
" \n",
" </div>\n",
" </div>\n",
" </div>\n",
" "
],
"text/plain": [
"HighLevelGraph with 9 layers.\n",
"<dask.highlevelgraph.HighLevelGraph object at 0x7f9598777760>\n",
" 0. arange-690b95eab87b800825077f22e660d393\n",
" 1. getitem-c3e3a02bf3fdd008122271129958abf7\n",
" 2. getitem-28f0bcca50fbb7eafde78524fc8ec8e6\n",
" 3. concatenate-78b27f68e705e10f38419def4321629a\n",
" 4. overlap-48b8a3d1e03462391c5a32aa8fd122a1\n",
" 5. getitem-729612624ce61ce35edf66170887d232\n",
" 6. lambda-f9fc0ccea52495eb82f414cd021b25d1\n",
" 7. block-info-_trim-34ee733cf60b11a59ee93fda12ad670c\n",
" 8. _trim-34ee733cf60b11a59ee93fda12ad670c"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bb.dask"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "6ddbd7d9",
"metadata": {},
"outputs": [
{
"ename": "KeyError",
"evalue": "('arange-690b95eab87b800825077f22e660d393', 2)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-16-645ca6b1cb5d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthreaded\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'arange-690b95eab87b800825077f22e660d393'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/anaconda3/envs/dask/lib/python3.8/site-packages/dask/threaded.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(dsk, result, cache, num_workers, pool, **kwargs)\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0mpool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultiprocessingPoolExecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m results = get_async(\n\u001b[0m\u001b[1;32m 80\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubmit\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_max_workers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/dask/lib/python3.8/site-packages/dask/local.py\u001b[0m in \u001b[0;36mget_async\u001b[0;34m(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[0mfinish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msucceeded\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 528\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnested_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"cache\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 530\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/dask/lib/python3.8/site-packages/dask/local.py\u001b[0m in \u001b[0;36mnested_get\u001b[0;34m(ind, coll)\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnested_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoll\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 310\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcoll\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: ('arange-690b95eab87b800825077f22e660d393', 2)"
]
}
],
"source": [
"from dask.threaded import get\n",
"\n",
"get(bb.dask, ('arange-690b95eab87b800825077f22e660d393', 2))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "84130bdf",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
@GenevieveBuckley
Copy link
Author

Rules in 1D

  • 1 task per chunk from da.ones (this shouldn't be counted in the layer number of tasks, I don't think?
  • If the depth is larger than the smallest chunksize, that screws up these calculations, but we won't worry about that for now
  • first getitem call has 1 task per input chunk
  • concatenate call - there are three concatenate calls, but if chunks are neigbouring they can share the edge ones
  • overlap - there are three overlap calls, but if chunks are neigbouring they can share the edge ones
  • There is one getitem call per chunk after the overlap part
  • actual function passed to overlap - generates 1 task per input array chunk
  • 2 tasks per chunk if trim=True (these are the block-info_trim and _trim tasks)
  • next getitem calls - the

re are four getitem calls, but if chunks are neigbouring they can share the edge ones

Rules in 2D

  • 1 task per chunk from da.ones (this shouldn't be counted in the layer number of tasks, I don't think?
  • If the depth is larger than the smallest chunksize, that screws up these calculations, but we won't worry about that for now
  • first getitem call has 1 task per input chunk
  • ???concatenate call - there are three concatenate calls, but if chunks are neigbouring they can share the edge ones
  • ???more getitems ??
  • ???more concatenate ??
  • ???overlap - there are three overlap calls, but if chunks are neigbouring they can share the edge ones
  • There is one getitem call per chunk after the overlap part
  • actual function passed to overlap - generates 1 task per input array chunk
  • 2 tasks per chunk if trim=True (these

are the block-info_trim and _trim tasks)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment