01_environment.ipynb

02_dataframe.ipynb

03_array.ipynb

04_delayed.ipynb

05_dask_gateway.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4405cb5a-cdc7-4e3c-96c1-a8dc2bf53509",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: dask_gateway==2023.9.0 in /opt/conda/lib/python3.11/site-packages (2023.9.0)\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.11/site-packages (from dask_gateway==2023.9.0) (3.8.6)\n",
      "Requirement already satisfied: click>=8.1.3 in /opt/conda/lib/python3.11/site-packages (from dask_gateway==2023.9.0) (8.1.7)\n",
      "Requirement already satisfied: dask>=2022.4.0 in /opt/conda/lib/python3.11/site-packages (from dask_gateway==2023.9.0) (2023.9.3)\n",
      "Requirement already satisfied: distributed>=2022.4.0 in /opt/conda/lib/python3.11/site-packages (from dask_gateway==2023.9.0) (2023.9.3)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.11/site-packages (from dask_gateway==2023.9.0) (6.0.1)\n",
      "Requirement already satisfied: tornado in /opt/conda/lib/python3.11/site-packages (from dask_gateway==2023.9.0) (6.3.3)\n",
      "Requirement already satisfied: cloudpickle>=1.5.0 in /opt/conda/lib/python3.11/site-packages (from dask>=2022.4.0->dask_gateway==2023.9.0) (2.2.1)\n",
      "Requirement already satisfied: fsspec>=2021.09.0 in /opt/conda/lib/python3.11/site-packages (from dask>=2022.4.0->dask_gateway==2023.9.0) (2023.9.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.11/site-packages (from dask>=2022.4.0->dask_gateway==2023.9.0) (23.2)\n",
      "Requirement already satisfied: partd>=1.2.0 in /opt/conda/lib/python3.11/site-packages (from dask>=2022.4.0->dask_gateway==2023.9.0) (1.4.1)\n",
      "Requirement already satisfied: toolz>=0.10.0 in /opt/conda/lib/python3.11/site-packages (from dask>=2022.4.0->dask_gateway==2023.9.0) (0.12.0)\n",
      "Requirement already satisfied: importlib-metadata>=4.13.0 in /opt/conda/lib/python3.11/site-packages (from dask>=2022.4.0->dask_gateway==2023.9.0) (6.8.0)\n",
      "Requirement already satisfied: jinja2>=2.10.3 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (3.1.2)\n",
      "Requirement already satisfied: locket>=1.0.0 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (1.0.0)\n",
      "Requirement already satisfied: msgpack>=1.0.0 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (1.0.6)\n",
      "Requirement already satisfied: psutil>=5.7.2 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (5.9.5)\n",
      "Requirement already satisfied: sortedcontainers>=2.0.5 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (2.4.0)\n",
      "Requirement already satisfied: tblib>=1.6.0 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (2.0.0)\n",
      "Requirement already satisfied: urllib3>=1.24.3 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (2.0.5)\n",
      "Requirement already satisfied: zict>=3.0.0 in /opt/conda/lib/python3.11/site-packages (from distributed>=2022.4.0->dask_gateway==2023.9.0) (3.0.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (3.2.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.11/site-packages (from aiohttp->dask_gateway==2023.9.0) (1.3.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.11/site-packages (from importlib-metadata>=4.13.0->dask>=2022.4.0->dask_gateway==2023.9.0) (3.17.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.11/site-packages (from jinja2>=2.10.3->distributed>=2022.4.0->dask_gateway==2023.9.0) (2.1.3)\n",
      "Requirement already satisfied: idna>=2.0 in /opt/conda/lib/python3.11/site-packages (from yarl<2.0,>=1.0->aiohttp->dask_gateway==2023.9.0) (3.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install dask_gateway==2023.9.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d9ef7be-a3f1-4fff-b45b-0bfd77b9498c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_gateway import Gateway\n",
    "import os\n",
    "gateway = Gateway(\n",
    "    \"http://traefik-dask-gateway\",\n",
    "    public_address=\"http://\"+os.environ[\"DASK_GATEWAY_IP\"],\n",
    "    auth=\"jupyterhub\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4fe7ec3e-0169-4c13-b8e7-6d2e483fc74c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ClusterReport<name=jhub.8691fb24197a4f67a45e3661260dd36d, status=RUNNING>]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gateway.list_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "14b7d8e9-e7fd-4111-9235-798f1a88a036",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stop_all_clusters():\n",
    "    for c in gateway.list_clusters():\n",
    "        gateway.stop_cluster(c.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cb9d2231-aebc-4067-b45e-c75c936ed171",
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_all_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7bd3e53a-19c5-4861-973b-faf276c33588",
   "metadata": {},
   "outputs": [],
   "source": [
    "options = gateway.cluster_options()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a0cf63ca-5658-45ed-a5c3-b019c99dbfe6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c453757250234ca587fdc4e9cfebc056",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<h2>Cluster Options</h2>'), GridBox(children=(HTML(value=\"<p style='font-weight: bo…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b26238bb-3889-439f-8633-00ed72e86f24",
   "metadata": {},
   "outputs": [],
   "source": [
    "options[\"worker_cores\"] = 1\n",
    "options[\"worker_memory\"] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2dd3761d-c46b-476e-bb9d-586242a85420",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster = gateway.new_cluster(options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cb7c5708-4409-4689-b4fb-3b17d62664e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.scale(2) # up to 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6702f5c2-145e-4fcf-9f45-901034800871",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ClusterReport<name=jhub.491a0cad1fc04dfc8a5893e43a7a10dd, status=RUNNING>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gateway.list_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "393bc3a8-109e-4986-b730-73b98ee4543e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/distributed/client.py:1390: VersionMismatchWarning: Mismatched versions found\n",
      "\n",
      "+-------------+----------+-----------+---------+\n",
      "| Package     | Client   | Scheduler | Workers |\n",
      "+-------------+----------+-----------+---------+\n",
      "| dask        | 2023.9.3 | 2023.9.2  | None    |\n",
      "| distributed | 2023.9.3 | 2023.9.2  | None    |\n",
      "| lz4         | 4.3.2    | None      | None    |\n",
      "+-------------+----------+-----------+---------+\n",
      "  warnings.warn(version_module.VersionMismatchWarning(msg[0][\"warning\"]))\n"
     ]
    }
   ],
   "source": [
    "client = cluster.get_client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6fdf0102-3ce1-41aa-9fbf-ccca79167618",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-eedd0dce-6ec6-11ee-8149-667ae350cb21</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "\n",
       "        <tr>\n",
       "        \n",
       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
       "        \n",
       "        </tr>\n",
       "\n",
       "        \n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard: </strong> <a href=\"http://149.165.155.150/clusters/jhub.491a0cad1fc04dfc8a5893e43a7a10dd/status\" target=\"_blank\">http://149.165.155.150/clusters/jhub.491a0cad1fc04dfc8a5893e43a7a10dd/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\"></td>\n",
       "            </tr>\n",
       "        \n",
       "\n",
       "        </table>\n",
       "\n",
       "        \n",
       "\n",
       "        \n",
       "            <details>\n",
       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
       "            <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
       "  <h3>GatewayCluster</h3>\n",
       "  <ul>\n",
       "    <li><b>Name: </b>jhub.491a0cad1fc04dfc8a5893e43a7a10dd\n",
       "    <li><b>Dashboard: </b><a href='http://149.165.155.150/clusters/jhub.491a0cad1fc04dfc8a5893e43a7a10dd/status' target='_blank'>http://149.165.155.150/clusters/jhub.491a0cad1fc04dfc8a5893e43a7a10dd/status</a>\n",
       "  </ul>\n",
       "</div>\n",
       "\n",
       "            </details>\n",
       "        \n",
       "\n",
       "    </div>\n",
       "</div>"
      ],
      "text/plain": [
       "<Client: 'tls://10.233.65.83:8786' processes=0 threads=0, memory=0 B>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2027e94c-08b4-468a-9bdd-1371e4d32fc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b9ec6f327364880bab284c22fe19b7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<h2>GatewayCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n<style scoped>\\n    …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bf8eba43-b6b8-4561-9b5b-e78b6aa93823",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dask.bag<slow_half, npartitions=8>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import dask.bag as db\n",
    "data = list(range(1,9))\n",
    "b = db.from_sequence(data)\n",
    "from time import sleep\n",
    "\n",
    "def slow_half( x):\n",
    "    sleep(1)\n",
    "    return x // 2\n",
    "\n",
    "res = b.map(slow_half)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "82c55ca6-b48c-4a0d-9beb-ad498d157d0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.8 ms, sys: 10.7 ms, total: 19.5 ms\n",
      "Wall time: 5.02 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0, 1, 1, 2, 2, 3, 3, 4]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "res.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0cbbb26e-fdec-4f92-a8a6-612e92c10511",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.array as da\n",
    "a = da.random.normal(size=(10000, 10000), chunks=(500, 500))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "327d47f8-4211-4883-9d34-f8865917f47d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.9 ms, sys: 6.34 ms, total: 30.2 ms\n",
      "Wall time: 2.21 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.00018126980730181626"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "a.mean().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bd8fa8a7-a7f1-479e-986b-e4a774282602",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_on_worker():\n",
    "    import dask\n",
    "    return dask.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e492eba8-4206-4f8c-811c-002e2973a2a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tls://10.233.65.84:40429': '2023.9.2',\n",
       " 'tls://10.233.65.85:39933': '2023.9.2'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.run(run_on_worker)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1620ebca-59ee-4f9c-8695-e7f36c2a3bea",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cluster.shutdown()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

06_array_distributed.ipynb

07_zarr_object_store.ipynb