Test Dask Gateway.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d22caf9-378b-47e0-9640-bd956ddcba5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install dask-gateway numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d9ef7be-a3f1-4fff-b45b-0bfd77b9498c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_gateway import Gateway\n",
    "gateway = Gateway(\n",
    "    \"http://149.165.xxx.xxx\",\n",
    "    auth=\"jupyterhub\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fe7ec3e-0169-4c13-b8e7-6d2e483fc74c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateway.list_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bd3e53a-19c5-4861-973b-faf276c33588",
   "metadata": {},
   "outputs": [],
   "source": [
    "options = gateway.cluster_options()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0cf63ca-5658-45ed-a5c3-b019c99dbfe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dd3761d-c46b-476e-bb9d-586242a85420",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster = gateway.new_cluster(options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb7c5708-4409-4689-b4fb-3b17d62664e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.scale(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6702f5c2-145e-4fcf-9f45-901034800871",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateway.list_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "393bc3a8-109e-4986-b730-73b98ee4543e",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = cluster.get_client()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2027e94c-08b4-468a-9bdd-1371e4d32fc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf8eba43-b6b8-4561-9b5b-e78b6aa93823",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.bag as db\n",
    "data = list(range(1,9))\n",
    "b = db.from_sequence(data)\n",
    "from time import sleep\n",
    "\n",
    "def slow_half( x):\n",
    "    sleep(1)\n",
    "    return x // 2\n",
    "\n",
    "res = b.map(slow_half)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82c55ca6-b48c-4a0d-9beb-ad498d157d0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "res.compute()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cbbb26e-fdec-4f92-a8a6-612e92c10511",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.array as da\n",
    "a = da.random.normal(size=(10000, 10000), chunks=(500, 500))\n",
    "a.mean().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1620ebca-59ee-4f9c-8695-e7f36c2a3bea",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10c8c6fc-c57c-4314-ad1a-5b0da4305bef",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}