dask-profiling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext snakeviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask import optimize\n",
    "from dask.datasets import timeseries\n",
    "from dask.dataframe.shuffle import shuffle\n",
    "from distributed import Client, wait"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://127.0.0.1:53407</li>\n",
       "  <li><b>Dashboard: </b><a href='http://127.0.0.1:8787/status' target='_blank'>http://127.0.0.1:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>12</li>\n",
       "  <li><b>Memory: </b>17.18 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: 'tcp://127.0.0.1:53407' processes=4 threads=12, memory=17.18 GB>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Client()\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.cancel(df)\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddf_h = timeseries(start='2000-01-01', end='2005-01-01', partition_freq='1d')\n",
    "ddf_h = ddf_h.persist()\n",
    "_ = wait(ddf_h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = shuffle(ddf_h, \"id\", shuffle=\"tasks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "*** Profile stats marshalled to file '/var/folders/b6/qzyt63c16db3p_64qvs_1rsxctzzv8/T/tmp0p44ngrl'. \n",
      "Embedding SnakeViz in this document...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<iframe id='snakeviz-fa94c9a8-8b45-11ea-9579-3af9d31f5fe3' frameborder=0 seamless width='100%' height='1000'></iframe>\n",
       "<script>document.getElementById(\"snakeviz-fa94c9a8-8b45-11ea-9579-3af9d31f5fe3\").setAttribute(\"src\", \"http://\" + document.location.hostname + \":8080/snakeviz/%2Fvar%2Ffolders%2Fb6%2Fqzyt63c16db3p_64qvs_1rsxctzzv8%2FT%2Ftmp0p44ngrl\")</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n"
     ]
    }
   ],
   "source": [
    "%%snakeviz\n",
    "\n",
    "# optimize_graph=False\n",
    "# _graph_to_futures = ~13s\n",
    "# collections_to_dsk = 0\n",
    "\n",
    "# optimize_graph=True\n",
    "# _graph_to_futures = ~7.5s\n",
    "# collections_to_dsk = ~3.1s\n",
    "\n",
    "df = client.persist(result, optimize_graph=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "*** Profile stats marshalled to file '/var/folders/b6/qzyt63c16db3p_64qvs_1rsxctzzv8/T/tmp092vg6fh'. \n",
      "Embedding SnakeViz in this document...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<iframe id='snakeviz-cad6fc72-8b45-11ea-9579-3af9d31f5fe3' frameborder=0 seamless width='100%' height='1000'></iframe>\n",
       "<script>document.getElementById(\"snakeviz-cad6fc72-8b45-11ea-9579-3af9d31f5fe3\").setAttribute(\"src\", \"http://\" + document.location.hostname + \":8080/snakeviz/%2Fvar%2Ffolders%2Fb6%2Fqzyt63c16db3p_64qvs_1rsxctzzv8%2FT%2Ftmp092vg6fh\")</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%snakeviz\n",
    "\n",
    "# optimize_graph=False\n",
    "# _graph_to_futures = ~13s\n",
    "# collections_to_dsk = 0\n",
    "\n",
    "# optimize_graph=True\n",
    "# _graph_to_futures = ~7.5s\n",
    "# collections_to_dsk = ~3.1s\n",
    "\n",
    "df = client.persist(result, optimize_graph=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}