dask_cold_start_random_slowdown.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "GPU_id = '3,5,6,7'\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = GPU_id\n",
    "num_gpus = len(GPU_id.split(','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dask 2.10.1\n",
      "cupy 7.1.1\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import dask\n",
    "import dask.array as da\n",
    "import cupy as cp\n",
    "\n",
    "import subprocess\n",
    "from dask.distributed import Client, wait\n",
    "from dask_cuda import LocalCUDACluster\n",
    "\n",
    "print('dask',dask.__version__)\n",
    "print('cupy',cp.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://10.33.227.156:33513</li>\n",
       "  <li><b>Dashboard: </b><a href='http://10.33.227.156:8787/status' target='_blank'>http://10.33.227.156:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>4</li>\n",
       "  <li><b>Memory: </b>1.08 TB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: 'tcp://10.33.227.156:33513' processes=4 threads=4, memory=1.08 TB>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmd = \"hostname --all-ip-addresses\"\n",
    "process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)\n",
    "output, error = process.communicate()\n",
    "IPADDR = str(output.decode()).split()[0]\n",
    "\n",
    "cluster = LocalCUDACluster(ip=IPADDR)\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mean(n):\n",
    "  n_samples = 10**n\n",
    "  n_features = 20\n",
    "  chunks = 4\n",
    "\n",
    "  rs = da.random.RandomState(RandomState=cp.random.RandomState)  # <-- we specify cupy here\n",
    "  dx = rs.normal(0, 1, size=(n_samples, n_features), chunks=(n_samples//chunks, n_features))\n",
    "\n",
    "  dx, = dask.persist(dx)\n",
    "  mean,std = dask.compute(dx.mean(axis=0),dx.std(axis=0))\n",
    "  del rs,dx\n",
    "  return mean,std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.03 s, sys: 592 ms, total: 1.62 s\n",
      "Wall time: 24.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "mean,std = get_mean(8) \n",
    "# there is 20% chance that this takes 24 seconds\n",
    "# for the rest 80% chance it takes 2 sconds\n",
    "# just rerun it for a few times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 100 ms, sys: 12 ms, total: 112 ms\n",
      "Wall time: 2.13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "mean,std = get_mean(8) # the 2nd run always takes 2 seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 136 ms, sys: 12 ms, total: 148 ms\n",
      "Wall time: 2.19 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "get_mean(3) # run with a small array first\n",
    "mean,std = get_mean(8) # always takes 2 seconds"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}