Investing the cost of making a contiguous final array

dask_cost_of_c_contiguous_final_array.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6.6 | packaged by conda-forge | (default, Jul 26 2018, 09:53:17) \n",
      "[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]\n",
      "0.19.0\n",
      "1.15.0\n"
     ]
    }
   ],
   "source": [
    "import dask.array as da\n",
    "import numpy as np\n",
    "import pickle\n",
    "import dask\n",
    "import sys\n",
    "print(sys.version)\n",
    "print(dask.__version__)\n",
    "print(np.__version__)\n",
    "\n",
    "shape = (50, 1024, 1024)\n",
    "chunks = (shape[0], shape[1]//2, shape[2]//2)\n",
    "dtype = 'float32'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask import compute\n",
    "a00 = da.zeros(shape=chunks, chunks=-1, dtype=dtype)\n",
    "a01 = da.zeros(shape=(chunks[0], chunks[1], chunks[2]-1), chunks=-1, dtype=dtype)\n",
    "a10 = da.zeros(shape=(chunks[0], chunks[1]-1, chunks[2]), chunks=-1, dtype=dtype)\n",
    "a11 = da.zeros(shape=(chunks[0], chunks[1]-1, chunks[2]-1), chunks=-1, dtype=dtype)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.8 ms, sys: 106 µs, total: 1.91 ms\n",
      "Wall time: 1.49 ms\n",
      "CPU times: user 3.59 ms, sys: 24 µs, total: 3.61 ms\n",
      "Wall time: 3.29 ms\n",
      "CPU times: user 115 ms, sys: 248 ms, total: 363 ms\n",
      "Wall time: 505 ms\n"
     ]
    }
   ],
   "source": [
    "block = [[a00, a01], [a10, a11]]\n",
    "%time _ = compute(block, scheduler='synchronous')\n",
    "%time _ = compute(block, scheduler='threads')\n",
    "%time _ = compute(block, scheduler='processes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed_block = compute(block)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 55.6 ms, sys: 128 ms, total: 183 ms\n",
      "Wall time: 182 ms\n"
     ]
    }
   ],
   "source": [
    "%time _ = np.block(computed_block)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mark/miniconda3/envs/owl/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/mark/miniconda3/envs/owl/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 242 ms, sys: 107 ms, total: 350 ms\n",
      "Wall time: 354 ms\n",
      "CPU times: user 33.1 ms, sys: 80.6 ms, total: 114 ms\n",
      "Wall time: 113 ms\n",
      "CPU times: user 475 ms, sys: 1 s, total: 1.48 s\n",
      "Wall time: 1.64 s\n"
     ]
    }
   ],
   "source": [
    "%time _ = da.block(block).compute(scheduler='synchronous')\n",
    "%time _ = da.block(block).compute(scheduler='threads')\n",
    "%time _ = da.block(block).compute(scheduler='processes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = da.zeros(shape=(shape[0], shape[1]-1, shape[2]-1), \n",
    "             chunks=(chunks[0], (chunks[1], chunks[1]-1), (chunks[2], chunks[2]-1)), \n",
    "             dtype=dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 39.5 ms, sys: 92.2 ms, total: 132 ms\n",
      "Wall time: 130 ms\n",
      "CPU times: user 32.3 ms, sys: 76.7 ms, total: 109 ms\n",
      "Wall time: 108 ms\n",
      "CPU times: user 157 ms, sys: 334 ms, total: 491 ms\n",
      "Wall time: 606 ms\n"
     ]
    }
   ],
   "source": [
    "%time _ = z.compute(scheduler='synchronous')\n",
    "%time _ = z.compute(scheduler='threads')\n",
    "%time _ = z.compute(scheduler='processes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}