rechunk-benchmarks (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import dask.array as da"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 256 x 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = da.zeros((256, 256), chunks=(256, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loops, best of 3: 12.9 s per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit x.rechunk((1, 256)).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loops, best of 3: 1.9 s per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit x.rechunk((16, 16)).rechunk((1, 256)).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loops, best of 3: 1.58 s per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit x.rechunk((64, 4)).rechunk((4, 64)).rechunk((1, 256)).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chunks = (256, 1)\n",
    "rechunked = x\n",
    "while chunks[0] > 1:\n",
    "    chunks = (int(chunks[0] / 2), int(chunks[1] * 2))\n",
    "    rechunked = rechunked.rechunk(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loops, best of 3: 1.64 s per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit rechunked.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1024 x 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = da.zeros((1024, 1024), chunks=(1024, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 35s, sys: 28 s, total: 4min 3s\n",
      "Wall time: 3min 45s\n"
     ]
    }
   ],
   "source": [
    "%time _ = y.rechunk((1, 1024)).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loops, best of 3: 14.6 s per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit y.rechunk((32, 32)).rechunk((1, 1024)).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loops, best of 3: 7.29 s per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit y.rechunk((256, 4)).rechunk((32, 32)).rechunk((4, 256)).rechunk((1, 1024)).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.32 s, sys: 3.04 s, total: 12.4 s\n",
      "Wall time: 10.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = (y\n",
    " .rechunk((512, 2))\n",
    " .rechunk((256, 4))\n",
    " .rechunk((128, 8))\n",
    " .rechunk((64, 16))\n",
    " .rechunk((32, 32))\n",
    " .rechunk((16, 64))\n",
    " .rechunk((8, 128))\n",
    " .rechunk((4, 256))\n",
    " .rechunk((2, 512))\n",
    " .rechunk((1, 1024)).compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.68 s, sys: 1.86 s, total: 8.54 s\n",
      "Wall time: 7.28 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = (y\n",
    "#  .rechunk((512, 2))\n",
    " .rechunk((256, 4))\n",
    "#  .rechunk((128, 8))\n",
    " .rechunk((64, 16))\n",
    "#  .rechunk((32, 32))\n",
    " .rechunk((16, 64))\n",
    "#  .rechunk((8, 128))\n",
    " .rechunk((4, 256))\n",
    "#  .rechunk((2, 512))\n",
    " .rechunk((1, 1024)).compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}