jacobtomlinson/Performance comparison of s3fs, goofys and pysssix.ipynb

## Performance comparison of s3fs, goofys and pysssix.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance comparison of s3fs, goofys and pysssix\n",
    "\n",
    "I've been experimenting with an ~80GB zarr file created with xarray and wanted to compare performance between using s3fs and FUSE (goofys and pysssix specifically)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "import iris\n",
    "import iris.util\n",
    "import numpy as np\n",
    "\n",
    "import xarray\n",
    "import zarr\n",
    "import s3fs\n",
    "\n",
    "import distributed\n",
    "from dask_kubernetes import KubeCluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimate_cube_size(cube):\n",
    "    \"\"\"Take an object with a shape attribute and estimate the in memory size.\"\"\"\n",
    "    import functools\n",
    "    import operator\n",
    "    num_points = functools.reduce(operator.mul, cube.shape, 1)\n",
    "    if cube[(0,) * len(cube.shape)].data.dtype != 'float32':\n",
    "        return False\n",
    "    return human_bytes((num_points * 32) / 8)\n",
    "\n",
    "def human_bytes(num, suffix='B'):\n",
    "    \"\"\"Take an integer of bytes and return a string representation in human terms.\"\"\"\n",
    "    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n",
    "        if abs(num) < 1024.0:\n",
    "            return \"%3.1f%s%s\" % (num, unit, suffix)\n",
    "        num /= 1024.0\n",
    "    return \"%.1f%s%s\" % (num, 'Yi', suffix)\n",
    "\n",
    "def update_worker_memory(cluster, new_limit):\n",
    "    \"\"\"Modify the memory requests of a worker template after a cluster has been created.\"\"\"\n",
    "    cluster.pod_template.spec.containers[0].resources.limits[\"memory\"] = new_limit\n",
    "    cluster.pod_template.spec.containers[0].resources.requests[\"memory\"] = new_limit\n",
    "    if '--memory-limit' in cluster.pod_template.spec.containers[0].args:\n",
    "        index = cluster.pod_template.spec.containers[0].args.index('--memory-limit')\n",
    "        cluster.pod_template.spec.containers[0].args[index + 1] = new_limit\n",
    "    return cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14f741dd9f704c36aa46abbae788a5fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<h2>KubeCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n  <style scoped>\\n    .…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cluster = KubeCluster()\n",
    "cluster = update_worker_memory(cluster, '3G')\n",
    "cluster.scale(100)\n",
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://100.99.62.19:43655\n",
       "  <li><b>Dashboard: </b><a href='/user/jacobtomlinson/proxy/8787/status' target='_blank'>/user/jacobtomlinson/proxy/8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>100</li>\n",
       "  <li><b>Cores: </b>100</li>\n",
       "  <li><b>Memory: </b>300.00 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://100.99.62.19:43655' processes=100 cores=100>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = distributed.Client(cluster.scheduler_address)\n",
    "c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset with s3fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:                         (forecast_period: 59, forecast_reference_time: 20, latitude: 600, longitude: 800, pressure: 3, realization: 12)\n",
       "Coordinates:\n",
       "  * forecast_period                 (forecast_period) timedelta64[ns] 00:00:00 ...\n",
       "  * forecast_reference_time         (forecast_reference_time) datetime64[ns] 2016-01-01 ...\n",
       "  * latitude                        (latitude) float32 -89.85 -89.549995 ...\n",
       "  * longitude                       (longitude) float32 0.225 0.67499995 ...\n",
       "  * pressure                        (pressure) float32 250.0 500.0 850.0\n",
       "  * realization                     (realization) int32 0 1 2 3 4 5 6 7 8 9 ...\n",
       "    time                            (forecast_reference_time, forecast_period) datetime64[ns] dask.array<shape=(20, 59), chunksize=(20, 59)>\n",
       "Data variables:\n",
       "    wet_bulb_potential_temperature  (forecast_reference_time, realization, forecast_period, pressure, latitude, longitude) float32 dask.array<shape=(20, 12, 59, 3, 600, 800), chunksize=(20, 3, 59, 3, 100, 100)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3fs_dataset = xarray.open_zarr(\n",
    "    s3fs.S3Map(\n",
    "        root='informatics-pangeo-scratch/jacobtomlinson/hypermegacube.zarr', \n",
    "        s3=s3fs.S3FileSystem(), \n",
    "        check=False\n",
    "    )\n",
    ")\n",
    "\n",
    "s3fs_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset with goofys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:                         (forecast_period: 59, forecast_reference_time: 20, latitude: 600, longitude: 800, pressure: 3, realization: 12)\n",
       "Coordinates:\n",
       "  * forecast_period                 (forecast_period) timedelta64[ns] 00:00:00 ...\n",
       "  * forecast_reference_time         (forecast_reference_time) datetime64[ns] 2016-01-01 ...\n",
       "  * latitude                        (latitude) float32 -89.85 -89.549995 ...\n",
       "  * longitude                       (longitude) float32 0.225 0.67499995 ...\n",
       "  * pressure                        (pressure) float32 250.0 500.0 850.0\n",
       "  * realization                     (realization) int32 0 1 2 3 4 5 6 7 8 9 ...\n",
       "    time                            (forecast_reference_time, forecast_period) datetime64[ns] dask.array<shape=(20, 59), chunksize=(20, 59)>\n",
       "Data variables:\n",
       "    wet_bulb_potential_temperature  (forecast_reference_time, realization, forecast_period, pressure, latitude, longitude) float32 dask.array<shape=(20, 12, 59, 3, 600, 800), chunksize=(20, 3, 59, 3, 100, 100)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "goofys_dataset = xarray.open_zarr('/scratch/jacobtomlinson/hypermegacube.zarr')\n",
    "\n",
    "goofys_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset with pysssix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:                         (forecast_period: 59, forecast_reference_time: 20, latitude: 600, longitude: 800, pressure: 3, realization: 12)\n",
       "Coordinates:\n",
       "  * forecast_period                 (forecast_period) timedelta64[ns] 00:00:00 ...\n",
       "  * forecast_reference_time         (forecast_reference_time) datetime64[ns] 2016-01-01 ...\n",
       "  * latitude                        (latitude) float32 -89.85 -89.549995 ...\n",
       "  * longitude                       (longitude) float32 0.225 0.67499995 ...\n",
       "  * pressure                        (pressure) float32 250.0 500.0 850.0\n",
       "  * realization                     (realization) int32 0 1 2 3 4 5 6 7 8 9 ...\n",
       "    time                            (forecast_reference_time, forecast_period) datetime64[ns] dask.array<shape=(20, 59), chunksize=(20, 59)>\n",
       "Data variables:\n",
       "    wet_bulb_potential_temperature  (forecast_reference_time, realization, forecast_period, pressure, latitude, longitude) float32 dask.array<shape=(20, 12, 59, 3, 600, 800), chunksize=(20, 3, 59, 3, 100, 100)>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pysssix_dataset = xarray.open_zarr('/s3/informatics-pangeo-scratch/jacobtomlinson/hypermegacube.zarr')\n",
    "\n",
    "pysssix_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Estimate the array size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'76.0GiB'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimate_cube_size(s3fs_dataset.wet_bulb_potential_temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'76.0GiB'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimate_cube_size(goofys_dataset.wet_bulb_potential_temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'76.0GiB'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimate_cube_size(pysssix_dataset.wet_bulb_potential_temperature)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time taking a mean of the entire array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12.1 s ± 2.09 s per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<TimeitResult : 12.1 s ± 2.09 s per loop (mean ± std. dev. of 7 runs, 1 loop each)>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3fs_timings = %timeit -o s3fs_dataset.wet_bulb_potential_temperature.data.mean().compute()\n",
    "s3fs_timings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.2 s ± 764 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<TimeitResult : 10.2 s ± 764 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "goofys_timings = %timeit -o goofys_dataset.wet_bulb_potential_temperature.data.mean().compute()\n",
    "goofys_timings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36 s ± 1.61 s per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<TimeitResult : 36 s ± 1.61 s per loop (mean ± std. dev. of 7 runs, 1 loop each)>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pysssix_timings = %timeit -o pysssix_dataset.wet_bulb_potential_temperature.data.mean().compute()\n",
    "pysssix_timings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXhwAmGBaFiCAgqFi2GLQBrVahLj9RCoh6XSooVC7C77rUW6mCrSJVa6uI9UrhQsvirRdxQ7EiWlAUN2hYlat14SrGIEsUBBQU+Nw/zkkcwoQMkJk5k7yfj0cemfM9y/dzZs7MZ77f851zzN0RERGJmjrpDkBERCQeJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJagUMLNsM3MzaxVn3iFmttXMWqYjtkxnZkeZ2RtmtsXM7kp3PJI4M7vDzB7ax/xhZjYvlTGlk5n1MrMP0x1HlChBJShMImV/u83sm5jpKw50u+6+w91z3b2kOuOtRf4/8LG7N3T3W9MdDICZ5ZjZX8xsvZmVmtnTZnZkzPw8M3vWzLaZ2f+a2cUV1m9lZlPMbG14fH0Ubq99OL9D+IWn7Pj73Mz+aGZZqd7Xg+Hut7v7tVC+TzsPZntmdqOZLTWzb81sYpz555nZ++HzPi/2C2P4mj1sZl+ZWYmZXXswsUj1UIJKUJhEct09F1gD9IkpeyTd8dViRwP/k+4gKhgBFACdgVbADuD+mPmTgC+BI4CrgSkxyac58BbBe/NUoCFQGJadFbONXTHH44nA2cC/JnGfMLO6iZSlUTEwGvhrxRlm1gKYSfDaNCM4ZmKXuxs4CmgD9AJuN7OeyQ13j/iq7XmM2GtycNxdf/v5B3wMnF2h7DRgEbAZKAHGAXXDedmAA63C6Z8An4brVJz3KPAA8AKwBXgdODqmnt7AB8CmcLm3gAGVxFlpTHGW7QDsJPjA/AwoBX4O/Ah4J6zv/grrXAP8E/gCeA44KmbeBIIPjK+AxcApMfPuAR4BZoT7uBLouo/nuwewNNyPt4BuYfkM4DuCBLAVOD3Ouv2BFWEcnwCj4uzz4DDWDcCIajg+pgJjYqYvAlaEjw8L64x9TR8HRoeP7wMWV7H9DsDOCmUPAg9Wsvxvwuen7O87YGLMa/he+Dp8CPw8Zr1eYdlvgHXA5Hhlcer7HOgcPh5CcHwfE05fCzwacxz8OXy8PlyuLMYTgWHA/HDfNgEfUeF9V8n+3le2fzFl1wMvxUw3Ab4F2obTpcAZMfPvBaZVsv1E9y8HGA+sDY+ve4F6VT23MfWMIHhvHBlzLK8Mn4uFQKcKMd0ErAK+jnnd1xIc++8S5/0R9T+1oKrPdwQH5+HA6UAfgoN3D2bWF5gO9HX31yvZ1s+AkeG21gJ3hOseSfAt8EYgjyDp/PBgY4qRBZwAHEPwof0fBAd9j7B8sJmdHMZyGfCLcJvNgWXs+Y30TSAfaAo8AzxuZvVi5vcHphB8UMwnSLZ7MbMjgGcJPsyaAhOBOWbW2N0vB54EfutBa2JhnE18RfB8NgnrvMnMelXY50LgOOB84C4zO2Yfz1FcZmYxk5OBnmZ2pJnlApcDz4fzOgBb3P2TmOVXELS2IGgJPbWfdbcCziFI3ntx97LnJ5fgNSklSIoQHF/nAY0IEsJ4M+scs3pboB7QmuBDvrKyWK8CPcPHZwCrCY6hsulX4qxzBjGtQndfFlNeRPDaPwT8Od4+JqAzwfMMgLtvIugJ6Ry2rg6Pnc+er0lFie7fHQTvm3yC92lP4Fcx22lLJc9jeD71YqCnu39uZqcAfyJ4XzYF/gt4ukJr6VKC46CpmRWEy3YFGhN8sS2uZH+iK90ZMhP/iNOCirPMLcCM8HFZK+lm4H+BDjHLxWtBPRQz/0Jgefh4KPByzLw6BN8847ag9hVTnHkdwjiaxpRtA/rFTD8HDAsfvwxcETOvHkFCbB5n2wZ8DfwgnL4H+FvM/JOATZXE9a/AqxXKlgGXxTxfv96P124i8LsK+9wsZv5K4IJK1m1D8MG+FlgCXAe0JPi2/1TMcocRJE4HdgH/ABqF884hOGcWu93rgLnh42JgUMy8Swi+MW8BZleIe1P458AC4NAq9v3QcP9u2Mcyc4Frwse9wmOgXsz8vcribOPfgMfCx6vD13BaOP054Td/9mxBxWsVDgPeiZk+PNzXJlXsZ7wW1COErdSYsiXAZUD7cLsWM68P8N5B7t9nwJkx6/Ur2+Y+ntuPCFpdLwMNY+ZNBW6tEMcnwMkx9f4sZl7n8Dj9CZX0mmTCn1pQ1cTMOpnZ82a2zsy+Am4j6OuO9e/AX939vSo293nM46+B3PBxS4KuQQDcfTfBm+BgYoq1y91LY6a/Ieh+iJ0ui+VoYKKZbTKzTQTdYzsJzrlgZiPN7J9mtpngfEt2hbor28eKWhK8EWN9QnC+oEpmdpqZvWJmG8JYBlWIY5e7b0wwlksJWomtgOFAN4Jv2pMJvtGW+TPBc3E4QVJ4AfhbOG8rQWslViOCBARB66ZF2Qx3f8zdmxC0qOtXiLtJOC+XIPH8jUqELbyHgSJ3/2NMeV8zW2xmX4Sv45lUeJ3c/bsKm4tXFusVoIeZtSH4EH4KOMPMOhB8qXp3H+tWVPE4gcpfn33Z1/O+NZxuGGdePFXuX/h8H8mex27F4zbe83gEQcvnTnePrf9oYFTZ+y18rfIqbC/2s2EVwRfSu4D1ZvZIeH4zoyhBVZ/JBOdJjnX3RsAYgpZDrP7AADMbdoB1rCVMAABmVod9f1AnEtOB+pTgm36TmL8cd19iZucQtAr6E3StHU6Q3A6k7hKCN2esNuwjMVfwGEG3aGt3bwxMO8A4AMa6+zPuvsvdF7v7le6e5+6F7j4rZrkCYIq7f+nu2wm6pk4Pu/veAxqFH26xy68KH88H+lfoMtwnd99G0G3cM6wjntsJkv3wsgIzO5SgRfhb4Igw2b3Ens+Px6uyipBWEXSdDgNeCb/0bAWuJGgNH8g2D9YqgucZADNrTHBcrXL3tQTnUQtilo99TeJta5/7F+7j5+x57FY8buPt8zrgAuC/zax7TPmnwG0V3m8N3D22O3iP7bn7dHc/laDLPhu4s5L9iSwlqOrTENjs7lvDPvx4I6rWEIzEGmVmPz+AOmYDJ5vZ+WHf878TdCcdTEwHaiLwazP7AYCZHWZmF8XU+x1Bq6o+QWLMPsB6ZgMnmtnFZlbXzK4keKPPrWrF8EM+Fyh19+1mdirwLwcYR1mLNRH/AAaZWUMzq0+QFP7X3be6+5cELZ0xZtYgHCnWi6ALCuAPBF9CpppZOws0JjiXEZeZZQMDgE/cfWuc+RcQDH650N13xMzKIeiaXQ/sDs+P9kxwHysVfji/SnD+s+x8zCsVpitaD2RVSNz7JTw+sgmSR5YFvz8sG3r/BNDNzPqEy9wBvOHuH4fz/wu4zcwam1k+QUt72kHu3wyC0YBNw3OptxJnhGGc7b9IMEDpWTM7MSyeBFxnZoXhMZEbtn4bVPJcdDKzHmZ2CMGXw28IupszihJU9bkRGGJmWwn6kGfGW8jdVxMkqTFmNnB/Kgi/6V1OMKppI8EH2dsEo9gOOKYD4e4zCFoGT4Xdh8sJzq9AMKjhVYL+9NVhrBsOsJ51QF+CN3cpwYfATz04yV3Vuk7wLfc+M9tCcIL68X2vVS1uIHhvrSb44O1JMJKvzL8SdKNtJPgQvNrdPwhj/hw4maAV8yZBN9MSgg/d2BPpWRb+DoqgZV1A8M07nssIBrJ8YN//duqBsGvzJoLXqzRcf85B7fn3XiH4ovJqJdN7CBP3H4AlYRdW1wOo806CD+JfEAwG+oZgJBwe/M7wUoLh/l8AXQiSeplRBC2eYuBFglGYC/ZRVyL7dxvBcPZVBO+P18N9rJK7P0dw7D5vZid4MKDqeuA/Cc47vk8w+KeylmcOMJbgGFtL8EXttkTqjhKL39qWTBC2oj4n+E3Wm+mOR0SkOqkFlWEs+DV847Cb4naCE8dL0hyWiEi1U4LKPGcQDFVfT9BV2N/dv01vSCIi1U9dfCIiEklqQYmISCRlxEUFmzVr5m3btk13GCIiUg2WLFmy0d3zqlouIxJU27ZtKSoqSncYIiJSDcys4tVh4lIXn4iIRJISlIiIRJISlIiIRFJGnIOK57vvvqO4uJjt27enOxRJQHZ2Nq1ataJevXpVLywiQgYnqOLiYho2bEjbtm3Zjws/Sxq4O6WlpRQXF9OuXbt0hyMiGSJju/i2b99O06ZNlZwygJnRtGlTtXZFZL9kbIIClJwyiF4rEdlfGZ2gRESk5srYc1B7mTSperc3dOgBrXb11VdTVFSEu3P88cczbdo0cnNz2bBhAz/96U/59ttvefDBBzn99NOrN14RkRpGLahqNm7cOFasWMHKlStp06YNDz30EADz58+nQ4cOLFu2TMlJRCQBSlAHYdu2bfTu3ZuCggK6dOnCzJkzadSoERCMXPvmm28wM5YvX86vfvUr5syZQ9euXdm2bRuDBg2iS5cu5OfnM27cuDTviYhUh1GjRjFq1Kh0h1Fj1JwuvjSYO3cuLVu25LnnngNg8+bNAAwePJg5c+bQqVMnxo4dS4MGDRgzZgxFRUU89NBDLFmyhM8++4x33nkHgE2bqrx7uYhIraMW1EHIz89n3rx53HzzzSxcuJDGjRsDMHXqVEpKSujYsSMzZ87ca71jjjmG1atXc9111zF37tzyVpeIiHxPCeogHH/88SxZsoT8/HxGjhzJmDFjyudlZWVx6aWX8uSTT+613mGHHcaKFSvo2bMn48ePZ8iQIakMW0QkI6iL7yCUlJRw+OGHM2DAAHJzc5k6dSoffvghxx13HO7Os88+S4cOHfZab+PGjdSvX5+LLrqIY489lkGDBqU+eBGRiKs5CeoAh4UfjLfffpsRI0ZQp04d6tWrx/jx47nqqqv46quvcHcKCgqYMGHCXut99tlnDB48mN27dwPwu9/9LtWhi4hEXs1JUGlw7rnncu655+5R9vrrr8dddtCgQeUtpYKCApYuXZrs8EREMprOQYmISCQpQYmISCQpQYmISCTpHJSIpEx1XzIzakpKgv81eT9TOR5NLSgREYkkJSgREYmkpHXxmVk28CpwSFjPE+5+u5lNA3oAm8NFB7n78oOtb9KS6m1TD/1h1e3YrKws8vPzy6effvppFixYUH7NvTI9e/bkvvvuo7CwkClTpjBu3DjMjN27d3PXXXfRr18/Bg0axCuvvFJ+uaQGDRrwxhtv7FXnsmXLGD9+PH/+85+rYS+/9+CDDzJhwgROOukkHnnkkYTXe/vttxk7dizTpk2r1nhERJJ5DmoHcKa7bzWzesBrZvZ8OG+Euz+RxLpTIicnh+XLE8+txcXF3HXXXSxdupTGjRuzdetWNmzYUD7/3nvv5eKLL97nNu6++25+/etfH3DMlfnTn/7E888/T7t27fZrvfz8fIqLi1mzZg1t2rSp9rhEpPZKWhefB7aGk/XCP09WfZlg/fr1NGzYkNzcXAByc3P3KyFs2bKFlStXUlBQAMCGDRs455xzOOmkk7jmmms4+uij2bhxIwD3338/Xbp0oUuXLjzwwAPl24hXPmzYMFavXk3fvn0ZO3Ys7du3L0+cu3fv5rjjjmPjxo08/vjjdOnShYKCAs4444zybfbp04dHH3304J4cEZEKknoOysyyzGw5sB74u7svCmfdZWYrzWycmR1SybpDzazIzIpiWxlR8s0339C1a1e6du1K//79q1y+oKCA5s2b065dOwYPHsyzzz67x/wRI0aUb++KK67Ya/2ioiK6dOlSPn3HHXdw5plnsnTpUvr378+aNWsAWLJkCVOnTmXRokW89dZbTJ48mWXLllVaPnHiRFq2bMnLL7/ML3/5SwYMGFDezTdv3jwKCgpo1qwZY8aM4YUXXmDFihXMnj27PI7CwkIWLlx4QM+hiEhlkpqg3H2Xu3cFWgHdzawLMBLoAHQDDgdurmTdSe5e6O6FeXl5yQzzgJV18S1fvpxZs2YBYGZxlzUzsrKymDt3Lk888QTHH388N954I6NHjy5f5t577y3fXrzzQGvXriX2uXjttde47LLLAOjVqxeHHXZYeXn//v059NBDyc3N5cILL2ThwoWVllf085//nIcffhiAKVOmMHjwYABOO+00Bg0axOTJk9m1a1f58kcccQQlZeNrRUSqSUpG8bn7JmAB0Mvd14bdfzuAqUD3VMSQKk2bNuXLL7/co+yLL76gWbNmQJCounfvzsiRI3n00Ufj3o6jMjk5OWzfvr182j1+j+n+llfUunVrmjdvzksvvcSiRYs477zzAJg4cSJ33nknn376KV27dqW0tBSA7du3k5OTk/B+iIgkImkJyszyzKxJ+DgHOBt4z8xahGUGXAC8k6wY0qFbt268/vrrfP7550DQLbdjxw5at25NSUnJHheJXb58OUcffXTC2+7YsSMffvhh+fSPf/xjHnvsMQBefPHF8sR4xhln8PTTT/P111+zbds2Zs2axemnn15peTxDhgxhwIABXHLJJWRlZQHw0UcfcfLJJzNmzBiaNWvGp59+CsD777+/R9ejiEh1SOYovhbAdDPLIkiEj7n738zsJTPLAwxYDgyrjsoSGRaeCs2bN+ePf/wj559/Prt37yY3N5cZM2ZQp04dvvvuO2666SZKSkrIzs4mLy+PiRMnlq87YsQI7rzzzvLpxYsXU79+/fLpDh06sHnzZrZs2ULDhg25/fbbufzyy5k5cyY9evSgRYsWNGzYkJNOOolBgwbRvXvQOB0yZAgnnngiQKXlFfXt25fBgweXd++VxffBBx/g7px11lnlgzVefvllevfuXU3PoEjmOvXUu9MdQo1iiXb7pFNhYaEXFRXtUfbuu+/SsWPHNEWUPuPGjaNhw4YMGTKEHTt2kJWVRd26dXnzzTcZPnz4fg1735eioiJuvPHGKgc/7Nixgx49evDaa69Rt+6+v+/U1tdMvleTLwFUW1THpY7MbIm7F1a1nK7Fl2GGDx/O448/DsCaNWu45JJL2L17N/Xr12fy5MnVUsc999zDhAkTEvrB7po1a7jnnnuqTE4iIvtLLShJGb1mohZU5ktlC0rX4hMRkUhSghIRkUhSghIRkUhSghIRkUiqMUOvqvvkayInAmvS7TYSMXHiRBo0aMCVV15Z6TKXXXYZv/3tb2nfvn0KIxORmqjGJKh0qEm320jEsGFV/6Z6+PDh/OEPf6i2Ie8iUnupiy+Fqvt2G6NHj2bgwIGceeaZtG/fvjwpDBw4kGeeeaZ8vSuuuILZs2ezatUqunfvTteuXTnhhBP44IMP2LZtG71796agoIAuXbowc+ZMAG655RY6derECSecwE033VRe33333cfOnTvp1q0bCxYsAGDkyJHceuutAJx++unMmzePnTt3HtyTJSK1nlpQB6HsdhsA7dq1K7+ieWVib7dx1llnceGFF9KnT5/y+bGXOurcufNeP5SteLsNgJUrV/LWW2+xbds2TjzxRHr37s2QIUMYN24c/fr1Y/PmzbzxxhtMnz6dG2+8kRtuuIErrriCb7/9ll27djFnzhxatmzJc889B8DmzZv54osvmDVrFu+99x5mxqZNm/aos27dukybNo2LL76YBx98kLlz57JoUXAnlTp16nDcccexYsUKfvjDHx7AsyoiElAL6iCk+3YbAP369SMnJ4dmzZrxk5/8hMWLF9OjRw8+/PBD1q9fz4wZM7jooouoW7cuP/rRj7j77rv5/e9/zyeffEJOTg75+fnMmzePm2++mYULF9K4cWMaNWpEdnY2Q4YM4amnnqJBgwZ7xdK5c2cGDhxInz59mDJlyh7XDNTtN0SkOihBVbNU3m6jbHvxpgcOHMgjjzzC1KlTyy/4+rOf/YzZs2eTk5PDueeey0svvcTxxx/PkiVLyM/PZ+TIkYwZM4a6deuyePFiLrroIp5++ml69eoVN563336bJk2asG7duj3KdfsNEakOSlDVLJW32wB45pln2L59O6WlpSxYsIBu3boBwVXLy27p3rlzZwBWr17NMcccw/XXX0/fvn1ZuXIlJSUlNGjQgAEDBnDTTTexdOlStm7dyubNmzn//PN54IEH4g4EeeqppygtLeXVV1/l+uuv36Mb8P333y+vU0TkQNWYc1DVcX2o6pDK220AdO/end69e7NmzRp+85vf0LJly/I4OnbsyAUXXFC+/syZM/nrX/9KvXr1OPLII7ntttv4xz/+wYgRI6hTpw716tVjwoQJbNmyhX79+rF9+3bcnXHjxu2xjxs3buSWW25h/vz5tG7dmmuvvZYbbriB6dOns27dOnJycmjRokVSnl8RqT10sdgME3u7jdGjR5Obm1s+yi7W119/TX5+fvmQ9lTG16hRI66++uq95tXW10y+p4vFZj5dLFYqNXz4cA455JB9LjNv3jw6dOjAddddl9LkBNCkSROuuuqqlNYpIjVTjeniqy2ys7MZOHAgwB4jAGOdffbZrFmzJoVRfS/2DrwiIgcjo1tQmdA9KQG9ViKyvzI2QWVnZ1NaWqoPvgzg7pSWlpKdnZ3uUEQkg2RsF1+rVq0oLi7e41p2El3Z2dm0atUq3WGISAbJ2ARVr169/bqOnYiIZJaM7eITEZGaLWkJysyyzWyxma0ws1VmdkdY3s7MFpnZB2Y208zqV7UtERGpfZLZgtoBnOnuBUBXoJeZnQL8Hhjn7u2BL4G9f9EpIiK1XtISlAe2hpP1wj8HzgSeCMunAxfEWV1ERGq5pJ6DMrMsM1sOrAf+DnwEbHL3srvZFQNHVbLuUDMrMrMijdQTEal9kpqg3H2Xu3cFWgHdgXgXYov7QyZ3n+Tuhe5eWPEeSCIiUvOlZBSfu28CFgCnAE3MrGx4eytAd7YTEZG9JHMUX56ZNQkf5wBnA+8CLwMXh4tdBTyTrBhERCRzJfOHui2A6WaWRZAIH3P3v5nZ/wCPmtmdwDLgL0mMQUREMlTSEpS7rwROjFO+muB8lIiISKV0JQkREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYkkJSgREYmkpCUoM2ttZi+b2btmtsrMbgjLR5vZZ2a2PPw7P1kxiIhI5qqbxG3vBH7p7kvNrCGwxMz+Hs4b5+73JbFuERHJcElLUO6+FlgbPt5iZu8CRyWrPhERqVlScg7KzNoCJwKLwqJrzWylmU0xs8MqWWeomRWZWdGGDRtSEaaIiERI0hOUmeUCTwK/cPevgAnAsUBXghbW2Hjrufskdy9098K8vLxkhykiIhGT1ARlZvUIktMj7v4UgLuvc/dd7r4bmAx0T2YMIiKSmZI5is+AvwDvuvv9MeUtYhbrD7yTrBhERCRzJXMU32nAQOBtM1selo0CLjezroADHwPXJDEGERHJUMkcxfcaYHFmzUlWnSIiUnPoShIiIhJJSlAiIhJJSlAiIhJJSlAiIhJJSlAiIhJJSlAiIhJJSlAiIhJJSlAiIhJJCSUoMzvUzOqEj483s77hdfZERESSItEW1KtAtpkdBcwHBgPTkhWUiIhIognK3P1r4ELgP9y9P9ApeWGJiEhtl3CCMrMfAVcAz4VlybzQrIiI1HKJJqhfACOBWe6+ysyOAV5OXlgiIlLbJdQKcvdXgFdiplcD1ycrKBERkX0mKDN7luC+TXG5e99qj0hERISqW1D3hf8vBI4E/hpOX05ws0EREZGk2GeCCrv2MLPfuvsZMbOeNbNXkxqZiIjUaokOksgLB0YAYGbtgLzkhCQiIpL4UPEbgQVmtjqcbgtck5SIRERESHwU31wzaw90CIvec/cdyQtLRERqu/35se0PCVpOdYECM8PdH05KVCIiUusllKDM7L+AY4HlwK6w2AElKBERSYpEW1CFQCd3r/Q3URWZWWuCBHYksBuY5O5/NLPDgZkErbGPgUvc/cv9CVpERGq+REfxvUOQaPbHTuCX7t4ROAX4NzPrBNwCzHf39gRXRr9lP7crIiK1QKItqGbA/5jZYqB8cMS+riTh7muBteHjLWb2LnAU0A/oGS42HVgA3Ly/gYuISM2WaIIafTCVmFlb4ERgEdA8TF64+1ozO6KSdYYCQwHatGlzMNWLiEgGSqiLL7yixHtAw/Dv3bKrTFTFzHKBJ4FfuPtXiQbm7pPcvdDdC/Py9JtgEZHaJtFbvl8CLAb+BbgEWGRmFyewXj2C5PSIuz8VFq8zsxbh/BbA+gMJXEREarZEu/huBbq5+3oAM8sD5gFPVLaCmRnwF4LW1v0xs2YDVwH3hP+fOYC4RUSkhks0QdUpS06hUqpufZ0GDATeNrPlYdkogsT0mJldDawhaJWJiIjsIdEENdfMXgBmhNOXAs/vawV3fw2wSmaflWC9IiJSSyV6Lb4RZnYh8GOCpDPJ3WclNTIREanVEr3UUTtgTtlABzPLMbO27v5xMoMTEZHaK9ErSTxOcLmiMrvCMhERkaRINEHVdfdvyybCx/WTE5KIiEjiCWqDmZVf1sjM+gEbkxOSiIhI4qP4hgGPmNl4gttsFANXJi0qERGp9RIdxfcRcEp42SJz9y3JDUtERGq7RC911NzM/gI8Hl6ZvFP4Q1sREZGkSPQc1DTgBaBlOP0+8ItkBCQiIgKJJ6hm7v4Y4VBzd9/J97d+FxERqXaJJqhtZtaUYIAEZnYKsDlpUYmISK2X6Ci+fye4CvmxZvY6kAdUebsNERGRA7XPFpSZdTOzI919KdCD4GrkO4AXCYaai4iIJEVVXXz/CZRdQeJUgvtCjQe+BCYlMS4REanlquriy3L3L8LHlxJcxfxJ4MmYezyJiIhUu6paUFlmVpbEzgJeipmX6PkrERGR/VZVkpkBvGJmG4FvgIUAZnYcGsUnIiJJtM8E5e53mdl8oAXwort7OKsOcF2ygxMRkdqrym46d38rTtn7yQlHREQkkOgPdUVERFJKCUpERCKZolZYAAAFQ0lEQVRJCUpERCIpaQnKzKaY2XozeyembLSZfWZmy8O/85NVv4iIZLZktqCmAb3ilI9z967h35wk1i8iIhksaQnK3V8FvqhyQRERkTjScQ7qWjNbGXYBHlbZQmY21MyKzKxow4YNqYxPREQiINUJagJwLNAVWAuMrWxBd5/k7oXuXpiXl5eq+EREJCJSmqDcfZ2773L33cBkoHsq6xcRkcyR0gRlZi1iJvsD71S2rIiI1G5JuyK5mc0AegLNzKwYuB3oaWZdCW4d/zFwTbLqFxGRzJa0BOXul8cp/kuy6qvNRo0aBcDdd9+d5khERKqPriQhIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRpAQlIiKRlLSLxUbOpEnpjiB5SkqC/zV5HwGGDk13BCKSQmpBiYhIJClBiYhIJClBiYhIJClBiYhIJClBiYhIJClBiYhIJClBiUTEqFGjGDVqVLrDEIkMJSgREYkkJSgREYmkpCUoM5tiZuvN7J2YssPN7O9m9kH4/7Bk1S8iIpktmZc6mgY8BDwcU3YLMN/d7zGzW8Lpm5MYg9Qgk5bU7Es5lWwJLllVs/dTl6uSxCWtBeXurwJfVCjuB0wPH08HLkhW/SIiktlSfbHY5u6+FsDd15rZEZUtaGZDCb9utWnTJkXhZaa7Tz013SGIiFS7yA6ScPdJ7l7o7oV5eXnpDkdERFIs1QlqnZm1AAj/r09x/SIikiFS3cU3G7gKuCf8/0yK6xeJrFMHqatWJFYyh5nPAN4EfmBmxWZ2NUFiOsfMPgDOCadFRET2krQWlLtfXsmss5JVp4iI1ByRHSQhIiK1mxKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEUt10VGpmHwNbgF3ATncvTEccIiISXWlJUKGfuPvGNNYvIiIRpi4+ERGJpHQlKAdeNLMlZjY03gJmNtTMisysaMOGDSkOT0RE0i1dCeo0dz8JOA/4NzM7o+IC7j7J3QvdvTAvLy/1EYqISFqlJUG5e0n4fz0wC+iejjhERCS6Up6gzOxQM2tY9hj4f8A7qY5DRESiLR2j+JoDs8ysrP7/dve5aYhDREQiLOUJyt1XAwWprldERDKLhpmLiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkKUGJiEgkpSVBmVkvM/unmX1oZrekIwYREYm2lCcoM8sCxgPnAZ2Ay82sU6rjEBGRaEtHC6o78KG7r3b3b4FHgX5piENERCKsbhrqPAr4NGa6GDi54kJmNhQYGk5uNbN/piA2EUmqa9IdgByka6rnJTw6kYXSkaAsTpnvVeA+CZiU/HBERCSK0tHFVwy0jpluBZSkIQ4REYmwdCSofwDtzaydmdUHLgNmpyEOERGJsJR38bn7TjO7FngByAKmuPuqVMchIiLRZu57nf4RERFJO11JQkREIkkJSkREIkkJSkREIkkJSkREIkkJSkREIkkJSkREIkkJSkREIun/AOtquoVSl2AIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "from collections import namedtuple\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "bar_width = 0.35\n",
    "opacity = 0.4\n",
    "error_config = {'ecolor': '0.3'}\n",
    "\n",
    "rects1 = ax.bar(0, s3fs_timings.average, bar_width,\n",
    "                alpha=opacity, color='r',\n",
    "                yerr=s3fs_timings.stdev, error_kw=error_config,\n",
    "                label='s3fs')\n",
    "\n",
    "rects2 = ax.bar(bar_width, goofys_timings.average, bar_width,\n",
    "                alpha=opacity, color='g',\n",
    "                yerr=goofys_timings.stdev, error_kw=error_config,\n",
    "                label='FUSE (goofys)')\n",
    "\n",
    "\n",
    "rects3 = ax.bar(2 * bar_width, pysssix_timings.average, bar_width,\n",
    "                alpha=opacity, color='b',\n",
    "                yerr=pysssix_timings.stdev, error_kw=error_config,\n",
    "                label='FUSE (pysssix)')\n",
    "\n",
    "# ax.set_xlabel('Method')\n",
    "ax.set_ylabel('Seconds')\n",
    "ax.set_title('Taking a mean of an ~80GB zarr with 100 workers')\n",
    "ax.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)\n",
    "ax.legend()\n",
    "\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}