Quantizing and compressing with zarr.

zarrQuantization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "70ba227f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import zarr\n",
    "\n",
    "from numcodecs import Blosc,FixedScaleOffset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5e1d2f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make an array\n",
    "x = np.random.random( [64,64] ).astype('float64')\n",
    "\n",
    "store = zarr.DirectoryStore('quantized.zarr')\n",
    "compressor = Blosc(cname='zstd', clevel=3, shuffle=Blosc.BITSHUFFLE)\n",
    "\n",
    "# compress and store directly\n",
    "zr = zarr.array( x, store=store, path='sansFilter', compressor=compressor, dtype='float64' )\n",
    "\n",
    "# scale store as a byte\n",
    "filters = [ FixedScaleOffset( scale=255, offset=0, dtype='float64', astype='uint8') ]\n",
    "zf = zarr.array( x, store=store, path='withFilter', filters=filters, compressor=compressor, dtype='float64' )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "243c3666",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sans filter:  0.0\n",
      "with filter:  0.001959563916785645\n"
     ]
    }
   ],
   "source": [
    "d = zarr.load('quantized.zarr')\n",
    "yr = d['sansFilter']\n",
    "yf = d['withFilter']\n",
    "\n",
    "print( 'sans filter: ', (x - yr).max() )\n",
    "print( 'with filter: ', (x - yf).max() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "6b9a4ab0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"zarr-info\"><tbody><tr><th style=\"text-align: left\">Name</th><td style=\"text-align: left\">/sansFilter</td></tr><tr><th style=\"text-align: left\">Type</th><td style=\"text-align: left\">zarr.core.Array</td></tr><tr><th style=\"text-align: left\">Data type</th><td style=\"text-align: left\">float64</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(64, 64)</td></tr><tr><th style=\"text-align: left\">Chunk shape</th><td style=\"text-align: left\">(64, 64)</td></tr><tr><th style=\"text-align: left\">Order</th><td style=\"text-align: left\">C</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">False</td></tr><tr><th style=\"text-align: left\">Compressor</th><td style=\"text-align: left\">Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE, blocksize=0)</td></tr><tr><th style=\"text-align: left\">Store type</th><td style=\"text-align: left\">zarr.storage.DirectoryStore</td></tr><tr><th style=\"text-align: left\">No. bytes</th><td style=\"text-align: left\">32768 (32.0K)</td></tr><tr><th style=\"text-align: left\">No. bytes stored</th><td style=\"text-align: left\">28660 (28.0K)</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">1.1</td></tr><tr><th style=\"text-align: left\">Chunks initialized</th><td style=\"text-align: left\">1/1</td></tr></tbody></table>"
      ],
      "text/plain": [
       "Name               : /sansFilter\n",
       "Type               : zarr.core.Array\n",
       "Data type          : float64\n",
       "Shape              : (64, 64)\n",
       "Chunk shape        : (64, 64)\n",
       "Order              : C\n",
       "Read-only          : False\n",
       "Compressor         : Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE,\n",
       "                   : blocksize=0)\n",
       "Store type         : zarr.storage.DirectoryStore\n",
       "No. bytes          : 32768 (32.0K)\n",
       "No. bytes stored   : 28660 (28.0K)\n",
       "Storage ratio      : 1.1\n",
       "Chunks initialized : 1/1"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zr.info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "688dee17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"zarr-info\"><tbody><tr><th style=\"text-align: left\">Name</th><td style=\"text-align: left\">/withFilter</td></tr><tr><th style=\"text-align: left\">Type</th><td style=\"text-align: left\">zarr.core.Array</td></tr><tr><th style=\"text-align: left\">Data type</th><td style=\"text-align: left\">float64</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(64, 64)</td></tr><tr><th style=\"text-align: left\">Chunk shape</th><td style=\"text-align: left\">(64, 64)</td></tr><tr><th style=\"text-align: left\">Order</th><td style=\"text-align: left\">C</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">False</td></tr><tr><th style=\"text-align: left\">Filter [0]</th><td style=\"text-align: left\">FixedScaleOffset(scale=255, offset=0, dtype='<f8', astype='|u1')</td></tr><tr><th style=\"text-align: left\">Compressor</th><td style=\"text-align: left\">Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE, blocksize=0)</td></tr><tr><th style=\"text-align: left\">Store type</th><td style=\"text-align: left\">zarr.storage.DirectoryStore</td></tr><tr><th style=\"text-align: left\">No. bytes</th><td style=\"text-align: left\">32768 (32.0K)</td></tr><tr><th style=\"text-align: left\">No. bytes stored</th><td style=\"text-align: left\">4618 (4.5K)</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">7.1</td></tr><tr><th style=\"text-align: left\">Chunks initialized</th><td style=\"text-align: left\">1/1</td></tr></tbody></table>"
      ],
      "text/plain": [
       "Name               : /withFilter\n",
       "Type               : zarr.core.Array\n",
       "Data type          : float64\n",
       "Shape              : (64, 64)\n",
       "Chunk shape        : (64, 64)\n",
       "Order              : C\n",
       "Read-only          : False\n",
       "Filter [0]         : FixedScaleOffset(scale=255, offset=0, dtype='<f8',\n",
       "                   : astype='|u1')\n",
       "Compressor         : Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE,\n",
       "                   : blocksize=0)\n",
       "Store type         : zarr.storage.DirectoryStore\n",
       "No. bytes          : 32768 (32.0K)\n",
       "No. bytes stored   : 4618 (4.5K)\n",
       "Storage ratio      : 7.1\n",
       "Chunks initialized : 1/1"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zf.info"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}