example_qaqc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example -- QA/QC masking\n",
    "\n",
    "This uses project data from https://github.com/ceholden/glance-workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import intake\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "\n",
    "import stems.masking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# catalog.yml and this notebook are currently in root of repo\n",
    "cat = intake.open_catalog('./catalog.yml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "src = cat.landsat(column=66, row=36, continent='NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/projectnb/measures/users/ceholden/module_conda/envs/glance/lib/python3.7/site-packages/xarray/backends/api.py:783: FutureWarning: In xarray version 0.13 `auto_combine` will be deprecated.\n",
      "  coords=coords)\n",
      "/projectnb/measures/users/ceholden/module_conda/envs/glance/lib/python3.7/site-packages/xarray/backends/api.py:783: FutureWarning: Also `open_mfdataset` will no longer accept a `concat_dim` argument.\n",
      "To get equivalent behaviour from now on please use the new\n",
      "`combine_nested` function instead (or the `combine='nested'` option to\n",
      "`open_mfdataset`).The datasets supplied have global dimension coordinates. You may want\n",
      "to use the new `combine_by_coords` function (or the\n",
      "`combine='by_coords'` option to `open_mfdataset` to order the datasets\n",
      "before concatenation. Alternatively, to continue concatenating based\n",
      "on the order the datasets are supplied in in future, please use the\n",
      "new `combine_nested` function (or the `combine='nested'` option to\n",
      "open_mfdataset).\n",
      "  coords=coords)\n"
     ]
    }
   ],
   "source": [
    "ds = src.to_dask()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = ds.isel(time=slice(0, 100), x=slice(0, 250), y=slice(0, 250))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mSignature:\u001b[0m \u001b[0mstems\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasking\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbitpack_to_coding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbitpack\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbitinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mDocstring:\u001b[0m\n",
       "Unpack a bipacked QA/QC band to some coding (e.g. CFMask)\n",
       "\n",
       "Parameters\n",
       "----------\n",
       "bitpack : np.ndarray, dask.array.Array, or xr.DataArray\n",
       "    Bitpacked data\n",
       "bitinfo : Dict[int, Sequence[Tuple[Int, Int, Int]]\n",
       "    A dict mapping output codes to bit unpacking info(s)\n",
       "    (offsets, widths, and values) for use in unpacking. Should be ordered\n",
       "    in order of preference with values listed first possibly overwriting\n",
       "    later ones\n",
       "fill : int, float, etc, optional\n",
       "    Fill value to initialize array with (e.g., your \"clear\" value, since\n",
       "    qa/qc bands usually indicate issues with data)\n",
       "dtype : np.dtype, optional\n",
       "    Output NumPy datatype. If ``None``, output will be same\n",
       "    datatype as input ``bitpack``\n",
       "\n",
       "Returns\n",
       "-------\n",
       "array\n",
       "    CFmask look-alike array\n",
       "\u001b[0;31mFile:\u001b[0m      /projectnb/measures/users/ceholden/GLANCE/stems/stems/masking.py\n",
       "\u001b[0;31mType:\u001b[0m      function\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "?stems.masking.bitpack_to_coding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CFMASK LOOK ALIKE\n",
    "bitinfo = {\n",
    "    0: [(0, 1, 1)],\n",
    "    1: [(1, 1, 1)],\n",
    "    2: [(2, 1, 1)],\n",
    "    3: [(3, 1, 1)],\n",
    "    4: [(4, 1, 1)],\n",
    "    5: [(5, 1, 1)],\n",
    "    8: [(8, 2, 2)],\n",
    "    10: [(10, 1, 1)]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FINAL MASK\n",
    "bitinfo = {\n",
    "    0: [\n",
    "        (0, 1, 1),\n",
    "        (3, ),\n",
    "        (4, ),\n",
    "        (5, ),\n",
    "        (8, 2, 2),\n",
    "        (10, )\n",
    "    ],\n",
    "    1: [(1, ),\n",
    "        (2, )\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfmask = stems.masking.bitpack_to_coding(\n",
    "    sub['pixel_qa'],\n",
    "    bitinfo,\n",
    "    fill=0,\n",
    "    dtype=np.bool\n",
    ")\n",
    "cfmask.name = 'cfmask_clone'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr>\n",
       "<td>\n",
       "<table>  <thead>    <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><th> Bytes </th><td> 6.25 MB </td> <td> 6.25 MB </td></tr>\n",
       "    <tr><th> Shape </th><td> (100, 250, 250) </td> <td> (100, 250, 250) </td></tr>\n",
       "    <tr><th> Count </th><td> 8012 Tasks </td><td> 1 Chunks </td></tr>\n",
       "    <tr><th> Type </th><td> bool </td><td> numpy.ndarray </td></tr>\n",
       "  </tbody></table>\n",
       "</td>\n",
       "<td>\n",
       "<svg width=\"208\" height=\"198\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"38\" y2=\"28\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"10\" y1=\"120\" x2=\"38\" y2=\"148\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"120\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"148\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"10.000000,0.000000 38.235294,28.235294 38.235294,148.235294 10.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"130\" y2=\"0\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"38\" y1=\"28\" x2=\"158\" y2=\"28\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"38\" y2=\"28\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"130\" y1=\"0\" x2=\"158\" y2=\"28\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"10.000000,0.000000 130.000000,0.000000 158.235294,28.235294 38.235294,28.235294\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"38\" y1=\"28\" x2=\"158\" y2=\"28\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"38\" y1=\"148\" x2=\"158\" y2=\"148\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"148\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"158\" y1=\"28\" x2=\"158\" y2=\"148\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"38.235294,28.235294 158.235294,28.235294 158.235294,148.235294 38.235294,148.235294\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Text -->\n",
       "  <text x=\"98.235294\" y=\"168.235294\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >250</text>\n",
       "  <text x=\"178.235294\" y=\"88.235294\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,178.235294,88.235294)\">250</text>\n",
       "  <text x=\"14.117647\" y=\"154.117647\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,14.117647,154.117647)\">100</text>\n",
       "</svg>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "dask.array<_bitpack_to_coding_nparray, shape=(100, 250, 250), dtype=bool, chunksize=(100, 250, 250)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cfmask.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "swir1 = sub['swir1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.DataArray 'swir1' ()>\n",
       "array(1835.0369, dtype=float32)\n",
       "Coordinates:\n",
       "    crs      int64 0\n",
       "    column   int64 66\n",
       "    row      int64 36"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swir1.mean().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.DataArray 'swir1' ()>\n",
       "array(1575.5331, dtype=float32)\n",
       "Coordinates:\n",
       "    crs      int64 0\n",
       "    column   int64 66\n",
       "    row      int64 36"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swir1.where(cfmask).mean().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}