Last active
April 9, 2024 01:21
-
-
Save ctuguinay/2a92f52129b3045147d66fbf2f071ec6 to your computer and use it in GitHub Desktop.
echopype_issue_1260_investigate_null.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# Import libraries\n", | |
| "import echopype as ep\n", | |
| "import numpy as np" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "In Echopype Code I set print statements in merge_save function of echopype/convert/set_groups_ek80.py. This is to show where exactly I am capturing the output I will show below:\n", | |
| "\n", | |
| " @staticmethod\n", | |
| " def merge_save(ds_combine: List[xr.Dataset], ds_invariant: xr.Dataset) -> xr.Dataset:\n", | |
| " \"\"\"Merge data from all complex or all power/angle channels\"\"\"\n", | |
| " # Combine all channels into one Dataset\n", | |
| " print(\"Channel 1 Ping Time Pre Combine:\", ds_combine[0][\"ping_time\"])\n", | |
| " print(\"Channel 2 Ping Time Pre Combine:\", ds_combine[1][\"ping_time\"])\n", | |
| " print(\"Channel 3 Ping Time Pre Combine:\", ds_combine[2][\"ping_time\"])\n", | |
| " ds_combine = xr.concat(ds_combine, dim=\"channel\")\n", | |
| " ds_combine = xr.merge(\n", | |
| " [ds_invariant, ds_combine], combine_attrs=\"override\"\n", | |
| " ) # override keeps the Dataset attributes\n", | |
| " return set_time_encodings(ds_combine)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Channel 1 Ping Time Pre Combine: <xarray.DataArray 'ping_time' (ping_time: 17866)>\n", | |
| "array(['2024-03-04T05:27:28.251616768', '2024-03-04T05:27:30.251905024',\n", | |
| " '2024-03-04T05:27:32.251759104', ..., '2024-03-04T15:33:25.252228096',\n", | |
| " '2024-03-04T15:33:27.254448128', '2024-03-04T15:33:29.263218176'],\n", | |
| " dtype='datetime64[ns]')\n", | |
| "Coordinates:\n", | |
| " * ping_time (ping_time) datetime64[ns] 2024-03-04T05:27:28.251616768 ... 2...\n", | |
| "Attributes:\n", | |
| " long_name: Timestamp of each ping\n", | |
| " standard_name: time\n", | |
| " axis: T\n", | |
| "Channel 2 Ping Time Pre Combine: <xarray.DataArray 'ping_time' (ping_time: 8931)>\n", | |
| "array(['2024-03-04T05:27:29.442363904', '2024-03-04T05:27:33.442920960',\n", | |
| " '2024-03-04T05:27:37.441874944', ..., '2024-03-04T15:33:18.443037184',\n", | |
| " '2024-03-04T15:33:22.444801024', '2024-03-04T15:33:26.442637824'],\n", | |
| " dtype='datetime64[ns]')\n", | |
| "Coordinates:\n", | |
| " * ping_time (ping_time) datetime64[ns] 2024-03-04T05:27:29.442363904 ... 2...\n", | |
| "Attributes:\n", | |
| " long_name: Timestamp of each ping\n", | |
| " standard_name: time\n", | |
| " axis: T\n", | |
| "Channel 3 Ping Time Pre Combine: <xarray.DataArray 'ping_time' (ping_time: 8934)>\n", | |
| "array(['2024-03-04T05:27:27.453144064', '2024-03-04T05:27:31.442492928',\n", | |
| " '2024-03-04T05:27:35.442065920', ..., '2024-03-04T15:33:20.442040832',\n", | |
| " '2024-03-04T15:33:24.444175872', '2024-03-04T15:33:28.441876992'],\n", | |
| " dtype='datetime64[ns]')\n", | |
| "Coordinates:\n", | |
| " * ping_time (ping_time) datetime64[ns] 2024-03-04T05:27:27.453144064 ... 2...\n", | |
| "Attributes:\n", | |
| " long_name: Timestamp of each ping\n", | |
| " standard_name: time\n", | |
| " axis: T\n" | |
| ] | |
| }, | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": [ | |
| "/home/exouser/miniforge3/envs/echopype/lib/python3.9/site-packages/xarray/core/duck_array_ops.py:187: RuntimeWarning: invalid value encountered in cast\n", | |
| " return data.astype(dtype, **kwargs)\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "# Convert Raw (from https://rawdata.oceanobservatories.org/files/CE02SHBP/MJ01C/ZPLSCB101_10.33.13.7/2024/03/04/)\n", | |
| "ed = ep.open_raw(\"/home/exouser/echopype/ZPLSCB101-D20240304-T052727.raw\", sonar_model=\"EK80\")" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "The ping time values for the 3 channel arrays are inconsistent with each other:\n", | |
| "\n", | |
| "(Also, note that these ping time values come from the parser_obj in SetGroupsEK80)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "The misalignment of the ping time values causes the merging of these channel to have backscatter data that contains padded NaNs; however, there still are non-NaN values here:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n", | |
| "<defs>\n", | |
| "<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n", | |
| "<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n", | |
| "<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n", | |
| "<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n", | |
| "</symbol>\n", | |
| "<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n", | |
| "<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n", | |
| "<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n", | |
| "<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n", | |
| "<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n", | |
| "</symbol>\n", | |
| "</defs>\n", | |
| "</svg>\n", | |
| "<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n", | |
| " *\n", | |
| " */\n", | |
| "\n", | |
| ":root {\n", | |
| " --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n", | |
| " --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n", | |
| " --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n", | |
| " --xr-border-color: var(--jp-border-color2, #e0e0e0);\n", | |
| " --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n", | |
| " --xr-background-color: var(--jp-layout-color0, white);\n", | |
| " --xr-background-color-row-even: var(--jp-layout-color1, white);\n", | |
| " --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n", | |
| "}\n", | |
| "\n", | |
| "html[theme=dark],\n", | |
| "body[data-theme=dark],\n", | |
| "body.vscode-dark {\n", | |
| " --xr-font-color0: rgba(255, 255, 255, 1);\n", | |
| " --xr-font-color2: rgba(255, 255, 255, 0.54);\n", | |
| " --xr-font-color3: rgba(255, 255, 255, 0.38);\n", | |
| " --xr-border-color: #1F1F1F;\n", | |
| " --xr-disabled-color: #515151;\n", | |
| " --xr-background-color: #111111;\n", | |
| " --xr-background-color-row-even: #111111;\n", | |
| " --xr-background-color-row-odd: #313131;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-wrap {\n", | |
| " display: block !important;\n", | |
| " min-width: 300px;\n", | |
| " max-width: 700px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-text-repr-fallback {\n", | |
| " /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n", | |
| " display: none;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-header {\n", | |
| " padding-top: 6px;\n", | |
| " padding-bottom: 6px;\n", | |
| " margin-bottom: 4px;\n", | |
| " border-bottom: solid 1px var(--xr-border-color);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-header > div,\n", | |
| ".xr-header > ul {\n", | |
| " display: inline;\n", | |
| " margin-top: 0;\n", | |
| " margin-bottom: 0;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-obj-type,\n", | |
| ".xr-array-name {\n", | |
| " margin-left: 2px;\n", | |
| " margin-right: 10px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-obj-type {\n", | |
| " color: var(--xr-font-color2);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-sections {\n", | |
| " padding-left: 0 !important;\n", | |
| " display: grid;\n", | |
| " grid-template-columns: 150px auto auto 1fr 20px 20px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-item {\n", | |
| " display: contents;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-item input {\n", | |
| " display: none;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-item input + label {\n", | |
| " color: var(--xr-disabled-color);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-item input:enabled + label {\n", | |
| " cursor: pointer;\n", | |
| " color: var(--xr-font-color2);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-item input:enabled + label:hover {\n", | |
| " color: var(--xr-font-color0);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary {\n", | |
| " grid-column: 1;\n", | |
| " color: var(--xr-font-color2);\n", | |
| " font-weight: 500;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary > span {\n", | |
| " display: inline-block;\n", | |
| " padding-left: 0.5em;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary-in:disabled + label {\n", | |
| " color: var(--xr-font-color2);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary-in + label:before {\n", | |
| " display: inline-block;\n", | |
| " content: '►';\n", | |
| " font-size: 11px;\n", | |
| " width: 15px;\n", | |
| " text-align: center;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary-in:disabled + label:before {\n", | |
| " color: var(--xr-disabled-color);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary-in:checked + label:before {\n", | |
| " content: '▼';\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary-in:checked + label > span {\n", | |
| " display: none;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary,\n", | |
| ".xr-section-inline-details {\n", | |
| " padding-top: 4px;\n", | |
| " padding-bottom: 4px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-inline-details {\n", | |
| " grid-column: 2 / -1;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-details {\n", | |
| " display: none;\n", | |
| " grid-column: 1 / -1;\n", | |
| " margin-bottom: 5px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-section-summary-in:checked ~ .xr-section-details {\n", | |
| " display: contents;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-array-wrap {\n", | |
| " grid-column: 1 / -1;\n", | |
| " display: grid;\n", | |
| " grid-template-columns: 20px auto;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-array-wrap > label {\n", | |
| " grid-column: 1;\n", | |
| " vertical-align: top;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-preview {\n", | |
| " color: var(--xr-font-color3);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-array-preview,\n", | |
| ".xr-array-data {\n", | |
| " padding: 0 5px !important;\n", | |
| " grid-column: 2;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-array-data,\n", | |
| ".xr-array-in:checked ~ .xr-array-preview {\n", | |
| " display: none;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-array-in:checked ~ .xr-array-data,\n", | |
| ".xr-array-preview {\n", | |
| " display: inline-block;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-dim-list {\n", | |
| " display: inline-block !important;\n", | |
| " list-style: none;\n", | |
| " padding: 0 !important;\n", | |
| " margin: 0;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-dim-list li {\n", | |
| " display: inline-block;\n", | |
| " padding: 0;\n", | |
| " margin: 0;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-dim-list:before {\n", | |
| " content: '(';\n", | |
| "}\n", | |
| "\n", | |
| ".xr-dim-list:after {\n", | |
| " content: ')';\n", | |
| "}\n", | |
| "\n", | |
| ".xr-dim-list li:not(:last-child):after {\n", | |
| " content: ',';\n", | |
| " padding-right: 5px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-has-index {\n", | |
| " font-weight: bold;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-list,\n", | |
| ".xr-var-item {\n", | |
| " display: contents;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-item > div,\n", | |
| ".xr-var-item label,\n", | |
| ".xr-var-item > .xr-var-name span {\n", | |
| " background-color: var(--xr-background-color-row-even);\n", | |
| " margin-bottom: 0;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-item > .xr-var-name:hover span {\n", | |
| " padding-right: 5px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-list > li:nth-child(odd) > div,\n", | |
| ".xr-var-list > li:nth-child(odd) > label,\n", | |
| ".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n", | |
| " background-color: var(--xr-background-color-row-odd);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-name {\n", | |
| " grid-column: 1;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-dims {\n", | |
| " grid-column: 2;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-dtype {\n", | |
| " grid-column: 3;\n", | |
| " text-align: right;\n", | |
| " color: var(--xr-font-color2);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-preview {\n", | |
| " grid-column: 4;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-index-preview {\n", | |
| " grid-column: 2 / 5;\n", | |
| " color: var(--xr-font-color2);\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-name,\n", | |
| ".xr-var-dims,\n", | |
| ".xr-var-dtype,\n", | |
| ".xr-preview,\n", | |
| ".xr-attrs dt {\n", | |
| " white-space: nowrap;\n", | |
| " overflow: hidden;\n", | |
| " text-overflow: ellipsis;\n", | |
| " padding-right: 10px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-name:hover,\n", | |
| ".xr-var-dims:hover,\n", | |
| ".xr-var-dtype:hover,\n", | |
| ".xr-attrs dt:hover {\n", | |
| " overflow: visible;\n", | |
| " width: auto;\n", | |
| " z-index: 1;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-attrs,\n", | |
| ".xr-var-data,\n", | |
| ".xr-index-data {\n", | |
| " display: none;\n", | |
| " background-color: var(--xr-background-color) !important;\n", | |
| " padding-bottom: 5px !important;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-attrs-in:checked ~ .xr-var-attrs,\n", | |
| ".xr-var-data-in:checked ~ .xr-var-data,\n", | |
| ".xr-index-data-in:checked ~ .xr-index-data {\n", | |
| " display: block;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-data > table {\n", | |
| " float: right;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-var-name span,\n", | |
| ".xr-var-data,\n", | |
| ".xr-index-name div,\n", | |
| ".xr-index-data,\n", | |
| ".xr-attrs {\n", | |
| " padding-left: 25px !important;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-attrs,\n", | |
| ".xr-var-attrs,\n", | |
| ".xr-var-data,\n", | |
| ".xr-index-data {\n", | |
| " grid-column: 1 / -1;\n", | |
| "}\n", | |
| "\n", | |
| "dl.xr-attrs {\n", | |
| " padding: 0;\n", | |
| " margin: 0;\n", | |
| " display: grid;\n", | |
| " grid-template-columns: 125px auto;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-attrs dt,\n", | |
| ".xr-attrs dd {\n", | |
| " padding: 0;\n", | |
| " margin: 0;\n", | |
| " float: left;\n", | |
| " padding-right: 10px;\n", | |
| " width: auto;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-attrs dt {\n", | |
| " font-weight: normal;\n", | |
| " grid-column: 1;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-attrs dt:hover span {\n", | |
| " display: inline-block;\n", | |
| " background: var(--xr-background-color);\n", | |
| " padding-right: 10px;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-attrs dd {\n", | |
| " grid-column: 2;\n", | |
| " white-space: pre-wrap;\n", | |
| " word-break: break-all;\n", | |
| "}\n", | |
| "\n", | |
| ".xr-icon-database,\n", | |
| ".xr-icon-file-text2,\n", | |
| ".xr-no-icon {\n", | |
| " display: inline-block;\n", | |
| " vertical-align: middle;\n", | |
| " width: 1em;\n", | |
| " height: 1.5em !important;\n", | |
| " stroke-width: 0;\n", | |
| " stroke: currentColor;\n", | |
| " fill: currentColor;\n", | |
| "}\n", | |
| "</style><pre class='xr-text-repr-fallback'><xarray.DataArray 'backscatter_r' (channel: 3, ping_time: 35731,\n", | |
| " range_sample: 4190)>\n", | |
| "array([[[ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 8.725166 , 14.063745 , 17.450333 , ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " ...,\n", | |
| " [ 8.666371 , 14.040227 , 17.462091 , ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 8.666371 , 14.051986 , 17.450333 , ..., nan,\n", | |
| " nan, nan]],\n", | |
| "\n", | |
| " [[ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 7.1141853, 10.453737 , 12.488041 , ..., nan,\n", | |
| " nan, nan],\n", | |
| "...\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan]],\n", | |
| "\n", | |
| " [[ 7.0318723, 9.677644 , 11.500286 , ..., -132.9706 ,\n", | |
| " -133.64085 , -134.6286 ],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " ...,\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 6.9495597, 9.595331 , 11.4179735, ..., -141.42531 ,\n", | |
| " -142.77759 , -143.80061 ],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan]]], dtype=float32)\n", | |
| "Coordinates:\n", | |
| " * channel (channel) <U36 'WBT Tube 267819-7 ES38-18DK-Split_ES' ... '...\n", | |
| " * ping_time (ping_time) datetime64[ns] 2024-03-04T05:27:27.453144064 .....\n", | |
| " * range_sample (range_sample) int64 0 1 2 3 4 5 ... 4185 4186 4187 4188 4189\n", | |
| "Attributes:\n", | |
| " long_name: Raw backscatter measurements (real part)\n", | |
| " units: dB</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'backscatter_r'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>channel</span>: 3</li><li><span class='xr-has-index'>ping_time</span>: 35731</li><li><span class='xr-has-index'>range_sample</span>: 4190</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-d6e4ad40-3092-4f01-b873-37e847e16e5a' class='xr-array-in' type='checkbox' checked><label for='section-d6e4ad40-3092-4f01-b873-37e847e16e5a' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>nan nan nan nan nan nan nan nan ... nan nan nan nan nan nan nan nan</span></div><div class='xr-array-data'><pre>array([[[ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 8.725166 , 14.063745 , 17.450333 , ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " ...,\n", | |
| " [ 8.666371 , 14.040227 , 17.462091 , ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 8.666371 , 14.051986 , 17.450333 , ..., nan,\n", | |
| " nan, nan]],\n", | |
| "\n", | |
| " [[ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 7.1141853, 10.453737 , 12.488041 , ..., nan,\n", | |
| " nan, nan],\n", | |
| "...\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan]],\n", | |
| "\n", | |
| " [[ 7.0318723, 9.677644 , 11.500286 , ..., -132.9706 ,\n", | |
| " -133.64085 , -134.6286 ],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " ...,\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 6.9495597, 9.595331 , 11.4179735, ..., -141.42531 ,\n", | |
| " -142.77759 , -143.80061 ],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan]]], dtype=float32)</pre></div></div></li><li class='xr-section-item'><input id='section-72bc1334-7593-493f-b831-a20caf22443b' class='xr-section-summary-in' type='checkbox' checked><label for='section-72bc1334-7593-493f-b831-a20caf22443b' class='xr-section-summary' >Coordinates: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>channel</span></div><div class='xr-var-dims'>(channel)</div><div class='xr-var-dtype'><U36</div><div class='xr-var-preview xr-preview'>'WBT Tube 267819-7 ES38-18DK-Spl...</div><input id='attrs-b5f24994-1b7d-4fe7-a534-ca3966021b3a' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-b5f24994-1b7d-4fe7-a534-ca3966021b3a' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-8f0e5f68-ac4b-4552-85f7-6b056e189323' class='xr-var-data-in' type='checkbox'><label for='data-8f0e5f68-ac4b-4552-85f7-6b056e189323' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>Vendor channel ID</dd></dl></div><div class='xr-var-data'><pre>array(['WBT Tube 267819-7 ES38-18DK-Split_ES',\n", | |
| " 'WBT Tube 267824-15-1 ES120-7CD_ES',\n", | |
| " 'WBT Tube 267824-15-2 ES200-7CD_ES'], dtype='<U36')</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>ping_time</span></div><div class='xr-var-dims'>(ping_time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2024-03-04T05:27:27.453144064 .....</div><input id='attrs-f09a031c-228e-4179-8c68-ac180dbb441e' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-f09a031c-228e-4179-8c68-ac180dbb441e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-b219b8fc-afba-4873-88d7-636326e4d589' class='xr-var-data-in' type='checkbox'><label for='data-b219b8fc-afba-4873-88d7-636326e4d589' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>Timestamp of each ping</dd><dt><span>standard_name :</span></dt><dd>time</dd><dt><span>axis :</span></dt><dd>T</dd></dl></div><div class='xr-var-data'><pre>array(['2024-03-04T05:27:27.453144064', '2024-03-04T05:27:28.251616768',\n", | |
| " '2024-03-04T05:27:29.442363904', ..., '2024-03-04T15:33:27.254448128',\n", | |
| " '2024-03-04T15:33:28.441876992', '2024-03-04T15:33:29.263218176'],\n", | |
| " dtype='datetime64[ns]')</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>range_sample</span></div><div class='xr-var-dims'>(range_sample)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 ... 4186 4187 4188 4189</div><input id='attrs-7935229e-67f0-413b-83d8-c14925bbc984' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-7935229e-67f0-413b-83d8-c14925bbc984' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-d71164c1-2a41-40a8-a7cb-3bb5c394803f' class='xr-var-data-in' type='checkbox'><label for='data-d71164c1-2a41-40a8-a7cb-3bb5c394803f' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>Along-range sample number, base 0</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 4187, 4188, 4189])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-c6057b7b-b86d-4bf9-88e1-54c20e27e8e8' class='xr-section-summary-in' type='checkbox' ><label for='section-c6057b7b-b86d-4bf9-88e1-54c20e27e8e8' class='xr-section-summary' >Indexes: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-index-name'><div>channel</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-c6f019ac-3dbf-44c3-96a7-0f2a123664e6' class='xr-index-data-in' type='checkbox'/><label for='index-c6f019ac-3dbf-44c3-96a7-0f2a123664e6' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Index(['WBT Tube 267819-7 ES38-18DK-Split_ES',\n", | |
| " 'WBT Tube 267824-15-1 ES120-7CD_ES',\n", | |
| " 'WBT Tube 267824-15-2 ES200-7CD_ES'],\n", | |
| " dtype='object', name='channel'))</pre></div></li><li class='xr-var-item'><div class='xr-index-name'><div>ping_time</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-e7e7b505-26b4-4c71-a8b4-d025081fa434' class='xr-index-data-in' type='checkbox'/><label for='index-e7e7b505-26b4-4c71-a8b4-d025081fa434' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(DatetimeIndex(['2024-03-04 05:27:27.453144064',\n", | |
| " '2024-03-04 05:27:28.251616768',\n", | |
| " '2024-03-04 05:27:29.442363904',\n", | |
| " '2024-03-04 05:27:30.251905024',\n", | |
| " '2024-03-04 05:27:31.442492928',\n", | |
| " '2024-03-04 05:27:32.251759104',\n", | |
| " '2024-03-04 05:27:33.442920960',\n", | |
| " '2024-03-04 05:27:34.251890176',\n", | |
| " '2024-03-04 05:27:35.442065920',\n", | |
| " '2024-03-04 05:27:36.252597760',\n", | |
| " ...\n", | |
| " '2024-03-04 15:33:20.442040832',\n", | |
| " '2024-03-04 15:33:21.252228096',\n", | |
| " '2024-03-04 15:33:22.444801024',\n", | |
| " '2024-03-04 15:33:23.252869120',\n", | |
| " '2024-03-04 15:33:24.444175872',\n", | |
| " '2024-03-04 15:33:25.252228096',\n", | |
| " '2024-03-04 15:33:26.442637824',\n", | |
| " '2024-03-04 15:33:27.254448128',\n", | |
| " '2024-03-04 15:33:28.441876992',\n", | |
| " '2024-03-04 15:33:29.263218176'],\n", | |
| " dtype='datetime64[ns]', name='ping_time', length=35731, freq=None))</pre></div></li><li class='xr-var-item'><div class='xr-index-name'><div>range_sample</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-8c53e301-2dcf-4fcc-90bf-7dfd1c58f6e6' class='xr-index-data-in' type='checkbox'/><label for='index-8c53e301-2dcf-4fcc-90bf-7dfd1c58f6e6' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Int64Index([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,\n", | |
| " ...\n", | |
| " 4180, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189],\n", | |
| " dtype='int64', name='range_sample', length=4190))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-9e6e108c-e962-4e19-82ef-2a93dce25d35' class='xr-section-summary-in' type='checkbox' checked><label for='section-9e6e108c-e962-4e19-82ef-2a93dce25d35' class='xr-section-summary' >Attributes: <span>(2)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>Raw backscatter measurements (real part)</dd><dt><span>units :</span></dt><dd>dB</dd></dl></div></li></ul></div></div>" | |
| ], | |
| "text/plain": [ | |
| "<xarray.DataArray 'backscatter_r' (channel: 3, ping_time: 35731,\n", | |
| " range_sample: 4190)>\n", | |
| "array([[[ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 8.725166 , 14.063745 , 17.450333 , ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " ...,\n", | |
| " [ 8.666371 , 14.040227 , 17.462091 , ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 8.666371 , 14.051986 , 17.450333 , ..., nan,\n", | |
| " nan, nan]],\n", | |
| "\n", | |
| " [[ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 7.1141853, 10.453737 , 12.488041 , ..., nan,\n", | |
| " nan, nan],\n", | |
| "...\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan]],\n", | |
| "\n", | |
| " [[ 7.0318723, 9.677644 , 11.500286 , ..., -132.9706 ,\n", | |
| " -133.64085 , -134.6286 ],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " ...,\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan],\n", | |
| " [ 6.9495597, 9.595331 , 11.4179735, ..., -141.42531 ,\n", | |
| " -142.77759 , -143.80061 ],\n", | |
| " [ nan, nan, nan, ..., nan,\n", | |
| " nan, nan]]], dtype=float32)\n", | |
| "Coordinates:\n", | |
| " * channel (channel) <U36 'WBT Tube 267819-7 ES38-18DK-Split_ES' ... '...\n", | |
| " * ping_time (ping_time) datetime64[ns] 2024-03-04T05:27:27.453144064 .....\n", | |
| " * range_sample (range_sample) int64 0 1 2 3 4 5 ... 4185 4186 4187 4188 4189\n", | |
| "Attributes:\n", | |
| " long_name: Raw backscatter measurements (real part)\n", | |
| " units: dB" | |
| ] | |
| }, | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "ed[\"Sonar/Beam_group1\"][\"backscatter_r\"]" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "The above shows that the uncalibrated (and most likely the calibrated) data is not all NaNs. We can check by computing calibrated Sv:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# Test compute Sv\n", | |
| "env_params = {\n", | |
| "'long_name': 'Coastal Endurance, Oregon Shelf Cabled Benthic Experiment Package',\n", | |
| "'tilt_correction': 0,\n", | |
| "'colorbar_range': [-90, -50],\n", | |
| "'vertical_range': [0, 80],\n", | |
| "'deployed_depth': 82,\n", | |
| "'depth_offset': 2.0,\n", | |
| "'average_salinity': 33,\n", | |
| "'average_temperature': 10,\n", | |
| "'instrument_orientation': 'up'\n", | |
| "}\n", | |
| "ds_Sv = ep.calibrate.compute_Sv(ed, env_params=env_params, waveform_mode='CW', encode_mode='power')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "Now we calculate the number of NaNs and non-NaNs in the calibrated backscatter data to show the percentage that is NaN vs not NaN:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Calibrated NaN percentage: 73.98749856029987 %\n", | |
| "Calibrated Non-NaN percentage: 26.012501439700124 %\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "# Count Calibrated NaN and non-NaN values\n", | |
| "nan_count = np.sum(np.isnan(ds_Sv[\"Sv\"].data))\n", | |
| "non_nan_count = np.sum(~np.isnan(ds_Sv[\"Sv\"].data))\n", | |
| "\n", | |
| "total_count = nan_count + non_nan_count\n", | |
| "\n", | |
| "# Compute percentages\n", | |
| "nan_percentage = (nan_count / total_count) * 100\n", | |
| "non_nan_percentage = (non_nan_count / total_count) * 100\n", | |
| "\n", | |
| "print(f\"Calibrated NaN percentage: {nan_percentage} %\")\n", | |
| "print(f\"Calibrated Non-NaN percentage: {non_nan_percentage} %\")" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "echopype", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.9.18" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment