Skip to content

Instantly share code, notes, and snippets.

@peterm790

peterm790/GEFS_Kerchunk.ipynb

Created October 28, 2022 13:53
Show Gist options
  • Save peterm790/a50dcec3f63dc6ca7034b22a9f34b340 to your computer and use it in GitHub Desktop.
Save peterm790/a50dcec3f63dc6ca7034b22a9f34b340 to your computer and use it in GitHub Desktop.
Download ZIP
GEFS Kerchunk
Display the source blob
Display the rendered blob
Raw
GEFS_Kerchunk.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "7658ffed",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import fsspec\n",
"from datetime import datetime, timedelta\n",
"import xarray as xr\n",
"import ujson"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ed57dfd0",
"metadata": {},
"outputs": [],
"source": [
"import kerchunk\n",
"from kerchunk.grib2 import scan_grib\n",
"from kerchunk.combine import MultiZarrToZarr"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "08f6b6f9",
"metadata": {},
"outputs": [],
"source": [
"fs_local = fsspec.filesystem('')\n",
"fs_s3 = fsspec.filesystem('s3', anon = True)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "213620f4",
"metadata": {},
"outputs": [],
"source": [
"dates = fs_s3.ls(f\"s3://noaa-gefs-pds/\")\n",
"date = sorted(dates)[-2].split('/')[-1] # latest date available "
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "1dda0770",
"metadata": {},
"outputs": [],
"source": [
"runs = fs_s3.ls(f\"s3://noaa-gefs-pds/{date}/\")\n",
"run = runs[-1].split('/')[-1] # latest run available "
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "9bbbebc9",
"metadata": {},
"outputs": [],
"source": [
"members = []\n",
"for i in range(1,31):\n",
" members.append(str(i).zfill(2))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "157dc13a",
"metadata": {},
"outputs": [],
"source": [
"afilter = {\n",
" 'typeOfLevel': 'heightAboveGround', \n",
" 'level': 2, \n",
" 'cfVarName': 't2m'\n",
"}\n",
"\n",
"s3_so = {\n",
" 'anon': True, \n",
" 'skip_instance_cache': True\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "f53a6050",
"metadata": {},
"outputs": [],
"source": [
"fs_local.mkdir('individual/')\n",
"\n",
"def gen_json(file_url):\n",
" out = scan_grib(file_url, storage_options= s3_so, filter=afilter)[0]\n",
" name = f\"individual/{file_url.split('/')[-1]}.json\"\n",
" with fs_local.open(name, 'w') as f:\n",
" f.write(ujson.dumps(out))\n",
" \n",
"fs_local.mkdir('combined/')\n",
"\n",
"def combine(json_urls):\n",
" mzz = MultiZarrToZarr(json_urls,concat_dims = ['valid_time'])\n",
" name = f\"combined/{'.'.join(json_urls[0].split('/')[-1].split('.')[:-2])}.combined.json\"\n",
" with fs_local.open(name, 'w') as f:\n",
" f.write(ujson.dumps(mzz.translate()))\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "4e43163f",
"metadata": {},
"outputs": [],
"source": [
"for ensemble_member in members:\n",
" files = fs_s3.glob(f\"s3://noaa-gefs-pds/{date}/{run}/atmos/pgrb2ap5/gep{ensemble_member}*\")\n",
" files = [f for f in files if f.split('.')[-1] != 'idx']\n",
" files = sorted(['s3://'+f for f in files])\n",
" files = files[:2] #just doing the first 2 files for now\n",
" for file in files:\n",
" gen_json(file)\n",
" jsons = fs_local.glob(f\"individual/gep{ensemble_member}*\")\n",
" combine(jsons)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "3e21af41",
"metadata": {},
"outputs": [],
"source": [
"combined = fs_local.glob(f\"combined/gep*\")"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "bad65031",
"metadata": {},
"outputs": [],
"source": [
"mzz = MultiZarrToZarr(combined, \n",
" remote_protocol='s3',\n",
" remote_options={'anon':True},\n",
" concat_dims = ['number'],\n",
" identical_dims = ['valid_time', 'longitude', 'latitude', 'step','time'])\n",
"\n",
"\n",
"out = mzz.translate()"
]
},
{
"cell_type": "code",
"execution_count": 66,
"id": "57aa4690",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2 {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (number: 30, valid_time: 2, longitude: 361,\n",
" latitude: 720, heightAboveGround: 1, step: 1, time: 1)\n",
"Coordinates:\n",
" * heightAboveGround (heightAboveGround) int64 2\n",
" * latitude (latitude) float64 90.0 90.0 90.0 90.0 ... 90.0 90.0 90.0\n",
" * longitude (longitude) float64 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0\n",
" * number (number) float64 1.0 2.0 3.0 4.0 ... 27.0 28.0 29.0 30.0\n",
" * step (step) timedelta64[ns] 00:00:00\n",
" * time (time) datetime64[ns] 2022-10-28T06:00:00\n",
" * valid_time (valid_time) datetime64[ns] 2022-10-28T06:00:00 2022-1...\n",
"Data variables:\n",
" 2t (number, valid_time, longitude, latitude) float64 ...\n",
"Attributes:\n",
" centre: kwbc\n",
" centreDescription: US National Weather Service - NCEP\n",
" edition: 2\n",
" subCentre: 2</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-c47978d0-569b-40f4-858e-4ddddb3b5fb2' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-c47978d0-569b-40f4-858e-4ddddb3b5fb2' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>number</span>: 30</li><li><span class='xr-has-index'>valid_time</span>: 2</li><li><span class='xr-has-index'>longitude</span>: 361</li><li><span class='xr-has-index'>latitude</span>: 720</li><li><span class='xr-has-index'>heightAboveGround</span>: 1</li><li><span class='xr-has-index'>step</span>: 1</li><li><span class='xr-has-index'>time</span>: 1</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-d477f97a-f0c1-41bc-bf00-cc2504ef99ae' class='xr-section-summary-in' type='checkbox' checked><label for='section-d477f97a-f0c1-41bc-bf00-cc2504ef99ae' class='xr-section-summary' >Coordinates: <span>(7)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>heightAboveGround</span></div><div class='xr-var-dims'>(heightAboveGround)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>2</div><input id='attrs-3ec8b329-1a14-4454-b58e-239d2ae79efa' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-3ec8b329-1a14-4454-b58e-239d2ae79efa' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-7e1e705d-9fbb-4c83-bf7a-5db2bfd646ef' class='xr-var-data-in' type='checkbox'><label for='data-7e1e705d-9fbb-4c83-bf7a-5db2bfd646ef' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>height above the surface</dd><dt><span>positive :</span></dt><dd>up</dd><dt><span>standard_name :</span></dt><dd>height</dd><dt><span>units :</span></dt><dd>m</dd></dl></div><div class='xr-var-data'><pre>array([2])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>latitude</span></div><div class='xr-var-dims'>(latitude)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>90.0 90.0 90.0 ... 90.0 90.0 90.0</div><input id='attrs-2eb9f0dc-3cc3-4120-a475-be9395af278d' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-2eb9f0dc-3cc3-4120-a475-be9395af278d' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-a8cd672c-2e72-40e1-873e-6ab1e17622c2' class='xr-var-data-in' type='checkbox'><label for='data-a8cd672c-2e72-40e1-873e-6ab1e17622c2' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>latitude</dd><dt><span>standard_name :</span></dt><dd>latitude</dd><dt><span>units :</span></dt><dd>degrees_north</dd></dl></div><div class='xr-var-data'><pre>array([90., 90., 90., ..., 90., 90., 90.])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>longitude</span></div><div class='xr-var-dims'>(longitude)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0</div><input id='attrs-9dfcc1ff-7fec-403a-87eb-de2896a00d32' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-9dfcc1ff-7fec-403a-87eb-de2896a00d32' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-82d70fef-1de8-4ce7-80d9-40288dad7d15' class='xr-var-data-in' type='checkbox'><label for='data-82d70fef-1de8-4ce7-80d9-40288dad7d15' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>longitude</dd><dt><span>standard_name :</span></dt><dd>longitude</dd><dt><span>units :</span></dt><dd>degrees_east</dd></dl></div><div class='xr-var-data'><pre>array([0., 0., 0., ..., 0., 0., 0.])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>number</span></div><div class='xr-var-dims'>(number)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>1.0 2.0 3.0 4.0 ... 28.0 29.0 30.0</div><input id='attrs-88b93ef2-2cf3-4221-b381-af9edb36352c' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-88b93ef2-2cf3-4221-b381-af9edb36352c' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-83fb392d-3ba1-4621-8e22-49e0d4216ead' class='xr-var-data-in' type='checkbox'><label for='data-83fb392d-3ba1-4621-8e22-49e0d4216ead' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>ensemble member numerical id</dd><dt><span>standard_name :</span></dt><dd>realization</dd><dt><span>units :</span></dt><dd>1</dd></dl></div><div class='xr-var-data'><pre>array([ 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14.,\n",
" 15., 16., 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28.,\n",
" 29., 30.])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>step</span></div><div class='xr-var-dims'>(step)</div><div class='xr-var-dtype'>timedelta64[ns]</div><div class='xr-var-preview xr-preview'>00:00:00</div><input id='attrs-e78a765c-b1fd-4c0b-bde9-1b1a0c4f9b4e' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-e78a765c-b1fd-4c0b-bde9-1b1a0c4f9b4e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-d270fd33-a342-43bf-80ac-99f6eef01852' class='xr-var-data-in' type='checkbox'><label for='data-d270fd33-a342-43bf-80ac-99f6eef01852' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>time since forecast_reference_time</dd><dt><span>standard_name :</span></dt><dd>forecast_period</dd></dl></div><div class='xr-var-data'><pre>array([0], dtype=&#x27;timedelta64[ns]&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2022-10-28T06:00:00</div><input id='attrs-cc729adf-facb-436a-b6c4-05e559cf59e6' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-cc729adf-facb-436a-b6c4-05e559cf59e6' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-eb7a4976-030b-40a2-94b9-26b42cf03959' class='xr-var-data-in' type='checkbox'><label for='data-eb7a4976-030b-40a2-94b9-26b42cf03959' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>initial time of forecast</dd><dt><span>standard_name :</span></dt><dd>forecast_reference_time</dd></dl></div><div class='xr-var-data'><pre>array([&#x27;2022-10-28T06:00:00.000000000&#x27;], dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>valid_time</span></div><div class='xr-var-dims'>(valid_time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2022-10-28T06:00:00 2022-10-28T0...</div><input id='attrs-1a17fda3-2c85-413e-9be1-662161b0770a' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-1a17fda3-2c85-413e-9be1-662161b0770a' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-c7c23636-907f-40ea-829c-202190fdcaa5' class='xr-var-data-in' type='checkbox'><label for='data-c7c23636-907f-40ea-829c-202190fdcaa5' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>time</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array([&#x27;2022-10-28T06:00:00.000000000&#x27;, &#x27;2022-10-28T09:00:00.000000000&#x27;],\n",
" dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-93249ea1-d009-404e-bb95-dadacd549c2b' class='xr-section-summary-in' type='checkbox' checked><label for='section-93249ea1-d009-404e-bb95-dadacd549c2b' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>2t</span></div><div class='xr-var-dims'>(number, valid_time, longitude, latitude)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-11e6c806-54b3-4135-8eeb-b76cdbc7e498' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-11e6c806-54b3-4135-8eeb-b76cdbc7e498' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-62869495-ebe4-4321-a9a4-151340ee00bc' class='xr-var-data-in' type='checkbox'><label for='data-62869495-ebe4-4321-a9a4-151340ee00bc' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>NV :</span></dt><dd>0</dd><dt><span>cfName :</span></dt><dd>air_temperature</dd><dt><span>cfVarName :</span></dt><dd>t2m</dd><dt><span>dataDate :</span></dt><dd>20221028</dd><dt><span>dataTime :</span></dt><dd>600</dd><dt><span>dataType :</span></dt><dd>pf</dd><dt><span>endStep :</span></dt><dd>0</dd><dt><span>gridDefinitionDescription :</span></dt><dd>Latitude/longitude. Also called equidistant cylindrical, or Plate Carree</dd><dt><span>gridType :</span></dt><dd>regular_ll</dd><dt><span>missingValue :</span></dt><dd>9999</dd><dt><span>name :</span></dt><dd>2 metre temperature</dd><dt><span>numberOfPoints :</span></dt><dd>259920</dd><dt><span>paramId :</span></dt><dd>167</dd><dt><span>shortName :</span></dt><dd>2t</dd><dt><span>stepType :</span></dt><dd>instant</dd><dt><span>stepUnits :</span></dt><dd>1</dd><dt><span>totalNumber :</span></dt><dd>30</dd><dt><span>typeOfLevel :</span></dt><dd>heightAboveGround</dd><dt><span>units :</span></dt><dd>K</dd></dl></div><div class='xr-var-data'><pre>[15595200 values with dtype=float64]</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-42256e9e-9f4c-4fcd-b25d-e77a2cb3374a' class='xr-section-summary-in' type='checkbox' checked><label for='section-42256e9e-9f4c-4fcd-b25d-e77a2cb3374a' class='xr-section-summary' >Attributes: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>centre :</span></dt><dd>kwbc</dd><dt><span>centreDescription :</span></dt><dd>US National Weather Service - NCEP</dd><dt><span>edition :</span></dt><dd>2</dd><dt><span>subCentre :</span></dt><dd>2</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (number: 30, valid_time: 2, longitude: 361,\n",
" latitude: 720, heightAboveGround: 1, step: 1, time: 1)\n",
"Coordinates:\n",
" * heightAboveGround (heightAboveGround) int64 2\n",
" * latitude (latitude) float64 90.0 90.0 90.0 90.0 ... 90.0 90.0 90.0\n",
" * longitude (longitude) float64 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0\n",
" * number (number) float64 1.0 2.0 3.0 4.0 ... 27.0 28.0 29.0 30.0\n",
" * step (step) timedelta64[ns] 00:00:00\n",
" * time (time) datetime64[ns] 2022-10-28T06:00:00\n",
" * valid_time (valid_time) datetime64[ns] 2022-10-28T06:00:00 2022-1...\n",
"Data variables:\n",
" 2t (number, valid_time, longitude, latitude) float64 ...\n",
"Attributes:\n",
" centre: kwbc\n",
" centreDescription: US National Weather Service - NCEP\n",
" edition: 2\n",
" subCentre: 2"
]
},
"execution_count": 66,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fs_ = fsspec.filesystem(\"reference\", fo=out, remote_protocol='s3', remote_options={'anon':True})\n",
"m = fs_.get_mapper(\"\")\n",
"xr.open_dataset(m, engine=\"zarr\", backend_kwargs=dict(consolidated=False))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment