mdsumner/kerchunk_serialize_xarray.md

## kerchunk_serialize_xarray.md

      
    Raw
  

              kerchunk_serialize_xarray.md
            
          
    import fsspec
import kerchunk.hdf

fs = fsspec.filesystem('https')

urls = ["https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240119.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240120.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240121.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240122.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240123.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240124.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240125.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240126.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240127.nc",
        "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240128.nc"]


## turned off because I don't know what fsspec needs
# so = dict(
#     anon=True, default_fill_cache=False, default_cache_type='first'
# )
singles = []
for u in urls:
    with fs.open(u) as inf:
        h5chunks = kerchunk.hdf.SingleHdf5ToZarr(inf, u, inline_threshold=100)
        singles.append(h5chunks.translate())
        

from kerchunk.combine import MultiZarrToZarr
mzz = MultiZarrToZarr(
    singles,
    remote_protocol="https",
    remote_options={'anon': True},
    concat_dims=["time"]
)

out = mzz.translate()

import xarray as xr
ds = xr.open_dataset(
    "reference://", engine="zarr",
    backend_kwargs={
        "storage_options": {
            "fo": out,
            "remote_protocol": "https",
            #"remote_options": {"anon": True}
        },
        "consolidated": False
    }, 
    decode_times = True
)


## out.json
{'version': 1, 'refs': {'.zgroup': '{"zarr_format":2}', 'time/.zarray': '{\n    "chunks": [\n        2\n    ],\n    "compressor": null,\n    "dtype": "<f4",\n    "fill_value": null,\n    "filters": null,\n    "order": "C",\n    "shape": [\n        2\n    ],\n    "zarr_format": 2\n}', 'time/0': 'base64:MCwgMArXIzw=', 'time/.zattrs': '{\n    "_ARRAY_DIMENSIONS": [\n        "time"\n    ],\n    "long_name": "Center time of the day",\n    "units": "days since 1978-01-01 12:00:00"\n}', '.zattrs': '{"Conventions":"CF-1.6, ACDD-1.3","cdm_data_type":"Grid","comment":"Data was converted from NetCDF-3 to NetCDF-4 format with metadata updates in November 2017.","creator_email":"oisst-help@noaa.gov","creator_url":"https:\\/\\/www.ncei.noaa.gov\\/","date_created":"2024-02-03T09:12:00Z","date_modified":"2024-02-03T09:12:00Z","geospatial_lat_max":90.0,"geospatial_lat_min":-90.0,"geospatial_lat_resolution":0.25,"geospatial_lat_units":"degrees_north","geospatial_lon_max":360.0,"geospatial_lon_min":0.0,"geospatial_lon_resolution":0.25,"geospatial_lon_units":"degrees_east","history":"Final file created using preliminary as first guess, and 3 days of AVHRR data. Preliminary uses only 1 day of AVHRR data.","id":"oisst-avhrr-v02r01.20240119.nc","institution":"NOAA\\/National Centers for Environmental Information","instrument":"Earth Remote Sensing Instruments > Passive Remote Sensing > Spectrometers\\/Radiometers > Imaging Spectrometers\\/Radiometers > AVHRR > Advanced Very High Resolution Radiometer","instrument_vocabulary":"Global Change Master Directory (GCMD) Instrument Keywords","keywords":"Earth Science > Oceans > Ocean Temperature > Sea Surface Temperature","keywords_vocabulary":"Global Change Master Directory (GCMD) Earth Science Keywords","metadata_link":"https:\\/\\/doi.org\\/10.25921\\/RE9P-PT57","naming_authority":"gov.noaa.ncei","ncei_template_version":"NCEI_NetCDF_Grid_Template_v2.0","platform":"Ships, buoys, Argo floats, MetOp-A, MetOp-B","platform_vocabulary":"Global Change Master Directory (GCMD) Platform Keywords","processing_level":"NOAA Level 4","product_version":"Version v02r01","references":"Reynolds, et al.(2007) Daily High-Resolution-Blended Analyses for Sea Surface Temperature (available at https:\\/\\/doi.org\\/10.1175\\/2007JCLI1824.1). Banzon, et al.(2016) A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies (available at https:\\/\\/doi.org\\/10.5194\\/essd-8-165-2016). Huang et al. (2020) Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version v02r01, submitted.Climatology is based on 1971-2000 OI.v2 SST. Satellite data: Pathfinder AVHRR SST, Navy AVHRR SST, and NOAA ACSPO SST. Ice data: NCEP Ice and GSFC Ice.","sensor":"Thermometer, AVHRR","source":"ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfinder_AVHRR, Navy_AVHRR, NOAA_ACSP","standard_name_vocabulary":"CF Standard Name Table (v40, 25 January 2017)","summary":"NOAAs 1\\/4-degree Daily Optimum Interpolation Sea Surface Temperature (OISST) (sometimes referred to as Reynolds SST, which however also refers to earlier products at different resolution), currently available as version v02r01, is created by interpolating and extrapolating SST observations from different sources, resulting in a smoothed complete field. The sources of data are satellite (AVHRR) and in situ platforms (i.e., ships and buoys), and the specific datasets employed may change over time. At the marginal ice zone, sea ice concentrations are used to generate proxy SSTs.  A preliminary version of this file is produced in near-real time (1-day latency), and then replaced with a final version after 2 weeks. Note that this is the AVHRR-ONLY DOISST, available from Oct 1981, but there is a companion DOISST product that includes microwave satellite data, available from June 2002","time_coverage_end":"2024-01-19T23:59:59Z","time_coverage_start":"2024-01-19T00:00:00Z","title":"NOAA\\/NCEI 1\\/4 Degree Daily Optimum Interpolation Sea Surface Temperature (OISST) Analysis, Version 2.1 - Final"}', 'anom/.zarray': '{"chunks":[1,1,720,1440],"compressor":null,"dtype":"<i2","fill_value":-999,"filters":[{"elementsize":2,"id":"shuffle"},{"id":"zlib","level":4}],"order":"C","shape":[2,1,720,1440],"zarr_format":2}', 'anom/.zattrs': '{"_ARRAY_DIMENSIONS":["time","zlev","lat","lon"],"add_offset":0.0,"long_name":"Daily sea surface temperature anomalies","scale_factor":0.009999999776482582,"units":"Celsius","valid_max":1200,"valid_min":-1200}', 'anom/0.0.0.0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240128.nc', 716869, 666579], 'err/.zarray': '{"chunks":[1,1,720,1440],"compressor":null,"dtype":"<i2","fill_value":-999,"filters":[{"elementsize":2,"id":"shuffle"},{"id":"zlib","level":4}],"order":"C","shape":[2,1,720,1440],"zarr_format":2}', 'err/.zattrs': '{"_ARRAY_DIMENSIONS":["time","zlev","lat","lon"],"add_offset":0.0,"long_name":"Estimated error standard deviation of analysed_sst","scale_factor":0.009999999776482582,"units":"Celsius","valid_max":1000,"valid_min":0}', 'err/0.0.0.0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240128.nc', 1383448, 105074], 'ice/.zarray': '{"chunks":[1,1,720,1440],"compressor":null,"dtype":"<i2","fill_value":-999,"filters":[{"elementsize":2,"id":"shuffle"},{"id":"zlib","level":4}],"order":"C","shape":[2,1,720,1440],"zarr_format":2}', 'ice/.zattrs': '{"_ARRAY_DIMENSIONS":["time","zlev","lat","lon"],"add_offset":0.0,"long_name":"Sea ice concentration","scale_factor":0.009999999776482582,"units":"%","valid_max":100,"valid_min":0}', 'ice/0.0.0.0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240128.nc', 1488522, 51397], 'lat/.zarray': '{"chunks":[720],"compressor":null,"dtype":"<f4","fill_value":null,"filters":null,"order":"C","shape":[720],"zarr_format":2}', 'lat/.zattrs': '{"_ARRAY_DIMENSIONS":["lat"],"grids":"Uniform grid from -89.875 to 89.875 by 0.25","long_name":"Latitude","units":"degrees_north"}', 'lat/0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240119.nc', 22275, 2880], 'lon/.zarray': '{"chunks":[1440],"compressor":null,"dtype":"<f4","fill_value":null,"filters":null,"order":"C","shape":[1440],"zarr_format":2}', 'lon/.zattrs': '{"_ARRAY_DIMENSIONS":["lon"],"grids":"Uniform grid from 0.125 to 359.875 by 0.25","long_name":"Longitude","units":"degrees_east"}', 'lon/0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240119.nc', 25155, 5760], 'sst/.zarray': '{"chunks":[1,1,720,1440],"compressor":null,"dtype":"<i2","fill_value":-999,"filters":[{"elementsize":2,"id":"shuffle"},{"id":"zlib","level":4}],"order":"C","shape":[2,1,720,1440],"zarr_format":2}', 'sst/.zattrs': '{"_ARRAY_DIMENSIONS":["time","zlev","lat","lon"],"add_offset":0.0,"long_name":"Daily sea surface temperature","scale_factor":0.009999999776482582,"units":"Celsius","valid_max":4500,"valid_min":-300}', 'sst/0.0.0.0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240128.nc', 47587, 669282], 'zlev/.zarray': '{"chunks":[1],"compressor":null,"dtype":"<f4","fill_value":null,"filters":null,"order":"C","shape":[1],"zarr_format":2}', 'zlev/.zattrs': '{"_ARRAY_DIMENSIONS":["zlev"],"actual_range":"0, 0","long_name":"Sea surface height","positive":"down","units":"meters"}', 'zlev/0': ['https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/202401/oisst-avhrr-v02r01.20240119.nc', 30915, 4]}}