Skip to content

Instantly share code, notes, and snippets.

@aidanheerdegen

aidanheerdegen/Intake example.ipynb

Created May 3, 2023 01:16
Show Gist options
  • Save aidanheerdegen/d75a9ddc5d482fcfa6f8b172a3001295 to your computer and use it in GitHub Desktop.
Save aidanheerdegen/d75a9ddc5d482fcfa6f8b172a3001295 to your computer and use it in GitHub Desktop.
Download ZIP
Small example of querying intake catalogue
Display the source blob
Display the rendered blob
Raw
Intake example.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 45,
"id": "dc1b1e23-0ba7-4b64-b4df-0c30b472ae43",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import pathlib\n",
"import ast\n",
"import intake\n",
"from itertools import chain"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "08971dbd-118e-43e4-9aaf-344835edc512",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"wd = pathlib.Path('/g/data/tm70/aph502/intake')"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "6654db6a-9276-44ee-be6f-8e5d32aefb0d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"data_catalog = intake.open_esm_datastore(\n",
" wd / 'cosima_1deg_jra55_iaf_omip2_cycle6.json', \n",
" read_csv_kwargs={\n",
" \"converters\": {\n",
" \"variable\": ast.literal_eval,\n",
" \"variable_long_name\": ast.literal_eval,\n",
" \"variable_standard_name\": ast.literal_eval,\n",
" \"variable_cell_methods\": ast.literal_eval,\n",
" }\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "c8244312-742e-43af-8230-21d75b2efe03",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<p><strong>cosima_1deg_jra55_iaf_omip2_cycle6 catalog with 11 dataset(s) from 2013 asset(s)</strong>:</p> <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>unique</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>path</th>\n",
" <td>2013</td>\n",
" </tr>\n",
" <tr>\n",
" <th>realm</th>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable</th>\n",
" <td>338</td>\n",
" </tr>\n",
" <tr>\n",
" <th>frequency</th>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>start_date</th>\n",
" <td>855</td>\n",
" </tr>\n",
" <tr>\n",
" <th>end_date</th>\n",
" <td>793</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable_long_name</th>\n",
" <td>306</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable_standard_name</th>\n",
" <td>56</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable_cell_methods</th>\n",
" <td>5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>filename</th>\n",
" <td>1473</td>\n",
" </tr>\n",
" <tr>\n",
" <th>file_id</th>\n",
" <td>11</td>\n",
" </tr>\n",
" <tr>\n",
" <th>derived_variable</th>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"data_catalog"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "680d8ddf-4f40-4a3b-ac32-8a2d09347484",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>path</th>\n",
" <th>realm</th>\n",
" <th>variable</th>\n",
" <th>frequency</th>\n",
" <th>start_date</th>\n",
" <th>end_date</th>\n",
" <th>variable_long_name</th>\n",
" <th>variable_standard_name</th>\n",
" <th>variable_cell_methods</th>\n",
" <th>filename</th>\n",
" <th>file_id</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>seaIce</td>\n",
" <td>[time, time_bounds, TLON, TLAT, ULON, ULAT, NC...</td>\n",
" <td>1day</td>\n",
" <td>1958-01-01, 00:00:00</td>\n",
" <td>1958-02-01, 00:00:00</td>\n",
" <td>[model time, boundaries for time-averaging int...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>iceh.1958-01-daily</td>\n",
" <td>iceh_daily</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>seaIce</td>\n",
" <td>[time, time_bounds, TLON, TLAT, ULON, ULAT, NC...</td>\n",
" <td>1mon</td>\n",
" <td>1958-01-01, 00:00:00</td>\n",
" <td>1958-02-01, 00:00:00</td>\n",
" <td>[model time, boundaries for time-averaging int...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>iceh.1958-01</td>\n",
" <td>iceh</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>seaIce</td>\n",
" <td>[time, time_bounds, TLON, TLAT, ULON, ULAT, NC...</td>\n",
" <td>1day</td>\n",
" <td>1958-02-01, 00:00:00</td>\n",
" <td>1958-03-01, 00:00:00</td>\n",
" <td>[model time, boundaries for time-averaging int...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>iceh.1958-02-daily</td>\n",
" <td>iceh_daily</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>seaIce</td>\n",
" <td>[time, time_bounds, TLON, TLAT, ULON, ULAT, NC...</td>\n",
" <td>1mon</td>\n",
" <td>1958-02-01, 00:00:00</td>\n",
" <td>1958-03-01, 00:00:00</td>\n",
" <td>[model time, boundaries for time-averaging int...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>iceh.1958-02</td>\n",
" <td>iceh</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>seaIce</td>\n",
" <td>[time, time_bounds, TLON, TLAT, ULON, ULAT, NC...</td>\n",
" <td>1day</td>\n",
" <td>1958-03-01, 00:00:00</td>\n",
" <td>1958-04-01, 00:00:00</td>\n",
" <td>[model time, boundaries for time-averaging int...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>iceh.1958-03-daily</td>\n",
" <td>iceh_daily</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" path realm \\\n",
"0 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... seaIce \n",
"1 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... seaIce \n",
"2 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... seaIce \n",
"3 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... seaIce \n",
"4 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... seaIce \n",
"\n",
" variable frequency \\\n",
"0 [time, time_bounds, TLON, TLAT, ULON, ULAT, NC... 1day \n",
"1 [time, time_bounds, TLON, TLAT, ULON, ULAT, NC... 1mon \n",
"2 [time, time_bounds, TLON, TLAT, ULON, ULAT, NC... 1day \n",
"3 [time, time_bounds, TLON, TLAT, ULON, ULAT, NC... 1mon \n",
"4 [time, time_bounds, TLON, TLAT, ULON, ULAT, NC... 1day \n",
"\n",
" start_date end_date \\\n",
"0 1958-01-01, 00:00:00 1958-02-01, 00:00:00 \n",
"1 1958-01-01, 00:00:00 1958-02-01, 00:00:00 \n",
"2 1958-02-01, 00:00:00 1958-03-01, 00:00:00 \n",
"3 1958-02-01, 00:00:00 1958-03-01, 00:00:00 \n",
"4 1958-03-01, 00:00:00 1958-04-01, 00:00:00 \n",
"\n",
" variable_long_name variable_standard_name \\\n",
"0 [model time, boundaries for time-averaging int... [] \n",
"1 [model time, boundaries for time-averaging int... [] \n",
"2 [model time, boundaries for time-averaging int... [] \n",
"3 [model time, boundaries for time-averaging int... [] \n",
"4 [model time, boundaries for time-averaging int... [] \n",
"\n",
" variable_cell_methods filename \\\n",
"0 [time: mean, time: mean, time: mean, time: mea... iceh.1958-01-daily \n",
"1 [time: mean, time: mean, time: mean, time: mea... iceh.1958-01 \n",
"2 [time: mean, time: mean, time: mean, time: mea... iceh.1958-02-daily \n",
"3 [time: mean, time: mean, time: mean, time: mea... iceh.1958-02 \n",
"4 [time: mean, time: mean, time: mean, time: mea... iceh.1958-03-daily \n",
"\n",
" file_id \n",
"0 iceh_daily \n",
"1 iceh \n",
"2 iceh_daily \n",
"3 iceh \n",
"4 iceh_daily "
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_catalog.df.head()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "e54859fa-7682-424b-84d6-d3a7b343b38a",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"array(['seaIce', 'ocean'], dtype=object)"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_catalog.df.realm.unique()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "35ff46be-0ed2-4d6e-9377-3071885b34ac",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"['time',\n",
" 'time_bounds',\n",
" 'TLON',\n",
" 'TLAT',\n",
" 'ULON',\n",
" 'ULAT',\n",
" 'NCAT',\n",
" 'tmask',\n",
" 'blkmask',\n",
" 'tarea',\n",
" 'uarea',\n",
" 'dxt',\n",
" 'dyt',\n",
" 'dxu',\n",
" 'dyu',\n",
" 'HTN',\n",
" 'HTE',\n",
" 'ANGLE',\n",
" 'ANGLET',\n",
" 'hi',\n",
" 'hs',\n",
" 'aice',\n",
" 'uvel',\n",
" 'vvel',\n",
" 'frzmlt',\n",
" 'congel',\n",
" 'frazil',\n",
" 'snoice',\n",
" 'dvidtt',\n",
" 'dvidtd']"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_catalog.df.variable[0]"
]
},
{
"cell_type": "markdown",
"id": "5d1beac4-07fb-45f4-af34-c5180b911d94",
"metadata": {},
"source": [
"Pull out just the ocean variables"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "3fc8e6e7-78ef-4e0b-9249-1ac45100c660",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ocean_cat = data_catalog.search(realm=\"ocean\")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "7a31da92-411b-4380-a2ce-2ce3cd24bbc8",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<p><strong>cosima_1deg_jra55_iaf_omip2_cycle6 catalog with 9 dataset(s) from 549 asset(s)</strong>:</p> <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>unique</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>path</th>\n",
" <td>549</td>\n",
" </tr>\n",
" <tr>\n",
" <th>realm</th>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable</th>\n",
" <td>245</td>\n",
" </tr>\n",
" <tr>\n",
" <th>frequency</th>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>start_date</th>\n",
" <td>123</td>\n",
" </tr>\n",
" <tr>\n",
" <th>end_date</th>\n",
" <td>61</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable_long_name</th>\n",
" <td>218</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable_standard_name</th>\n",
" <td>56</td>\n",
" </tr>\n",
" <tr>\n",
" <th>variable_cell_methods</th>\n",
" <td>5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>filename</th>\n",
" <td>9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>file_id</th>\n",
" <td>9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>derived_variable</th>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"ocean_cat"
]
},
{
"cell_type": "markdown",
"id": "dd9cfabc-386c-470e-9399-1e6642e12590",
"metadata": {},
"source": [
"Get a list of all variables contained in ocean_cat"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "22bc66f7-6fa4-49d8-939d-992f7a06c34c",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'adic',\n",
" 'age_global',\n",
" 'agm',\n",
" 'aiso_bih',\n",
" 'alk',\n",
" 'area_t',\n",
" 'area_u',\n",
" 'aredi',\n",
" 'average_DT',\n",
" 'average_T1',\n",
" 'average_T2',\n",
" 'bih_fric_u',\n",
" 'bih_fric_v',\n",
" 'bmf_u',\n",
" 'bmf_v',\n",
" 'bottom_salt',\n",
" 'bottom_temp',\n",
" 'bottom_temp_max',\n",
" 'buoyfreq2_wt',\n",
" 'bv_freq',\n",
" 'caco3',\n",
" 'det',\n",
" 'dht',\n",
" 'dic',\n",
" 'diff_cbt_s',\n",
" 'diff_cbt_t',\n",
" 'dissicos_raw',\n",
" 'drag_coeff',\n",
" 'dxt',\n",
" 'dxu',\n",
" 'dyt',\n",
" 'dyu',\n",
" 'ekman_we',\n",
" 'eta_adjust',\n",
" 'eta_global',\n",
" 'eta_nonbouss',\n",
" 'eta_t',\n",
" 'evap',\n",
" 'evap_heat',\n",
" 'fe',\n",
" 'fgco2_raw',\n",
" 'fgco2nat_raw',\n",
" 'fgo2_raw',\n",
" 'fprec',\n",
" 'fprec_melt_heat',\n",
" 'frazil_3d',\n",
" 'frazil_3d_int_z',\n",
" 'geolat_c',\n",
" 'geolat_t',\n",
" 'geolon_c',\n",
" 'geolon_t',\n",
" 'gm_eddy_ke_source',\n",
" 'grid_xt_ocean',\n",
" 'grid_xu_ocean',\n",
" 'grid_yt_ocean',\n",
" 'grid_yu_ocean',\n",
" 'hblt_max',\n",
" 'horz_bih_diss',\n",
" 'ht',\n",
" 'hu',\n",
" 'intpp_raw',\n",
" 'ke_tot',\n",
" 'kmt',\n",
" 'kmu',\n",
" 'liceht',\n",
" 'lprec',\n",
" 'lw_heat',\n",
" 'melt',\n",
" 'mh_flux',\n",
" 'mixdownslope_salt',\n",
" 'mixdownslope_temp',\n",
" 'mld',\n",
" 'mld_max',\n",
" 'mld_min',\n",
" 'mld_sq',\n",
" 'net_sfc_heating',\n",
" 'neutral',\n",
" 'neutral_diffusion_salt',\n",
" 'neutral_diffusion_temp',\n",
" 'neutral_gm_salt',\n",
" 'neutral_gm_temp',\n",
" 'neutralrho_edges',\n",
" 'no3',\n",
" 'no3os_raw',\n",
" 'nv',\n",
" 'o2',\n",
" 'o2os_raw',\n",
" 'paco2',\n",
" 'patm_t',\n",
" 'pbot_adjust',\n",
" 'pbot_t',\n",
" 'pco2',\n",
" 'pe_tot',\n",
" 'phy',\n",
" 'phyos_raw',\n",
" 'pme_net',\n",
" 'pme_river',\n",
" 'pot_rho_0',\n",
" 'pot_rho_2',\n",
" 'pot_temp',\n",
" 'potrho',\n",
" 'potrho_edges',\n",
" 'psiu',\n",
" 'psiv',\n",
" 'rho',\n",
" 'rho_dzt',\n",
" 'rhoave',\n",
" 'runoff',\n",
" 'salt',\n",
" 'salt_advection',\n",
" 'salt_eta_smooth',\n",
" 'salt_global_ave',\n",
" 'salt_int_rhodz',\n",
" 'salt_nonlocal_KPP',\n",
" 'salt_rhodzt',\n",
" 'salt_rivermix',\n",
" 'salt_sigma_diff',\n",
" 'salt_submeso',\n",
" 'salt_surface_ave',\n",
" 'salt_tendency',\n",
" 'salt_vdiffuse_diff_cbt',\n",
" 'salt_vdiffuse_diff_cbt_conv',\n",
" 'salt_vdiffuse_k33',\n",
" 'salt_vdiffuse_sbc',\n",
" 'scalar_axis',\n",
" 'sea_level',\n",
" 'sea_level_max',\n",
" 'sea_level_sq',\n",
" 'sens_heat',\n",
" 'sfc_hflux_coupler',\n",
" 'sfc_hflux_from_runoff',\n",
" 'sfc_hflux_from_water_evap',\n",
" 'sfc_hflux_from_water_prec',\n",
" 'sfc_hflux_pme',\n",
" 'sfc_salt_flux_coupler',\n",
" 'sfc_salt_flux_ice',\n",
" 'sfc_salt_flux_restore',\n",
" 'sfc_salt_flux_runoff',\n",
" 'sss',\n",
" 'sss_sq',\n",
" 'sst',\n",
" 'sst_sq',\n",
" 'st_edges_ocean',\n",
" 'st_ocean',\n",
" 'stf07',\n",
" 'stf09',\n",
" 'surface_det',\n",
" 'surface_dic',\n",
" 'surface_fe',\n",
" 'surface_pot_temp_max',\n",
" 'surface_pot_temp_min',\n",
" 'surface_zoo',\n",
" 'sw_edges_ocean',\n",
" 'sw_heat',\n",
" 'sw_ocean',\n",
" 'swflx',\n",
" 'talkos_raw',\n",
" 'tau_x',\n",
" 'tau_y',\n",
" 'temp',\n",
" 'temp_advection',\n",
" 'temp_eta_smooth',\n",
" 'temp_global_ave',\n",
" 'temp_int_rhodz',\n",
" 'temp_nonlocal_KPP',\n",
" 'temp_rhodzt',\n",
" 'temp_rivermix',\n",
" 'temp_sigma_diff',\n",
" 'temp_submeso',\n",
" 'temp_surface_ave',\n",
" 'temp_tendency',\n",
" 'temp_vdiffuse_diff_cbt',\n",
" 'temp_vdiffuse_diff_cbt_conv',\n",
" 'temp_vdiffuse_k33',\n",
" 'temp_vdiffuse_sbc',\n",
" 'temp_xflux_adv',\n",
" 'temp_xflux_adv_int_z',\n",
" 'temp_xflux_gm_int_z',\n",
" 'temp_xflux_ndiffuse_int_z',\n",
" 'temp_xflux_sigma',\n",
" 'temp_xflux_submeso_int_z',\n",
" 'temp_yflux_adv',\n",
" 'temp_yflux_adv_int_z',\n",
" 'temp_yflux_gm_int_z',\n",
" 'temp_yflux_ndiffuse_int_z',\n",
" 'temp_yflux_sigma',\n",
" 'temp_yflux_submeso_int_z',\n",
" 'time',\n",
" 'time_bounds',\n",
" 'total_mass_seawater',\n",
" 'total_net_sfc_heating',\n",
" 'total_ocean_evap',\n",
" 'total_ocean_evap_heat',\n",
" 'total_ocean_fprec',\n",
" 'total_ocean_fprec_melt_heat',\n",
" 'total_ocean_heat',\n",
" 'total_ocean_hflux_coupler',\n",
" 'total_ocean_hflux_evap',\n",
" 'total_ocean_hflux_prec',\n",
" 'total_ocean_lprec',\n",
" 'total_ocean_lw_heat',\n",
" 'total_ocean_melt',\n",
" 'total_ocean_mh_flux',\n",
" 'total_ocean_pme_river',\n",
" 'total_ocean_river',\n",
" 'total_ocean_river_heat',\n",
" 'total_ocean_runoff',\n",
" 'total_ocean_runoff_heat',\n",
" 'total_ocean_salt',\n",
" 'total_ocean_sens_heat',\n",
" 'total_ocean_sfc_salt_flux_coupler',\n",
" 'total_ocean_swflx',\n",
" 'total_ocean_swflx_vis',\n",
" 'total_volume_seawater',\n",
" 'tx_trans',\n",
" 'tx_trans_gm',\n",
" 'tx_trans_int_z',\n",
" 'tx_trans_nrho_submeso',\n",
" 'tx_trans_rho',\n",
" 'tx_trans_rho_gm',\n",
" 'tx_trans_submeso',\n",
" 'ty_trans',\n",
" 'ty_trans_gm',\n",
" 'ty_trans_int_z',\n",
" 'ty_trans_nrho_submeso',\n",
" 'ty_trans_rho',\n",
" 'ty_trans_rho_gm',\n",
" 'ty_trans_submeso',\n",
" 'tz_trans',\n",
" 'u',\n",
" 'u_dot_grad_vert_pv',\n",
" 'usq',\n",
" 'usurf',\n",
" 'v',\n",
" 'vert_pv',\n",
" 'vsq',\n",
" 'vsurf',\n",
" 'wfiform',\n",
" 'wfimelt',\n",
" 'wt',\n",
" 'xt_ocean',\n",
" 'xu_ocean',\n",
" 'yt_ocean',\n",
" 'yu_ocean',\n",
" 'zoo'}"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"set(chain.from_iterable(ocean_cat.df.variable))"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "bae0b088-2c70-4916-9ea5-220d1e55dec7",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"array(['1day', '1mon', '1yr', 'fx'], dtype=object)"
]
},
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_catalog.df.frequency.unique()"
]
},
{
"cell_type": "code",
"execution_count": 63,
"id": "9eabaa7e-4a56-4ae4-872b-c637dcc63abc",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"['xt_ocean',\n",
" 'yt_ocean',\n",
" 'st_ocean',\n",
" 'st_edges_ocean',\n",
" 'time',\n",
" 'nv',\n",
" 'adic',\n",
" 'dic',\n",
" 'alk',\n",
" 'o2',\n",
" 'no3',\n",
" 'fe',\n",
" 'phy',\n",
" 'stf09',\n",
" 'stf07',\n",
" 'zoo',\n",
" 'det',\n",
" 'caco3',\n",
" 'average_T1',\n",
" 'average_T2',\n",
" 'average_DT',\n",
" 'time_bounds']"
]
},
"execution_count": 63,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_catalog.search(frequency=\"1yr\").df.variable[0]"
]
},
{
"cell_type": "code",
"execution_count": 64,
"id": "28bb906b-5631-4532-9d50-259b5aa9fbc8",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>path</th>\n",
" <th>realm</th>\n",
" <th>variable</th>\n",
" <th>frequency</th>\n",
" <th>start_date</th>\n",
" <th>end_date</th>\n",
" <th>variable_long_name</th>\n",
" <th>variable_standard_name</th>\n",
" <th>variable_cell_methods</th>\n",
" <th>filename</th>\n",
" <th>file_id</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, st_ocean, st_edges_ocean,...</td>\n",
" <td>1yr</td>\n",
" <td>1957-12-30, 00:00:00</td>\n",
" <td>1958-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, tcell zstar ...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_ann</td>\n",
" <td>ocean_bgc_ann</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, time, nv, st_ocean, st_ed...</td>\n",
" <td>1mon</td>\n",
" <td>1957-12-30, 00:00:00</td>\n",
" <td>1958-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, time, vertex...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_mth</td>\n",
" <td>ocean_bgc_mth</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, st_ocean, st_edges_ocean,...</td>\n",
" <td>1yr</td>\n",
" <td>1958-12-30, 00:00:00</td>\n",
" <td>1959-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, tcell zstar ...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_ann</td>\n",
" <td>ocean_bgc_ann</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, time, nv, st_ocean, st_ed...</td>\n",
" <td>1mon</td>\n",
" <td>1958-12-30, 00:00:00</td>\n",
" <td>1959-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, time, vertex...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_mth</td>\n",
" <td>ocean_bgc_mth</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, st_ocean, st_edges_ocean,...</td>\n",
" <td>1yr</td>\n",
" <td>1959-12-30, 00:00:00</td>\n",
" <td>1960-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, tcell zstar ...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_ann</td>\n",
" <td>ocean_bgc_ann</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>117</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, time, nv, st_ocean, st_ed...</td>\n",
" <td>1mon</td>\n",
" <td>2015-12-30, 00:00:00</td>\n",
" <td>2016-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, time, vertex...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_mth</td>\n",
" <td>ocean_bgc_mth</td>\n",
" </tr>\n",
" <tr>\n",
" <th>118</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, st_ocean, st_edges_ocean,...</td>\n",
" <td>1yr</td>\n",
" <td>2016-12-30, 00:00:00</td>\n",
" <td>2017-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, tcell zstar ...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_ann</td>\n",
" <td>ocean_bgc_ann</td>\n",
" </tr>\n",
" <tr>\n",
" <th>119</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, time, nv, st_ocean, st_ed...</td>\n",
" <td>1mon</td>\n",
" <td>2016-12-30, 00:00:00</td>\n",
" <td>2017-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, time, vertex...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_mth</td>\n",
" <td>ocean_bgc_mth</td>\n",
" </tr>\n",
" <tr>\n",
" <th>120</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, st_ocean, st_edges_ocean,...</td>\n",
" <td>1yr</td>\n",
" <td>2017-12-30, 00:00:00</td>\n",
" <td>2018-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, tcell zstar ...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_ann</td>\n",
" <td>ocean_bgc_ann</td>\n",
" </tr>\n",
" <tr>\n",
" <th>121</th>\n",
" <td>/g/data/ik11/outputs/access-om2/1deg_jra55_iaf...</td>\n",
" <td>ocean</td>\n",
" <td>[xt_ocean, yt_ocean, time, nv, st_ocean, st_ed...</td>\n",
" <td>1mon</td>\n",
" <td>2017-12-30, 00:00:00</td>\n",
" <td>2018-12-30, 00:00:00</td>\n",
" <td>[tcell longitude, tcell latitude, time, vertex...</td>\n",
" <td>[]</td>\n",
" <td>[time: mean, time: mean, time: mean, time: mea...</td>\n",
" <td>ocean_bgc_mth</td>\n",
" <td>ocean_bgc_mth</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>122 rows × 11 columns</p>\n",
"</div>"
],
"text/plain": [
" path realm \\\n",
"0 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"1 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"2 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"3 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"4 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
".. ... ... \n",
"117 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"118 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"119 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"120 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"121 /g/data/ik11/outputs/access-om2/1deg_jra55_iaf... ocean \n",
"\n",
" variable frequency \\\n",
"0 [xt_ocean, yt_ocean, st_ocean, st_edges_ocean,... 1yr \n",
"1 [xt_ocean, yt_ocean, time, nv, st_ocean, st_ed... 1mon \n",
"2 [xt_ocean, yt_ocean, st_ocean, st_edges_ocean,... 1yr \n",
"3 [xt_ocean, yt_ocean, time, nv, st_ocean, st_ed... 1mon \n",
"4 [xt_ocean, yt_ocean, st_ocean, st_edges_ocean,... 1yr \n",
".. ... ... \n",
"117 [xt_ocean, yt_ocean, time, nv, st_ocean, st_ed... 1mon \n",
"118 [xt_ocean, yt_ocean, st_ocean, st_edges_ocean,... 1yr \n",
"119 [xt_ocean, yt_ocean, time, nv, st_ocean, st_ed... 1mon \n",
"120 [xt_ocean, yt_ocean, st_ocean, st_edges_ocean,... 1yr \n",
"121 [xt_ocean, yt_ocean, time, nv, st_ocean, st_ed... 1mon \n",
"\n",
" start_date end_date \\\n",
"0 1957-12-30, 00:00:00 1958-12-30, 00:00:00 \n",
"1 1957-12-30, 00:00:00 1958-12-30, 00:00:00 \n",
"2 1958-12-30, 00:00:00 1959-12-30, 00:00:00 \n",
"3 1958-12-30, 00:00:00 1959-12-30, 00:00:00 \n",
"4 1959-12-30, 00:00:00 1960-12-30, 00:00:00 \n",
".. ... ... \n",
"117 2015-12-30, 00:00:00 2016-12-30, 00:00:00 \n",
"118 2016-12-30, 00:00:00 2017-12-30, 00:00:00 \n",
"119 2016-12-30, 00:00:00 2017-12-30, 00:00:00 \n",
"120 2017-12-30, 00:00:00 2018-12-30, 00:00:00 \n",
"121 2017-12-30, 00:00:00 2018-12-30, 00:00:00 \n",
"\n",
" variable_long_name variable_standard_name \\\n",
"0 [tcell longitude, tcell latitude, tcell zstar ... [] \n",
"1 [tcell longitude, tcell latitude, time, vertex... [] \n",
"2 [tcell longitude, tcell latitude, tcell zstar ... [] \n",
"3 [tcell longitude, tcell latitude, time, vertex... [] \n",
"4 [tcell longitude, tcell latitude, tcell zstar ... [] \n",
".. ... ... \n",
"117 [tcell longitude, tcell latitude, time, vertex... [] \n",
"118 [tcell longitude, tcell latitude, tcell zstar ... [] \n",
"119 [tcell longitude, tcell latitude, time, vertex... [] \n",
"120 [tcell longitude, tcell latitude, tcell zstar ... [] \n",
"121 [tcell longitude, tcell latitude, time, vertex... [] \n",
"\n",
" variable_cell_methods filename \\\n",
"0 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_ann \n",
"1 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_mth \n",
"2 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_ann \n",
"3 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_mth \n",
"4 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_ann \n",
".. ... ... \n",
"117 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_mth \n",
"118 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_ann \n",
"119 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_mth \n",
"120 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_ann \n",
"121 [time: mean, time: mean, time: mean, time: mea... ocean_bgc_mth \n",
"\n",
" file_id \n",
"0 ocean_bgc_ann \n",
"1 ocean_bgc_mth \n",
"2 ocean_bgc_ann \n",
"3 ocean_bgc_mth \n",
"4 ocean_bgc_ann \n",
".. ... \n",
"117 ocean_bgc_mth \n",
"118 ocean_bgc_ann \n",
"119 ocean_bgc_mth \n",
"120 ocean_bgc_ann \n",
"121 ocean_bgc_mth \n",
"\n",
"[122 rows x 11 columns]"
]
},
"execution_count": 64,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_catalog.search(variable=\"caco3\").df"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8a5b4149-dae8-406f-8d53-9d0373485fb3",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:analysis3-unstable]",
"language": "python",
"name": "conda-env-analysis3-unstable-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.15"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"state": {},
"version_major": 2,
"version_minor": 0
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment