Skip to content

Instantly share code, notes, and snippets.

@naomi-henderson

naomi-henderson/gc-cmip6-readdata.ipynb

Created February 21, 2020 19:10
Show Gist options
  • Save naomi-henderson/73c64156401f0a38f7a2d8584fe6b329 to your computer and use it in GitHub Desktop.
Save naomi-henderson/73c64156401f0a38f7a2d8584fe6b329 to your computer and use it in GitHub Desktop.
Download ZIP
GC-CMIP6-ReadData.ipynb
Display the source blob
Display the rendered blob
Raw
gc-cmip6-readdata.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "GC-CMIP6-ReadData.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyP4d3p75wv80J71QAK2/hUa",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/naomi-henderson/73c64156401f0a38f7a2d8584fe6b329/gc-cmip6-readdata.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "tepzI59MGKIw",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 119
},
"outputId": "fe2e2736-f0a3-488b-c576-57cacd018056"
},
"source": [
"#! pip install gcsfs\n",
"#! pip install zarr\n",
"#! pip install cftime"
],
"execution_count": 19,
"outputs": [
{
"output_type": "stream",
"text": [
"Collecting cftime\n",
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/53/35/e2fc52247871c51590d6660e684fdc619a93a29f40e3b64894bd4f8c9041/cftime-1.1.0-cp36-cp36m-manylinux1_x86_64.whl (316kB)\n",
"\u001b[K |████████████████████████████████| 317kB 3.5MB/s \n",
"\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from cftime) (1.17.5)\n",
"Installing collected packages: cftime\n",
"Successfully installed cftime-1.1.0\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "w2LpxGz8FKrg",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"outputId": "de686132-2976-4ca1-a6fc-dcc47181f5ae"
},
"source": [
"import pandas as pd\n",
"import gcsfs\n",
"import xarray as xr\n",
"import cftime\n",
"xr.__version__"
],
"execution_count": 20,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'0.14.1'"
]
},
"metadata": {
"tags": []
},
"execution_count": 20
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "boUP0_rsFWRe",
"colab_type": "code",
"colab": {}
},
"source": [
"fs = gcsfs.GCSFileSystem(token='anon', access='read_only')"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab_type": "code",
"id": "Pt8-wQarF291",
"colab": {}
},
"source": [
"dfcat = pd.read_csv('https://cmip6.storage.googleapis.com/cmip6-zarr-consolidated-stores-noQC.csv')"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "CjlngWViFRh9",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 204
},
"outputId": "901c2369-145c-4343-cbd4-a87d4331c123"
},
"source": [
"dfcat.head()"
],
"execution_count": 23,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>activity_id</th>\n",
" <th>institution_id</th>\n",
" <th>source_id</th>\n",
" <th>experiment_id</th>\n",
" <th>member_id</th>\n",
" <th>table_id</th>\n",
" <th>variable_id</th>\n",
" <th>grid_label</th>\n",
" <th>zstore</th>\n",
" <th>dcpp_init_year</th>\n",
" <th>status</th>\n",
" <th>issue_url</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>AerChemMIP</td>\n",
" <td>BCC</td>\n",
" <td>BCC-ESM1</td>\n",
" <td>piClim-CH4</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>Amon</td>\n",
" <td>ch4</td>\n",
" <td>gn</td>\n",
" <td>gs://cmip6/AerChemMIP/BCC/BCC-ESM1/piClim-CH4/...</td>\n",
" <td>NaN</td>\n",
" <td>good</td>\n",
" <td>none</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>AerChemMIP</td>\n",
" <td>BCC</td>\n",
" <td>BCC-ESM1</td>\n",
" <td>piClim-CH4</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>Amon</td>\n",
" <td>clt</td>\n",
" <td>gn</td>\n",
" <td>gs://cmip6/AerChemMIP/BCC/BCC-ESM1/piClim-CH4/...</td>\n",
" <td>NaN</td>\n",
" <td>good</td>\n",
" <td>none</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>AerChemMIP</td>\n",
" <td>BCC</td>\n",
" <td>BCC-ESM1</td>\n",
" <td>piClim-CH4</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>Amon</td>\n",
" <td>co2</td>\n",
" <td>gn</td>\n",
" <td>gs://cmip6/AerChemMIP/BCC/BCC-ESM1/piClim-CH4/...</td>\n",
" <td>NaN</td>\n",
" <td>good</td>\n",
" <td>none</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>AerChemMIP</td>\n",
" <td>BCC</td>\n",
" <td>BCC-ESM1</td>\n",
" <td>piClim-CH4</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>Amon</td>\n",
" <td>evspsbl</td>\n",
" <td>gn</td>\n",
" <td>gs://cmip6/AerChemMIP/BCC/BCC-ESM1/piClim-CH4/...</td>\n",
" <td>NaN</td>\n",
" <td>good</td>\n",
" <td>none</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>AerChemMIP</td>\n",
" <td>BCC</td>\n",
" <td>BCC-ESM1</td>\n",
" <td>piClim-CH4</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>Amon</td>\n",
" <td>hfls</td>\n",
" <td>gn</td>\n",
" <td>gs://cmip6/AerChemMIP/BCC/BCC-ESM1/piClim-CH4/...</td>\n",
" <td>NaN</td>\n",
" <td>good</td>\n",
" <td>none</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" activity_id institution_id source_id ... dcpp_init_year status issue_url\n",
"0 AerChemMIP BCC BCC-ESM1 ... NaN good none\n",
"1 AerChemMIP BCC BCC-ESM1 ... NaN good none\n",
"2 AerChemMIP BCC BCC-ESM1 ... NaN good none\n",
"3 AerChemMIP BCC BCC-ESM1 ... NaN good none\n",
"4 AerChemMIP BCC BCC-ESM1 ... NaN good none\n",
"\n",
"[5 rows x 12 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 23
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "I1YzwPcjGUxW",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"outputId": "b4f1c27c-bc1a-4e6f-8705-2f47a1ab9f6c"
},
"source": [
"zstore = dfcat.zstore.values[0]\n",
"xr.open_zarr(fs.get_mapper(zstore),consolidated=True)"
],
"execution_count": 25,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<pre>&lt;xarray.Dataset&gt;\n",
"Dimensions: (bnds: 2, lat: 64, lon: 128, plev: 19, time: 372)\n",
"Coordinates:\n",
" * lat (lat) float64 -87.86 -85.1 -82.31 -79.53 ... 82.31 85.1 87.86\n",
" lat_bnds (lat, bnds) float64 dask.array&lt;chunksize=(64, 2), meta=np.ndarray&gt;\n",
" * lon (lon) float64 0.0 2.812 5.625 8.438 ... 348.8 351.6 354.4 357.2\n",
" lon_bnds (lon, bnds) float64 dask.array&lt;chunksize=(128, 2), meta=np.ndarray&gt;\n",
" * plev (plev) float64 1e+05 9.25e+04 8.5e+04 7e+04 ... 1e+03 500.0 100.0\n",
" * time (time) object 1850-01-16 12:00:00 ... 1880-12-16 12:00:00\n",
" time_bnds (time, bnds) object dask.array&lt;chunksize=(372, 2), meta=np.ndarray&gt;\n",
"Dimensions without coordinates: bnds\n",
"Data variables:\n",
" ch4 (time, plev, lat, lon) float32 dask.array&lt;chunksize=(160, 19, 64, 128), meta=np.ndarray&gt;\n",
"Attributes:\n",
" Conventions: CF-1.7 CMIP-6.2\n",
" activity_id: AerChemMIP\n",
" branch_method: Standard\n",
" branch_time_in_child: 0.0\n",
" branch_time_in_parent: 0.0\n",
" cmor_version: 3.3.2\n",
" comment: The experiment is same as piClim-control, but wit...\n",
" contact: Dr. Tongwen Wu(twwu@cma.gov.cn)\n",
" creation_date: 2019-07-01T23:48:58Z\n",
" data_specs_version: 01.00.27\n",
" description: AerChemMIP:piClim-CH4\n",
" experiment: pre-industrial climatological SSTs and forcing, b...\n",
" experiment_id: piClim-CH4\n",
" external_variables: areacella\n",
" forcing_index: 1\n",
" frequency: mon\n",
" further_info_url: https://furtherinfo.es-doc.org/CMIP6.BCC.BCC-ESM1...\n",
" grid: T42\n",
" grid_label: gn\n",
" history: 2019-07-01T23:48:54Z ; CMOR rewrote data to be co...\n",
" initialization_index: 1\n",
" institution: Beijing Climate Center, Beijing 100081, China\n",
" institution_id: BCC\n",
" license: CMIP6 model data produced by BCC is licensed unde...\n",
" mip_era: CMIP6\n",
" nominal_resolution: 250 km\n",
" parent_activity_id: CMIP\n",
" parent_experiment_id: piControl\n",
" parent_mip_era: CMIP6\n",
" parent_source_id: BCC-ESM1\n",
" parent_time_units: days since 1850-01-01\n",
" parent_variant_label: r1i1p1f1\n",
" physics_index: 1\n",
" product: model-output\n",
" realization_index: 1\n",
" realm: atmos\n",
" references: Model described by Tongwen Wu et al. (JGR 2013; J...\n",
" run_variant: forcing: SST,SIC,greenhouse gases,aerosol emission\n",
" source: BCC-ESM 1 (2017): aerosol: none atmos: BCC_AGC...\n",
" source_id: BCC-ESM1\n",
" source_type: AER AGCM CHEM\n",
" sub_experiment: none\n",
" sub_experiment_id: none\n",
" table_id: Amon\n",
" table_info: Creation Date:(30 July 2018) MD5:e53ff52009d0b97d...\n",
" title: BCC-ESM1 output prepared for CMIP6\n",
" tracking_id: hdl:21.14100/9639d417-ea57-45cf-b5c3-55f10af9fe75\n",
" variable_id: ch4\n",
" variant_label: r1i1p1f1\n",
" status: 2019-11-01;created;by nhn2@columbia.edu</pre>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (bnds: 2, lat: 64, lon: 128, plev: 19, time: 372)\n",
"Coordinates:\n",
" * lat (lat) float64 -87.86 -85.1 -82.31 -79.53 ... 82.31 85.1 87.86\n",
" lat_bnds (lat, bnds) float64 dask.array<chunksize=(64, 2), meta=np.ndarray>\n",
" * lon (lon) float64 0.0 2.812 5.625 8.438 ... 348.8 351.6 354.4 357.2\n",
" lon_bnds (lon, bnds) float64 dask.array<chunksize=(128, 2), meta=np.ndarray>\n",
" * plev (plev) float64 1e+05 9.25e+04 8.5e+04 7e+04 ... 1e+03 500.0 100.0\n",
" * time (time) object 1850-01-16 12:00:00 ... 1880-12-16 12:00:00\n",
" time_bnds (time, bnds) object dask.array<chunksize=(372, 2), meta=np.ndarray>\n",
"Dimensions without coordinates: bnds\n",
"Data variables:\n",
" ch4 (time, plev, lat, lon) float32 dask.array<chunksize=(160, 19, 64, 128), meta=np.ndarray>\n",
"Attributes:\n",
" Conventions: CF-1.7 CMIP-6.2\n",
" activity_id: AerChemMIP\n",
" branch_method: Standard\n",
" branch_time_in_child: 0.0\n",
" branch_time_in_parent: 0.0\n",
" cmor_version: 3.3.2\n",
" comment: The experiment is same as piClim-control, but wit...\n",
" contact: Dr. Tongwen Wu(twwu@cma.gov.cn)\n",
" creation_date: 2019-07-01T23:48:58Z\n",
" data_specs_version: 01.00.27\n",
" description: AerChemMIP:piClim-CH4\n",
" experiment: pre-industrial climatological SSTs and forcing, b...\n",
" experiment_id: piClim-CH4\n",
" external_variables: areacella\n",
" forcing_index: 1\n",
" frequency: mon\n",
" further_info_url: https://furtherinfo.es-doc.org/CMIP6.BCC.BCC-ESM1...\n",
" grid: T42\n",
" grid_label: gn\n",
" history: 2019-07-01T23:48:54Z ; CMOR rewrote data to be co...\n",
" initialization_index: 1\n",
" institution: Beijing Climate Center, Beijing 100081, China\n",
" institution_id: BCC\n",
" license: CMIP6 model data produced by BCC is licensed unde...\n",
" mip_era: CMIP6\n",
" nominal_resolution: 250 km\n",
" parent_activity_id: CMIP\n",
" parent_experiment_id: piControl\n",
" parent_mip_era: CMIP6\n",
" parent_source_id: BCC-ESM1\n",
" parent_time_units: days since 1850-01-01\n",
" parent_variant_label: r1i1p1f1\n",
" physics_index: 1\n",
" product: model-output\n",
" realization_index: 1\n",
" realm: atmos\n",
" references: Model described by Tongwen Wu et al. (JGR 2013; J...\n",
" run_variant: forcing: SST,SIC,greenhouse gases,aerosol emission\n",
" source: BCC-ESM 1 (2017): aerosol: none atmos: BCC_AGC...\n",
" source_id: BCC-ESM1\n",
" source_type: AER AGCM CHEM\n",
" sub_experiment: none\n",
" sub_experiment_id: none\n",
" table_id: Amon\n",
" table_info: Creation Date:(30 July 2018) MD5:e53ff52009d0b97d...\n",
" title: BCC-ESM1 output prepared for CMIP6\n",
" tracking_id: hdl:21.14100/9639d417-ea57-45cf-b5c3-55f10af9fe75\n",
" variable_id: ch4\n",
" variant_label: r1i1p1f1\n",
" status: 2019-11-01;created;by nhn2@columbia.edu"
]
},
"metadata": {
"tags": []
},
"execution_count": 25
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "MK6qIgKbGZ2J",
"colab_type": "code",
"colab": {}
},
"source": [
""
],
"execution_count": 0,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment