Skip to content

Instantly share code, notes, and snippets.

@adyork

adyork/read_netcdf.ipynb

Last active October 20, 2022 15:58
Show Gist options
  • Save adyork/da01fd130b769a9c7707754f5e6c21ab to your computer and use it in GitHub Desktop.
Save adyork/da01fd130b769a9c7707754f5e6c21ab to your computer and use it in GitHub Desktop.
Download ZIP
read_netcdf.ipynb
Display the source blob
Display the rendered blob
Raw
read_netcdf.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "read_netcdf.ipynb",
"provenance": [],
"collapsed_sections": [],
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/adyork/da01fd130b769a9c7707754f5e6c21ab/read_netcdf.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"source": [
"# Read and Examine a netCDF file for parameter information\n",
"\n",
"Example notebook to look at contents of a netcdf file, get parameter list. With option to download a csv file with parameter information.\n",
"\n",
"Below you can change the \"url =\" to the netcdf file url you want to examine, and if using colab you can run all to get the info for that file.\n",
"\n",
"Or you can adapt this to provide the ncFile directly if you downloaded it already.\n",
"\n",
"Requirements: Parameter information in one or more fields (units, long_name, comments).\n",
"\n",
"To get the csv file download of the parameter information make sure download_csv = True below.\n",
"\n",
"Example data uses a NetCDF file from BCO-DMO\n",
"Data citation : \n",
"> Monismith, S. G., Woodson, C. B., Daly, M., Fong, D. (2022) ADCP data from Nearshore Isla Natividad, Baja California Mexico from July to August of 2018. Biological and Chemical Oceanography Data Management Office (BCO-DMO). (Version 1) Version Date 2022-03-24. http://lod.bco-dmo.org/id/dataset/872339 "
],
"metadata": {
"id": "GmiqJE023sbc"
}
},
{
"cell_type": "code",
"source": [
"import netCDF4 as nc\n",
"import os\n",
"from urllib.parse import urlparse"
],
"metadata": {
"id": "MvYnx0rgvAQe"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "jkvlr-MCut2x",
"outputId": "193db7aa-1e82-4028-eee8-9c250d602494"
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"0"
]
},
"metadata": {},
"execution_count": 2
}
],
"source": [
"# you can comment out this cell and provide the ncFile directly in the next cell\n",
"# if not using a url\n",
"\n",
"download_csv = True #if True will popup to download the parameter list for the file\n",
"\n",
"url = 'https://datadocs.bco-dmo.org/data/305/Abalone_Safe_Places/orig/monosmith_20220318/ADCP_nc_test/data/ADCP_BB15.nc'\n",
" \n",
"a = urlparse(url)\n",
"#print(a.path) #url path (not storage location) \n",
"\n",
"ncFile = os.path.basename(a.path)\n",
"\n",
"#download netcdf file\n",
"# downloads in colab to /content/filename\n",
"os.system('wget '+url+' -O '+ncFile)"
]
},
{
"cell_type": "code",
"source": [
"#ncFile = 'ADCP_BB15.nc'\n",
"print('File: '+ncFile)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "brgdcrUM6N6S",
"outputId": "09cd8ab1-22ca-4db8-ccf6-cb119dc779e0"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"File: ADCP_BB15.nc\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# get dataset from file\n",
"\n",
"dataset = nc.Dataset(ncFile)\n",
"#print header info\n",
"print(dataset)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "UGfJSJdsy4is",
"outputId": "e7ca73ca-77ee-446e-8ec7-9cb0ce2b058d"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"<class 'netCDF4._netCDF4.Dataset'>\n",
"root group (NETCDF3_CLASSIC data model, file format NETCDF3):\n",
" time_coverage_start: 26-Jul-2018 12:00:00\n",
" time_coverage_end: 14-Aug-2018 09:00:00\n",
" time_zone: MT\n",
" longitude: -115.186\n",
" latitude: 27.863\n",
" name: BB15\n",
" dimensions(sizes): z(30), time(2719)\n",
" variables(dimensions): float64 u(time, z), float64 v(time, z), float64 w(time, z), float64 P(time), float64 dnum(time), float64 unixtime(time), float64 z(z)\n",
" groups: \n"
]
}
]
},
{
"cell_type": "code",
"source": [
"#adapted from http://schubert.atmos.colostate.edu/~cslocum/netcdf_example.html (Author: Chris Slocum)\n",
"\n",
"import datetime as dt # Python standard library datetime module\n",
"import numpy as np\n",
"from netCDF4 import Dataset # http://code.google.com/p/netcdf4-python/\n",
"\n",
"def ncdump(nc_fid, verb=True):\n",
" '''\n",
" ncdump outputs dimensions, variables and their attribute information.\n",
" The information is similar to that of NCAR's ncdump utility.\n",
" ncdump requires a valid instance of Dataset.\n",
"\n",
" Parameters\n",
" ----------\n",
" nc_fid : netCDF4.Dataset\n",
" A netCDF4 dateset object\n",
" verb : Boolean\n",
" whether or not nc_attrs, nc_dims, and nc_vars are printed\n",
"\n",
" Returns\n",
" -------\n",
" nc_attrs : list\n",
" A Python list of the NetCDF file global attributes\n",
" nc_dims : list\n",
" A Python list of the NetCDF file dimensions\n",
" nc_vars : list\n",
" A Python list of the NetCDF file variables\n",
" '''\n",
" def print_ncattr(key):\n",
" \"\"\"\n",
" Prints the NetCDF file attributes for a given key\n",
"\n",
" Parameters\n",
" ----------\n",
" key : unicode\n",
" a valid netCDF4.Dataset.variables key\n",
" \"\"\"\n",
" try:\n",
" print(\"\\t\\ttype:\", repr(nc_fid.variables[key].dtype))\n",
" for ncattr in nc_fid.variables[key].ncattrs():\n",
" print('\\t\\t%s:' % ncattr,\\\n",
" repr(nc_fid.variables[key].getncattr(ncattr)))\n",
" except KeyError:\n",
" print(\"\\t\\tWARNING: %s does not contain variable attributes\" % key)\n",
"\n",
" # NetCDF global attributes\n",
" nc_attrs = nc_fid.ncattrs()\n",
" if verb:\n",
" print(\"NetCDF Global Attributes:\")\n",
" for nc_attr in nc_attrs:\n",
" print('\\t%s:' % nc_attr, repr(nc_fid.getncattr(nc_attr)))\n",
" nc_dims = [dim for dim in nc_fid.dimensions] # list of nc dimensions\n",
" # Dimension shape information.\n",
" if verb:\n",
" print(\"NetCDF dimension information:\")\n",
" for dim in nc_dims:\n",
" print(\"\\tName:\", dim )\n",
" print(\"\\t\\tsize:\", len(nc_fid.dimensions[dim]))\n",
" print_ncattr(dim)\n",
" # Variable information.\n",
" nc_vars = [var for var in nc_fid.variables] # list of nc variables\n",
" if verb:\n",
" print(\"NetCDF variable information:\")\n",
" for var in nc_vars:\n",
" if var not in nc_dims:\n",
" print('\\tName:', var)\n",
" print(\"\\t\\tdimensions:\", nc_fid.variables[var].dimensions)\n",
" print(\"\\t\\tsize:\", nc_fid.variables[var].size)\n",
" print_ncattr(var)\n",
" return nc_attrs, nc_dims, nc_vars\n"
],
"metadata": {
"id": "-MA7pnbe2YfL"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"nc_fid = Dataset(ncFile, 'r') # Dataset is the class behavior to open the file\n",
" # and create an instance of the ncCDF4 class\n",
"#get param info \n",
"nc_attrs, nc_dims, nc_vars = ncdump(nc_fid)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "bT5zEtVk2yoj",
"outputId": "eb393230-c1e1-4dde-bd5a-bd79d3195ef3"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"NetCDF Global Attributes:\n",
"\ttime_coverage_start: '26-Jul-2018 12:00:00'\n",
"\ttime_coverage_end: '14-Aug-2018 09:00:00'\n",
"\ttime_zone: 'MT'\n",
"\tlongitude: '-115.186'\n",
"\tlatitude: '27.863'\n",
"\tname: 'BB15'\n",
"NetCDF dimension information:\n",
"\tName: z\n",
"\t\tsize: 30\n",
"\t\ttype: dtype('float64')\n",
"\t\tunits: 'm'\n",
"\t\tcomment: 'vertical positive up, above bottom'\n",
"\tName: time\n",
"\t\tsize: 2719\n",
"\t\tWARNING: time does not contain variable attributes\n",
"NetCDF variable information:\n",
"\tName: u\n",
"\t\tdimensions: ('time', 'z')\n",
"\t\tsize: 81570\n",
"\t\ttype: dtype('float64')\n",
"\t\tunits: 'm/s'\n",
"\t\tcomment: 'East/West velocities'\n",
"\tName: v\n",
"\t\tdimensions: ('time', 'z')\n",
"\t\tsize: 81570\n",
"\t\ttype: dtype('float64')\n",
"\t\tunits: 'm/s'\n",
"\t\tcomment: 'North/South velocities'\n",
"\tName: w\n",
"\t\tdimensions: ('time', 'z')\n",
"\t\tsize: 81570\n",
"\t\ttype: dtype('float64')\n",
"\t\tunits: 'm/s'\n",
"\t\tcomment: 'vertical velocities, from beam 5'\n",
"\tName: P\n",
"\t\tdimensions: ('time',)\n",
"\t\tsize: 2719\n",
"\t\ttype: dtype('float64')\n",
"\t\tunits: 'dbar'\n",
"\tName: dnum\n",
"\t\tdimensions: ('time',)\n",
"\t\tsize: 2719\n",
"\t\ttype: dtype('float64')\n",
"\tName: unixtime\n",
"\t\tdimensions: ('time',)\n",
"\t\tsize: 2719\n",
"\t\ttype: dtype('float64')\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"if download_csv:\n",
" import csv,re\n",
" from google.colab import files\n",
"\n",
" param_filename = re.sub('\\.','_',ncFile)+'_params.csv'\n",
" print(\"Writing parameter info to file: \"+param_filename+'...\\n')\n",
" \n",
" #nc_dims = [dim for dim in nc_fid.dimensions]\n",
" nc_vars = [var for var in nc_fid.variables]\n",
" csv_lines = list(['param_name,param_description,units'])\n",
" \n",
" for var in nc_vars:\n",
" #print(var)\n",
" descrip = list() #will be filled with long_name and comment if exist else var name\n",
" unit = ''\n",
" #nc_fid.variables[var].dtype\n",
" if var in nc_fid.variables:\n",
" if 'units' in nc_fid.variables[var].ncattrs():\n",
" unit = nc_fid.variables[var].getncattr('units')\n",
" if 'long_name' in nc_fid.variables[var].ncattrs():\n",
" descrip.append(nc_fid.variables[var].getncattr('long_name'))\n",
" if 'comment' in nc_fid.variables[var].ncattrs():\n",
" descrip.append(nc_fid.variables[var].getncattr('comment'))\n",
"\n",
" descrip = ';'.join(descrip)\n",
"\n",
" if len(descrip) == 0:\n",
" descrip = var #use name if no description\n",
" \n",
" csv_line = var+',\"'+descrip+'\",\"'+unit+'\"'\n",
" csv_lines.append(csv_line)\n",
" \n",
" print('\\n'.join(csv_lines))\n",
"\n",
" file1 = open(param_filename, 'w')\n",
" file1.write('\\n'.join(csv_lines))\n",
" file1.close()\n",
"\n",
" files.download(param_filename)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 191
},
"id": "pRg_rBmID9Sf",
"outputId": "577e674d-b1f4-4daf-c7a5-deaff0eacac3"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Writing parameter info to file: ADCP_BB15_nc_params.csv...\n",
"\n",
"param_name,param_description,units\n",
"u,\"East/West velocities\",\"m/s\"\n",
"v,\"North/South velocities\",\"m/s\"\n",
"w,\"vertical velocities, from beam 5\",\"m/s\"\n",
"P,\"P\",\"dbar\"\n",
"dnum,\"dnum\",\"\"\n",
"unixtime,\"unixtime\",\"\"\n",
"z,\"vertical positive up, above bottom\",\"m\"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"<IPython.core.display.Javascript object>"
],
"application/javascript": [
"\n",
" async function download(id, filename, size) {\n",
" if (!google.colab.kernel.accessAllowed) {\n",
" return;\n",
" }\n",
" const div = document.createElement('div');\n",
" const label = document.createElement('label');\n",
" label.textContent = `Downloading \"${filename}\": `;\n",
" div.appendChild(label);\n",
" const progress = document.createElement('progress');\n",
" progress.max = size;\n",
" div.appendChild(progress);\n",
" document.body.appendChild(div);\n",
"\n",
" const buffers = [];\n",
" let downloaded = 0;\n",
"\n",
" const channel = await google.colab.kernel.comms.open(id);\n",
" // Send a message to notify the kernel that we're ready.\n",
" channel.send({})\n",
"\n",
" for await (const message of channel.messages) {\n",
" // Send a message to notify the kernel that we're ready.\n",
" channel.send({})\n",
" if (message.buffers) {\n",
" for (const buffer of message.buffers) {\n",
" buffers.push(buffer);\n",
" downloaded += buffer.byteLength;\n",
" progress.value = downloaded;\n",
" }\n",
" }\n",
" }\n",
" const blob = new Blob(buffers, {type: 'application/binary'});\n",
" const a = document.createElement('a');\n",
" a.href = window.URL.createObjectURL(blob);\n",
" a.download = filename;\n",
" div.appendChild(a);\n",
" a.click();\n",
" div.remove();\n",
" }\n",
" "
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"<IPython.core.display.Javascript object>"
],
"application/javascript": [
"download(\"download_d19d03ef-4016-48d1-9942-2b64931a51ee\", \"ADCP_BB15_nc_params.csv\", 235)"
]
},
"metadata": {}
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment