Last active
October 8, 2020 22:26
-
-
Save ocefpaf/ae0d650af68c0670e5f09d35c887129c to your computer and use it in GitHub Desktop.
dataset_attr
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "metadata": {}, | |
| "cell_type": "markdown", | |
| "source": "https://nbviewer.jupyter.org/gist/rsignell-usgs/77cf8dd01f1e6ffcb1260d2809954415" | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "from erddapy import ERDDAP\n\n\ne = ERDDAP(\n server=\"https://geoport.usgs.esipfed.org/erddap\",\n protocol=\"tabledap\"\n)", | |
| "execution_count": 1, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "kw = {\n \"standard_name\": \"sea_water_salinity\",\n \"min_lon\": -74.0,\n \"max_lon\": -73.0,\n \"min_lat\": 38.0,\n \"max_lat\": 39.0,\n \"min_time\": \"1974-07-10T00:00:00Z\",\n \"max_time\": \"1976-02-10T00:00:00Z\",\n \"cdm_data_type\": \"TimeSeries\",\n}", | |
| "execution_count": 2, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "import pandas as pd\n\n\nsearch_url = e.get_search_url(response=\"csv\", **kw)\nsearch = pd.read_csv(search_url)\ndsets = search[\"Dataset ID\"].values\n\nmsg = \"Found {} Datasets:\\n\\n{}\".format\nprint(msg(len(dsets), \"\\n\".join(dsets)))", | |
| "execution_count": 3, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "Found 2 Datasets:\n\n1051-A\n1061-A\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "url = e.get_info_url(\n dataset_id=\"1051-A\",\n response=\"nc\",\n)\n\nurl", | |
| "execution_count": 4, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": "'https://geoport.usgs.esipfed.org/erddap/info/1051-A/index.nc'" | |
| }, | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "import functools\n\n\nclass Dataset(ERDDAP):\n def __init__(self, server, dataset_id):\n super().__init__(server) \n self.dataset_id = dataset_id\n\n @functools.lru_cache(maxsize=None)\n def dataset_attr(self):\n url = self.get_info_url(\n dataset_id=self.dataset_id,\n response=\"csv\",\n )\n df = pd.read_csv(url)\n # The csv info url has two groups: variable and attribute.\n # Should probably validate this first.\n groups = df.groupby(\"Row Type\")\n vs = groups.get_group(\"variable\").dropna(how=\"all\", axis=1)\n variables = {k: f\"{dtype}\" for k, dtype in zip(vs[\"Variable Name\"], vs[\"Data Type\"])}\n\n attributes = {}\n attrs = groups.get_group(\"attribute\").dropna(how=\"all\", axis=1).groupby(\"Variable Name\")\n for attr, df in attrs:\n attributes.update(\n {attr: {k: f\"{dtype}: {val}\" for k, dtype, val in zip(df[\"Attribute Name\"], df[\"Data Type\"], df[\"Value\"])}}\n )\n return variables, attributes\n \n @property\n def attributes(self):\n return self.dataset_attr()[1]\n\n @property\n def data_variables(self):\n return self.dataset_attr()[0]", | |
| "execution_count": 5, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "ds = Dataset(e.server, dataset_id=\"1051-A\")", | |
| "execution_count": 6, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "ds.data_variables", | |
| "execution_count": 7, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": "{'longitude': 'float',\n 'crs': 'int',\n 'CD_310': 'float',\n 'CS_300': 'float',\n 'v_1206': 'float',\n 'rspd_4006': 'float',\n 'time': 'double',\n 'feature_type_instance': 'String',\n 'altitude': 'float',\n 'u_1205': 'float',\n 'latitude': 'float',\n 'platform': 'int'}" | |
| }, | |
| "execution_count": 7, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "ds.attributes[\"NC_GLOBAL\"][\"original_filename\"]", | |
| "execution_count": 8, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": "'String: 1051-A.cdf'" | |
| }, | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "ds.attributes[\"latitude\"]", | |
| "execution_count": 9, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": "{'_CoordinateAxisType': 'String: Lat',\n 'actual_range': 'float: 38.72222, 38.72222',\n 'axis': 'String: Y',\n 'colorBarMaximum': 'double: 90.0',\n 'colorBarMinimum': 'double: -90.0',\n 'ioos_category': 'String: Location',\n 'long_name': 'String: sensor latitude',\n 'standard_name': 'String: latitude',\n 'units': 'String: degrees_north',\n 'valid_max': 'float: 38.72222',\n 'valid_min': 'float: 38.72222'}" | |
| }, | |
| "execution_count": 9, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "scrolled": false, | |
| "trusted": true | |
| }, | |
| "cell_type": "code", | |
| "source": "ds.attributes", | |
| "execution_count": 10, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": "{'CD_310': {'_ChunkSizes': 'int: 1000',\n '_FillValue': 'float: -9999.9',\n 'actual_range': 'float: 0.0, 359.0',\n 'ancillary_variables': 'String: platform',\n 'code': 'String: R',\n 'colorBarMaximum': 'double: 359.0',\n 'colorBarMinimum': 'double: 0.0',\n 'coordinates': 'String: time z latitude longitude',\n 'coverage_content_type': 'String: physicalMeasurement',\n 'epic_code': 'int: 310',\n 'FORTRAN_format': 'String: f8.2',\n 'generic_name': 'String: vdir',\n 'grid_mapping': 'String: crs',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: Current direction',\n 'missing_value': 'float: -9999.9',\n 'platform': 'String: platform',\n 'sensor_depth': 'float: 60.0',\n 'standard_name': 'String: direction_of_sea_water_velocity',\n 'units': 'String: degree'},\n 'CS_300': {'_ChunkSizes': 'int: 1000',\n '_FillValue': 'float: -9999.9',\n 'actual_range': 'float: 0.0, 0.482547',\n 'ancillary_variables': 'String: platform',\n 'code': 'String: R',\n 'colorBarMaximum': 'double: 0.482547',\n 'colorBarMinimum': 'double: 0.0',\n 'coordinates': 'String: time z latitude longitude',\n 'coverage_content_type': 'String: physicalMeasurement',\n 'epic_code': 'int: 300',\n 'FORTRAN_format': 'String: f8.2',\n 'generic_name': 'String: vspd',\n 'grid_mapping': 'String: crs',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: Current speed',\n 'missing_value': 'float: -9999.9',\n 'platform': 'String: platform',\n 'sensor_depth': 'float: 60.0',\n 'standard_name': 'String: sea_water_speed',\n 'units': 'String: m/s'},\n 'NC_GLOBAL': {'cdm_data_type': 'String: TimeSeries',\n 'cdm_timeseries_variables': 'String: latitude, longitude, altitude, feature_type_instance',\n 'COMPOSITE': 'int: 0',\n 'contributor_name': 'String: B. Butman',\n 'contributor_role': 'String: principalInvestigator',\n 'Conventions': 'String: CF-1.6,ACDD-1.3, COARDS',\n 'COORD_SYSTEM': 'String: GEOGRAPHICAL',\n 'CREATION_DATE': 'String: 18:30 24-APR-97',\n 'creator_email': 'String: rsignell@usgs.gov',\n 'creator_name': 'String: Rich Signell',\n 'creator_phone': 'String: +1 (508) 548-8700',\n 'creator_url': 'String: https://www.usgs.gov',\n 'cycles': 'int: 9759',\n 'DATA_CMNT': 'String: M.A. B. TIME IN E.S.T. ? MV APPLIED.\\\\r\\\\n',\n 'DATA_ORIGIN': 'String: USGS/WHFC',\n 'DATA_TYPE': 'String: TIME',\n 'date_created': 'String: 2017-04-11T22:07:00Z',\n 'date_issued': 'String: 2017-04-11T22:07:00Z',\n 'date_metadata_modified': 'String: 2017-04-11T22:07:00Z',\n 'date_modified': 'String: 2017-04-11T22:07:00Z',\n 'days': 'int: 67',\n 'DELTA_T': 'String: 10.00 minutes\\\\r\\\\n',\n 'DEPTH_CONST': 'int: 0',\n 'DESCRIPT': 'String: ?',\n 'DRIFTER': 'int: 0',\n 'Easternmost_Easting': 'double: -73.63333',\n 'EXPERIMENT': 'String: ?',\n 'featureType': 'String: TimeSeries',\n 'FILL_FLAG': 'int: 0',\n 'geospatial_bounds': 'String: POINT(-73.63333129882812 38.72222137451172)',\n 'geospatial_bounds_crs': 'String: EPSG:4326',\n 'geospatial_lat_max': 'double: 38.72222',\n 'geospatial_lat_min': 'double: 38.72222',\n 'geospatial_lat_resolution': 'long: 0',\n 'geospatial_lat_units': 'String: degrees_north',\n 'geospatial_lon_max': 'double: -73.63333',\n 'geospatial_lon_min': 'double: -73.63333',\n 'geospatial_lon_resolution': 'long: 0',\n 'geospatial_lon_units': 'String: degrees_east',\n 'geospatial_vertical_max': 'double: -60.0',\n 'geospatial_vertical_min': 'double: -60.0',\n 'geospatial_vertical_positive': 'String: up',\n 'geospatial_vertical_resolution': 'String: 0',\n 'geospatial_vertical_units': 'String: m',\n 'history': \"String: Fri Nov 1 20:15:55 2019: ncatted -a project,global,a,c,, CMG_Portal MAB_SED/1051-A.cdf\\\\nOriginal file creation date was 18:22 MAY 13,'83\\\\r\\\\nConverted to EPIC/NetCDF Mon Dec 9 08:52:43 1996\\\\n\\\\nOriginal ascii file was a1051x01.ascChanged epic code 310 to 310 on Thu Apr 24 18:10:54 1997\\\\nChanged epic code 300 to 300 on Thu Apr 24 18:10:54 1997\\\\nChanged epic code 5018 to 4006 on Thu Apr 24 18:10:54 1997\\\\n\\\\n2017-04-11T22:07:00Z - pyaxiom - File created using pyaxiom\",\n 'id': 'String: 1051-A',\n 'infoUrl': 'String: https://stellwagen.er.usgs.gov/',\n 'institution': 'String: USGS Coastal and Marine Geology Program',\n 'keywords': 'String: CD_310, crs, CS_300, feature_type_instance, latitude, longitude, platform, rspd_4006, time, u_1205, v_1206, z',\n 'keywords_vocabulary': 'String: GCMD Science Keywords',\n 'latitude': 'float: 38.72222',\n 'license': 'String: The data may be used and redistributed for free but is not intended\\\\nfor legal use, since it may contain inaccuracies. Neither the data\\\\nContributor, ERD, NOAA, nor the United States Government, nor any\\\\nof their employees or contractors, makes any warranty, express or\\\\nimplied, including warranties of merchantability and fitness for a\\\\nparticular purpose, or assumes any legal liability for the accuracy,\\\\ncompleteness, or usefulness, of this information.',\n 'longitude': 'float: -73.63333',\n 'magnetic_variation(deg)': 'float: -11.0',\n 'MOORING': 'long: 105',\n 'naming_authority': 'String: gov.usgs.cmgp',\n 'ncei_template_version': 'String: NCEI_NetCDF_TimeSeries_Orthogonal_Template_v2.0',\n 'NCO': 'String: netCDF Operators version 4.8.1 (Homepage = http://nco.sf.net, Code = https://github.com/nco/nco)',\n 'Northernmost_Northing': 'double: 38.72222',\n 'original_filename': 'String: 1051-A.cdf',\n 'original_folder': 'String: MAB_SED',\n 'POS_CONST': 'int: 0',\n 'PROJECT': 'String: ?',\n 'project': 'String: U.S. Geological Survey Oceanographic Time-Series Data, CMG_Portal',\n 'project_summary': 'String: A series of studies to assess environmental hazards to petroleum development in the Middle Atlantic Bight.',\n 'project_title': 'String: Sediment Transport Observations in the Middle Atlantic Bight',\n 'publisher_email': 'String: emontgomery@usgs.gov',\n 'publisher_name': 'String: Ellyn Montgomery',\n 'publisher_phone': 'String: +1 (508) 548-8700',\n 'publisher_url': 'String: https://www.usgs.gov',\n 'source': 'String: USGS',\n 'sourceUrl': 'String: (local files)',\n 'Southernmost_Northing': 'double: 38.72222',\n 'standard_name_vocabulary': 'String: CF Standard Name Table v29',\n 'start_time': 'String: 11-Dec-1975 18:50:00',\n 'stop_time': 'String: 17-Feb-1976 13:10:00',\n 'subsetVariables': 'String: latitude, longitude, altitude, feature_type_instance',\n 'summary': 'String: USGS-CMG time-series data from the Sediment Transport Observations in the Middle Atlantic Bight project, mooring 105 and package 1051-A. A series of studies to assess environmental hazards to petroleum development in the Middle Atlantic Bight.',\n 'time_coverage_duration': 'String: PT5854800S',\n 'time_coverage_end': 'String: 1976-02-17T13:10:00Z',\n 'time_coverage_start': 'String: 1975-12-11T18:50:00Z',\n 'title': 'String: USGS-CMG time-series data: MAB_SED - 105 - 1051-A',\n 'VAR_DESC': 'String: u v spd ',\n 'WATER_DEPTH': 'int: 60',\n 'WATER_MASS': 'String: ?',\n 'Westernmost_Easting': 'double: -73.63333',\n 'WHOI_Buoy_Group_summary': \"String: 1051-A start time = 11 Dec 1975 13:50:00 cycles = 9759\\\\n stop time = 17 Feb 1976 08:10:00 # days = 67\\\\nExpt. = 'USGS' sampling interval = 10.00 minutes\\\\nLat = 38.722221\\\\nLon = -73.633331 File created: 18:22 MAY 13,'83\\\\ndepth = 60m Mag.var = -11.000000\\\\n# Variable Units Codes Depth Inst. Minimum Maximum\\\\n-- ------------ ------------ ----- ------- ------ --------- ---------\\\\n 1 TIME SECONDS T \\\\n 2 U.1 CMS/SEC R 60.00 -43.144 28.417\\\\n 3 V.1 CMS/SEC R 60.00 -43.799 42.205\\\\n 4 VDIR.1 DEG R 60.00 0.000 359.000\\\\n 5 VSPD.1 CMS/SEC R 60.00 0.000 48.255\\\\n 6 RAW SPEED MM/SEC H 60.00 0.000 484.000\\\\n---------------------------------------------------------------------\\\\n Comments: REDUCE\\\\n M.A. B. TIME IN E.S.T. ? MV APPLIED.\\\\n VARIABLES LISTED IN ORDER SHOWN ABOVE\\\\n\"},\n 'altitude': {'_CoordinateAxisType': 'String: Height',\n '_CoordinateZisPositive': 'String: up',\n '_FillValue': 'float: -9999.9',\n 'actual_range': 'float: -60.0, -60.0',\n 'axis': 'String: Z',\n 'colorBarMaximum': 'double: 8000.0',\n 'colorBarMinimum': 'double: -8000.0',\n 'grid_mapping': 'String: crs',\n 'ioos_category': 'String: Location',\n 'long_name': 'String: z of the sensor relative to the water surface',\n 'positive': 'String: up',\n 'standard_name': 'String: altitude',\n 'units': 'String: m',\n 'valid_max': 'float: -60.0',\n 'valid_min': 'float: -60.0'},\n 'crs': {'actual_range': 'int: -2147483647, -2147483647',\n 'epsg_code': 'String: EPSG:4326',\n 'grid_mapping_name': 'String: latitude_longitude',\n 'inverse_flattening': 'double: 298.257223563',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: http://www.opengis.net/def/crs/EPSG/0/4326',\n 'semi_major_axis': 'double: 6378137.0'},\n 'feature_type_instance': {'cf_role': 'String: timeseries_id',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: Identifier for each feature type instance'},\n 'latitude': {'_CoordinateAxisType': 'String: Lat',\n 'actual_range': 'float: 38.72222, 38.72222',\n 'axis': 'String: Y',\n 'colorBarMaximum': 'double: 90.0',\n 'colorBarMinimum': 'double: -90.0',\n 'ioos_category': 'String: Location',\n 'long_name': 'String: sensor latitude',\n 'standard_name': 'String: latitude',\n 'units': 'String: degrees_north',\n 'valid_max': 'float: 38.72222',\n 'valid_min': 'float: 38.72222'},\n 'longitude': {'_CoordinateAxisType': 'String: Lon',\n 'actual_range': 'float: -73.63333, -73.63333',\n 'axis': 'String: X',\n 'colorBarMaximum': 'double: 180.0',\n 'colorBarMinimum': 'double: -180.0',\n 'ioos_category': 'String: Location',\n 'long_name': 'String: sensor longitude',\n 'standard_name': 'String: longitude',\n 'units': 'String: degrees_east',\n 'valid_max': 'float: -73.63333',\n 'valid_min': 'float: -73.63333'},\n 'platform': {'actual_range': 'int: -2147483647, -2147483647',\n 'definition': 'String: http://mmisw.org/ont/ioos/definition/stationID',\n 'ioos_category': 'String: Unknown',\n 'ioos_code': 'String: 1051-A',\n 'long_name': 'String: USGS-CMG time-series data from the Sediment Transport Observations in the Middle Atlantic Bight project, mooring 105 and package 1051-A. A series of studies to assess environmental hazards to petroleum development in the Middle Atlantic Bight.',\n 'nodc_name': 'String: FIXED PLATFORM, MOORINGS',\n 'short_name': 'String: USGS-CMG time-series data: MAB_SED - 105 - 1051-A',\n 'type': 'String: fixed'},\n 'rspd_4006': {'_ChunkSizes': 'int: 1000',\n '_FillValue': 'float: -9999.9',\n 'actual_range': 'float: 0.0, 0.0',\n 'ancillary_variables': 'String: platform',\n 'code': 'String: H',\n 'coordinates': 'String: time z latitude longitude',\n 'coverage_content_type': 'String: physicalMeasurement',\n 'epic_code': 'int: 4006',\n 'FORTRAN_format': 'String: f8.2',\n 'generic_name': 'String: speed',\n 'grid_mapping': 'String: crs',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: ROTOR SPEED ',\n 'missing_value': 'float: -9999.9',\n 'platform': 'String: platform',\n 'sensor_depth': 'float: 60.0',\n 'units': 'String: cm s-1'},\n 'time': {'_ChunkSizes': 'int: 1000',\n '_CoordinateAxisType': 'String: Time',\n 'actual_range': 'double: 1.875558E8, 1.934106E8',\n 'axis': 'String: T',\n 'calendar': 'String: gregorian',\n 'ioos_category': 'String: Time',\n 'long_name': 'String: time of measurement',\n 'standard_name': 'String: time',\n 'time_origin': 'String: 01-JAN-1970 00:00:00',\n 'units': 'String: seconds since 1970-01-01T00:00:00Z'},\n 'u_1205': {'_ChunkSizes': 'int: 1000',\n '_FillValue': 'float: -9999.9',\n 'actual_range': 'float: -0.431436, 0.284169',\n 'ancillary_variables': 'String: platform',\n 'code': 'String: R',\n 'colorBarMaximum': 'double: 0.284169',\n 'colorBarMinimum': 'double: -0.431436',\n 'coordinates': 'String: time z latitude longitude',\n 'coverage_content_type': 'String: physicalMeasurement',\n 'epic_code': 'int: 1205',\n 'generic_name': 'String: u',\n 'grid_mapping': 'String: crs',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: Eastward (u) velocity',\n 'missing_value': 'float: -9999.9',\n 'platform': 'String: platform',\n 'sensor_depth': 'float: 60.0',\n 'standard_name': 'String: eastward_sea_water_velocity',\n 'units': 'String: m/s'},\n 'v_1206': {'_ChunkSizes': 'int: 1000',\n '_FillValue': 'float: -9999.9',\n 'actual_range': 'float: -0.437987, 0.422045',\n 'ancillary_variables': 'String: platform',\n 'code': 'String: R',\n 'colorBarMaximum': 'double: 0.422045',\n 'colorBarMinimum': 'double: -0.437987',\n 'coordinates': 'String: time z latitude longitude',\n 'coverage_content_type': 'String: physicalMeasurement',\n 'epic_code': 'int: 1206',\n 'generic_name': 'String: v',\n 'grid_mapping': 'String: crs',\n 'ioos_category': 'String: Unknown',\n 'long_name': 'String: Northward (v) velocity',\n 'missing_value': 'float: -9999.9',\n 'platform': 'String: platform',\n 'sensor_depth': 'float: 60.0',\n 'standard_name': 'String: northward_sea_water_velocity',\n 'units': 'String: m/s'}}" | |
| }, | |
| "execution_count": 10, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "_draft": { | |
| "nbviewer_url": "https://gist.github.com/ae0d650af68c0670e5f09d35c887129c" | |
| }, | |
| "gist": { | |
| "id": "ae0d650af68c0670e5f09d35c887129c", | |
| "data": { | |
| "description": "dataset_attr", | |
| "public": true | |
| } | |
| }, | |
| "kernelspec": { | |
| "name": "python3", | |
| "display_name": "Python 3", | |
| "language": "python" | |
| }, | |
| "language_info": { | |
| "name": "python", | |
| "version": "3.8.5", | |
| "mimetype": "text/x-python", | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "pygments_lexer": "ipython3", | |
| "nbconvert_exporter": "python", | |
| "file_extension": ".py" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 4 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment