Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/IOOS_catalog_new.ipynb

Last active August 1, 2018 19:48
Show Gist options
  • Save rsignell-usgs/487d51feea270a50b5d0580c3e4de5b5 to your computer and use it in GitHub Desktop.
Save rsignell-usgs/487d51feea270a50b5d0580c3e4de5b5 to your computer and use it in GitHub Desktop.
Download ZIP
notebook/CSW/CMG_Portal.ipynb
Display the source blob
Display the rendered blob
Raw
IOOS_catalog_new.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Why isn't ERDDAP:TableDap discovered in the IOOS Catalog?"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2017-05-12T10:32:22.023749",
"start_time": "2017-05-12T10:32:21.641099"
}
},
"outputs": [],
"source": [
"from owslib.csw import CatalogueServiceWeb\n",
"from owslib import fes\n",
"from pprint import pprint"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2017-05-12T10:32:22.187026",
"start_time": "2017-05-12T10:32:22.025752"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2.0.2\n"
]
}
],
"source": [
"#endpoint = 'http://geoport.whoi.edu/csw' \n",
"#endpoint = 'http://data.nodc.noaa.gov/geoportal/csw'\n",
"#endpoint = 'https://dev-catalog.ioos.us/csw'\n",
"#endpoint = 'https://gamone.whoi.edu/csw'\n",
"\n",
"endpoint = 'https://data.ioos.us/csw' \n",
"\n",
"csw = CatalogueServiceWeb(endpoint,timeout=60)\n",
"print(csw.version)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### What type of ServiceTypes is the endpoint supposed to support?"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2017-05-12T10:32:22.490541",
"start_time": "2017-05-12T10:32:22.189029"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['ERDDAP OPeNDAP',\n",
" 'Open Geospatial Consortium Web Map Service (WMS),Open Geospatial Consortium '\n",
" 'Web Feature Service (WFS)',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Sensor Observation Service '\n",
" '(SOS),THREDDS HTTP Service',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Coverage Service (WCS),Open '\n",
" 'Geospatial Consortium Sensor Observation Service (SOS),THREDDS NetCDF Subset '\n",
" 'Service',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Map Service (WMS),THREDDS '\n",
" 'NetCDF Subset Service,THREDDS HTTP Service',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Coverage Service '\n",
" '(WCS),THREDDS NetCDF Subset Service',\n",
" 'OGC:WMS,OGC:WFS',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Sensor Observation Service (SOS)',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Map Service (WMS)',\n",
" 'ERDDAP tabledap,OPeNDAP',\n",
" 'Open Geospatial Consortium Web Map Service (WMS),Open Geospatial Consortium '\n",
" 'Web Map Service - Cached (WMS-C)',\n",
" 'Open Geospatial Consortium Web Coverage Service (WCS),Open Geospatial '\n",
" 'Consortium Web Map Service (WMS)',\n",
" 'OPeNDAP:OPeNDAP,OGC:WMS,file',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Map Service (WMS),THREDDS '\n",
" 'HTTP Service',\n",
" 'Open Geospatial Consortium Web Coverage Service (WCS),Open Geospatial '\n",
" 'Consortium Web Map Service (WMS),Open Geospatial Consortium Web Map Service '\n",
" '- Cached (WMS-C)',\n",
" 'OGC:SOS',\n",
" 'ERDDAP griddap,OPeNDAP,Open Geospatial Consortium Web Map Service (WMS)',\n",
" 'THREDDS OPeNDAP',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Map Service (WMS),THREDDS '\n",
" 'NetCDF Subset Service',\n",
" 'Open Geospatial Consortium Web Feature Service (WFS),Open Geospatial '\n",
" 'Consortium Web Map Service (WMS)',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Coverage Service (WCS),Open '\n",
" 'Geospatial Consortium Web Map Service (WMS),THREDDS NetCDF Subset Service',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Sensor Observation Service '\n",
" '(SOS),THREDDS NetCDF Subset Service,THREDDS HTTP Service',\n",
" 'ERDDAP tabledap,OPeNDAP,ERDDAP Subset',\n",
" 'OPeNDAP:OPeNDAP,OGC:WMS,UNIDATA:NCSS',\n",
" 'OPeNDAP:OPeNDAP,file',\n",
" 'ERDDAP OPeNDAP,Open Geospatial Consortium Web Map Service (WMS)',\n",
" 'OPeNDAP:OPeNDAP,OGC:SOS,file',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Coverage Service (WCS),Open '\n",
" 'Geospatial Consortium Web Map Service (WMS)',\n",
" None,\n",
" 'OPeNDAP:OPeNDAP',\n",
" 'THREDDS OPeNDAP,Open Geospatial Consortium Web Coverage Service (WCS),Open '\n",
" 'Geospatial Consortium Web Map Service (WMS),THREDDS NetCDF Subset '\n",
" 'Service,THREDDS HTTP Service',\n",
" 'THREDDS OPeNDAP,THREDDS HTTP Service',\n",
" 'Open Geospatial Consortium Web Feature Service (WFS),Open Geospatial '\n",
" 'Consortium Web Map Service (WMS),Open Geospatial Consortium Web Map Service '\n",
" '- Cached (WMS-C)',\n",
" 'Open Geospatial Consortium Web Map Service (WMS)']\n"
]
}
],
"source": [
"try:\n",
" csw.get_operation_by_name('GetDomain')\n",
" csw.getdomain('apiso:ServiceType', 'property')\n",
" pprint(csw.results['values'])\n",
"except:\n",
" print('GetDomain not supported')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2017-05-12T10:32:22.501560",
"start_time": "2017-05-12T10:32:22.492544"
}
},
"outputs": [],
"source": [
"def get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000):\n",
" \"\"\"Iterate `maxrecords`/`pagesize` times until the requested value in\n",
" `maxrecords` is reached.\n",
" \"\"\"\n",
" from owslib.fes import SortBy, SortProperty\n",
" # Iterate over sorted results.\n",
" sortby = SortBy([SortProperty('dc:title', 'ASC')])\n",
" csw_records = {}\n",
" startposition = 0\n",
" nextrecord = getattr(csw, 'results', 1)\n",
" while nextrecord != 0:\n",
" csw.getrecords2(constraints=filter_list, startposition=startposition,\n",
" maxrecords=pagesize, sortby=sortby)\n",
" csw_records.update(csw.records)\n",
" if csw.results['nextrecord'] == 0:\n",
" break\n",
" startposition += pagesize + 1 # Last one is included.\n",
" if startposition >= maxrecords:\n",
" break\n",
" csw.records.update(csw_records)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Let's try a record originally harvested from THREDDS"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2017-05-12T10:32:23.409100",
"start_time": "2017-05-12T10:32:23.192733"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1\n",
"title:NECOFS (FVCOM) - Hampton - Latest Forecast\n",
"identifier:hampton_nocache\n",
"modified:2018-05-02\n",
"[{'scheme': 'WWW:LINK',\n",
" 'url': 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_HAMPTON_FORECAST.nc.html'},\n",
" {'scheme': 'WWW:LINK',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_HAMPTON_FORECAST.nc'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_HAMPTON_FORECAST.nc'},\n",
" {'scheme': 'OGC:WMS',\n",
" 'url': 'http://www.smast.umassd.edu:8080/ncWMS2/wms?service=WMS&version=1.3.0&request=GetCapabilities'},\n",
" {'scheme': 'file',\n",
" 'url': 'http://www.smast.umassd.edu:8080/thredds/fileServer/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_HAMPTON_FORECAST.nc'}]\n",
" \n"
]
}
],
"source": [
"val = 'NECOFS (FVCOM) - Hampton'\n",
"filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n",
" escapeChar='\\\\',wildCard='*',singleChar='?')\n",
"\n",
"filter_list = [filter1]\n",
"\n",
"get_csw_records(csw, filter_list, maxrecords=1000)\n",
"#csw.getrecords2(constraints=filter_list, maxrecords=1000)\n",
"print(len(csw.records.keys()))\n",
"for rec in list(csw.records.keys()):\n",
" print('title:'+csw.records[rec].title) \n",
" print('identifier:'+csw.records[rec].identifier)\n",
" print('modified:'+csw.records[rec].modified)\n",
" pprint(csw.records[rec].references)\n",
" print(' ')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Now let's try a record that was originally harvested from ERDDAP"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2017-05-12T10:32:23.190729",
"start_time": "2017-05-12T10:32:22.503563"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"6\n",
"title:ru26d-20141225T1450\n",
"identifier:ru26d-20141225T1450\n",
"modified:2018-07-30\n",
"[{'scheme': 'order',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20141225T1450.html'},\n",
" {'scheme': 'order',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20141225T1450.graph'},\n",
" {'scheme': 'ERDDAP:tabledap',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20141225T1450'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20141225T1450'},\n",
" {'scheme': 'search',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20141225T1450.subset'}]\n",
" \n",
"title:ru26d-20141225T1450\n",
"identifier:edu.rutgers.marine:ru26d-20141225T1450_f070_8f49_1646\n",
"modified:2018-07-31\n",
"[{'scheme': 'WWW:LINK',\n",
" 'url': 'http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20141225T1450/ru26d-20141225T1450.nc3.nc.html'},\n",
" {'scheme': 'WWW:LINK',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20141225T1450/ru26d-20141225T1450.nc3.nc'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20141225T1450/ru26d-20141225T1450.nc3.nc'},\n",
" {'scheme': 'OGC:SOS',\n",
" 'url': 'http://data.ioos.us/thredds/sos/deployments/rutgers/ru26d-20141225T1450/ru26d-20141225T1450.nc3.nc?service=SOS&version=1.0.0&request=GetCapabilities'},\n",
" {'scheme': 'file',\n",
" 'url': 'http://data.ioos.us/thredds/fileServer/deployments/rutgers/ru26d-20141225T1450/ru26d-20141225T1450.nc3.nc'}]\n",
" \n",
"title:ru26d-20160201T1313\n",
"identifier:ru26d-20160201T1313\n",
"modified:2018-07-30\n",
"[{'scheme': 'order',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20160201T1313.html'},\n",
" {'scheme': 'order',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20160201T1313.graph'},\n",
" {'scheme': 'ERDDAP:tabledap',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20160201T1313'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20160201T1313'},\n",
" {'scheme': 'search',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20160201T1313.subset'}]\n",
" \n",
"title:ru26d-20160201T1313\n",
"identifier:edu.rutgers.marine:ru26d-20160201T1313_f070_8f49_1646\n",
"modified:2018-07-31\n",
"[{'scheme': 'WWW:LINK',\n",
" 'url': 'http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20160201T1313/ru26d-20160201T1313.nc3.nc.html'},\n",
" {'scheme': 'WWW:LINK',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20160201T1313/ru26d-20160201T1313.nc3.nc'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20160201T1313/ru26d-20160201T1313.nc3.nc'},\n",
" {'scheme': 'OGC:SOS',\n",
" 'url': 'http://data.ioos.us/thredds/sos/deployments/rutgers/ru26d-20160201T1313/ru26d-20160201T1313.nc3.nc?service=SOS&version=1.0.0&request=GetCapabilities'},\n",
" {'scheme': 'file',\n",
" 'url': 'http://data.ioos.us/thredds/fileServer/deployments/rutgers/ru26d-20160201T1313/ru26d-20160201T1313.nc3.nc'}]\n",
" \n",
"title:ru26d-20171211T0920\n",
"identifier:ru26d-20171211T0920\n",
"modified:2018-07-30\n",
"[{'scheme': 'order',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20171211T0920.html'},\n",
" {'scheme': 'order',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20171211T0920.graph'},\n",
" {'scheme': 'ERDDAP:tabledap',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20171211T0920'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20171211T0920'},\n",
" {'scheme': 'search',\n",
" 'url': 'https://data.ioos.us/gliders/erddap/tabledap/ru26d-20171211T0920.subset'}]\n",
" \n",
"title:ru26d-20171211T0920\n",
"identifier:edu.rutgers.marine:ru26d-20171211T0920_f070_8f49_1646\n",
"modified:2018-07-31\n",
"[{'scheme': 'WWW:LINK',\n",
" 'url': 'http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20171211T0920/ru26d-20171211T0920.nc3.nc.html'},\n",
" {'scheme': 'WWW:LINK',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20171211T0920/ru26d-20171211T0920.nc3.nc'},\n",
" {'scheme': 'OPeNDAP:OPeNDAP',\n",
" 'url': 'http://data.ioos.us/thredds/dodsC/deployments/rutgers/ru26d-20171211T0920/ru26d-20171211T0920.nc3.nc'},\n",
" {'scheme': 'OGC:SOS',\n",
" 'url': 'http://data.ioos.us/thredds/sos/deployments/rutgers/ru26d-20171211T0920/ru26d-20171211T0920.nc3.nc?service=SOS&version=1.0.0&request=GetCapabilities'},\n",
" {'scheme': 'file',\n",
" 'url': 'http://data.ioos.us/thredds/fileServer/deployments/rutgers/ru26d-20171211T0920/ru26d-20171211T0920.nc3.nc'}]\n",
" \n"
]
}
],
"source": [
"val = 'ru26d'\n",
"filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n",
" escapeChar='\\\\',wildCard='*',singleChar='?')\n",
"\n",
"filter_list = [filter1]\n",
"\n",
"get_csw_records(csw, filter_list, maxrecords=1000)\n",
"#csw.getrecords2(constraints=filter_list, maxrecords=1000)\n",
"print(len(csw.records.keys()))\n",
"for rec in list(csw.records.keys()):\n",
" print('title:'+csw.records[rec].title) \n",
" print('identifier:'+csw.records[rec].identifier)\n",
" print('modified:'+csw.records[rec].modified)\n",
" pprint(csw.records[rec].references)\n",
" print(' ')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"_draft": {
"nbviewer_url": "https://gist.github.com/1cd61b798baefa8208acae7273c70b04"
},
"gist": {
"data": {
"description": "notebook/CSW/CMG_Portal.ipynb",
"public": true
},
"id": "1cd61b798baefa8208acae7273c70b04"
},
"gist_id": "",
"kernelspec": {
"display_name": "Python [conda root]",
"language": "python",
"name": "conda-root-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment