Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/CMG_Portal.ipynb

Created August 21, 2017 14:11
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rsignell-usgs/fb900abb8edbcb1cd7fa781beed21e4e to your computer and use it in GitHub Desktop.
Save rsignell-usgs/fb900abb8edbcb1cd7fa781beed21e4e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
CMG_Portal.ipynb
{
"cells": [
{
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-21T14:03:37.114292Z",
"start_time": "2017-08-21T14:03:36.762128Z"
},
"collapsed": true,
"trusted": false
},
"cell_type": "markdown",
"source": "# from owslib.csw import CatalogueServiceWeb\nfrom owslib import fes\nfrom pprint import pprint"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-21T14:03:37.203018Z",
"start_time": "2017-08-21T14:03:37.117035Z"
},
"trusted": false
},
"cell_type": "code",
"source": "#endpoint = 'http://geoport.whoi.edu/csw' \n#endpoint = 'http://data.nodc.noaa.gov/geoportal/csw'\n#endpoint = 'https://data.ioos.us/csw' \n#endpoint = 'https://dev-catalog.ioos.us/csw'\n\nendpoint = 'https://gamone.whoi.edu/csw'\ncsw = CatalogueServiceWeb(endpoint,timeout=60)\nprint(csw.version)",
"execution_count": 2,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": "2.0.2\n"
}
]
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-21T14:03:37.249376Z",
"start_time": "2017-08-21T14:03:37.206793Z"
},
"collapsed": true,
"trusted": false
},
"cell_type": "code",
"source": "def get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000):\n \"\"\"Iterate `maxrecords`/`pagesize` times until the requested value in\n `maxrecords` is reached.\n \"\"\"\n from owslib.fes import SortBy, SortProperty\n # Iterate over sorted results.\n sortby = SortBy([SortProperty('dc:title', 'ASC')])\n csw_records = {}\n startposition = 0\n nextrecord = getattr(csw, 'results', 1)\n while nextrecord != 0:\n csw.getrecords2(constraints=filter_list, startposition=startposition,\n maxrecords=pagesize, sortby=sortby)\n csw_records.update(csw.records)\n if csw.results['nextrecord'] == 0:\n break\n startposition += pagesize + 1 # Last one is included.\n if startposition >= maxrecords:\n break\n csw.records.update(csw_records)",
"execution_count": 3,
"outputs": []
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-21T14:03:37.635691Z",
"start_time": "2017-08-21T14:03:37.251987Z"
},
"trusted": false
},
"cell_type": "code",
"source": "val = 'CMG_Portal'\nfilter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\n\nval = 'Barnegat'\nfilter2 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\n\nfilter_list = [fes.And([filter1, filter2])]\n\nget_csw_records(csw, filter_list, maxrecords=1000)\n#csw.getrecords2(constraints=filter_list, maxrecords=1000)\nprint(len(csw.records.keys()))\nfor rec in list(csw.records.keys()):\n print('title:'+csw.records[rec].title) \n print('identifier:'+csw.records[rec].identifier)\n print('modified:'+csw.records[rec].modified)\n pprint(csw.records[rec].references[2])\n print(' ')",
"execution_count": 4,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": "5\ntitle:2010: Delineation of Water Bodies in Emergent Wetlands in Coastal New Jersey\nidentifier:580e7704e4b0f497e794b928\nmodified:2016-11-03T16:50:32Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/580e7704e4b0f497e794b928/MapServer'}\n \ntitle:COAWST Hindcast:Barnegat Bay:ADCIRC tides, real rivers, plume, lowpass Espresso bdry, NAM, new bathy\nidentifier:COAWST.Barnegat_Bay.Spring2012\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/usgs/data0/bbleh/spring2012/00_dir_roms.ncml'}\n \ntitle:Conceptual salt marsh units for wetland synthesis: Edwin B. Forsythe National Wildlife Refuge, New Jersey\nidentifier:57fe81fbe4b0824b2d148389\nmodified:2016-10-28T19:11:45Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/57fe81fbe4b0824b2d148389/MapServer/kml/mapImage.kmz'}\n \ntitle:USGS Barnegat Bay Model for Hurricane Sandy : run076\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/zdefne/run076/00_dir_roms.ncml'}\n \ntitle:USGS Barnegat Bay Model for non-Sandy conditions: run076ns\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076NS\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/zdefne/run076ns/00_dir_roms.ncml'}\n \n"
}
]
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-21T14:03:37.936187Z",
"start_time": "2017-08-21T14:03:37.639187Z"
},
"trusted": false
},
"cell_type": "code",
"source": "val = 'CMG_Portal'\nfilter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\n\nfilter_list = [filter1]\n\nget_csw_records(csw, filter_list, maxrecords=1000)\n#csw.getrecords2(constraints=filter_list, maxrecords=1000)\nprint(len(csw.records.keys()))\nfor rec in list(csw.records.keys()):\n print('title:'+csw.records[rec].title) \n print('identifier:'+csw.records[rec].identifier)\n print('modified:'+csw.records[rec].modified)\n pprint(csw.records[rec].references[2])\n print(' ')",
"execution_count": 5,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": "15\ntitle:USGS Chincoteague Bay Model for EPR\nidentifier:gov.usgs.cmg:USGS_COAWST_CHINCO_bedelevation\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/chincoteague_bedelevation/00_dir.ncml'}\n \ntitle:USGS Chincoteague Bay Model for Hurricane Sandy\nidentifier:gov.usgs.cmg:USGS_COAWST_CHINCO_SANDY\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/chinco_sandy/00_dir.ncml'}\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, NYB07 700m Nest\nidentifier:gov.usgs.cmg:USGS-CMG-COAWST-NYB07\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/jcwarner/Sandy/triple_nest/00_dir_NYB07.ncml'}\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, SHF05 100m Nest\nidentifier:gov.usgs.cmg:USGS-CMG-COAWST-SHF05\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/jcwarner/Sandy/triple_nest/00_dir_SHF05.ncml'}\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, USE31 5000m Nest\nidentifier:gov.usgs.cmg:USGS-CMG-COAWST-USE31\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/jcwarner/Sandy/triple_nest/00_dir_USE31.ncml'}\n \ntitle:2010: Delineation of Water Bodies in Emergent Wetlands in Coastal New Jersey\nidentifier:580e7704e4b0f497e794b928\nmodified:2016-11-03T16:50:32Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/580e7704e4b0f497e794b928/MapServer'}\n \ntitle:COAWST Forecast System : USGS : US East Coast and Gulf of Mexico (Experimental)\nidentifier:gov.usgs.cmgp:COAWST.USEAST.Forecast\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd'}\n \ntitle:COAWST Hindcast:Barnegat Bay:ADCIRC tides, real rivers, plume, lowpass Espresso bdry, NAM, new bathy\nidentifier:COAWST.Barnegat_Bay.Spring2012\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/usgs/data0/bbleh/spring2012/00_dir_roms.ncml'}\n \ntitle:COAWST Hindcast:MVCO/CBlast 2007:ripples with SWAN-40m res\nidentifier:COAWST.MVCO.CBLAST.spatial_7_ar0fd\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml'}\n \ntitle:Conceptual salt marsh units for wetland synthesis: Edwin B. Forsythe National Wildlife Refuge, New Jersey\nidentifier:57fe81fbe4b0824b2d148389\nmodified:2016-10-28T19:11:45Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/57fe81fbe4b0824b2d148389/MapServer/kml/mapImage.kmz'}\n \ntitle:Hudson River and Jamaica Bay Model for Hurricane Sandy\nidentifier:gov.usgs.cmg:HudJBay_Sandy\nmodified:2017-06-02\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://clancy.whoi.edu/thredds/dodsC/data1/dralston/hudson/sandy/sandy009/00_dir_roms.ncml'}\n \ntitle:Hurricane Sandy: 2013 NWI Data of Selected USFWS Wildlife Refuges along New Jersey and Long Island, New York.\nidentifier:5798c696e4b0589fa1c728a5\nmodified:2016-08-25T20:05:38Z\n{'scheme': 'OGC:WMS',\n 'url': 'https://www.sciencebase.gov/arcgis/services/Catalog/5798c696e4b0589fa1c728a5/MapServer/WMSServer?request=GetCapabilities&service=WMS'}\n \ntitle:Marsh Condition Change Map (Radar and optical mapping of surge persistence and marsh dieback along the New Jersey Mid-Atlantic coast after Hurricane Sandy, April 2016)\nidentifier:57f2aa95e4b0bc0bec001f03\nmodified:2016-10-11T15:03:16Z\n{'scheme': 'OGC:WMS',\n 'url': 'https://www.sciencebase.gov/catalogMaps/mapping/ows/57f2aa95e4b0bc0bec001f03?service=wms&request=getcapabilities&version=1.3.0'}\n \ntitle:USGS Barnegat Bay Model for Hurricane Sandy : run076\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/zdefne/run076/00_dir_roms.ncml'}\n \ntitle:USGS Barnegat Bay Model for non-Sandy conditions: run076ns\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076NS\nmodified:2017-08-21\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/zdefne/run076ns/00_dir_roms.ncml'}\n \n"
}
]
},
{
"metadata": {
"scrolled": true
},
"cell_type": "markdown",
"source": "### csw.get_operation_by_name('GetRecords').constraints"
},
{
"metadata": {
"collapsed": true,
"trusted": false
},
"cell_type": "code",
"source": "",
"execution_count": null,
"outputs": []
},
{
"metadata": {
"collapsed": true,
"trusted": false
},
"cell_type": "code",
"source": "",
"execution_count": null,
"outputs": []
},
{
"metadata": {
"collapsed": true,
"trusted": false
},
"cell_type": "code",
"source": "",
"execution_count": null,
"outputs": []
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-21T14:03:38.039047Z",
"start_time": "2017-08-21T14:03:37.939089Z"
}
},
"cell_type": "markdown",
"source": "# try:\n csw.get_operation_by_name('GetDomain')\n csw.getdomain('apiso:ServiceType', 'property')\n print(csw.results['values'])\nexcept:\n print('GetDomain not supported')"
}
],
"metadata": {
"gist_id": "",
"kernelspec": {
"name": "conda-env-IOOS3-py",
"display_name": "Python [conda env:IOOS3]",
"language": "python"
},
"language_info": {
"name": "python",
"version": "3.6.1",
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"pygments_lexer": "ipython3",
"nbconvert_exporter": "python",
"file_extension": ".py"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment