Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/CMG_Portal.ipynb

Created March 2, 2018 21:45
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rsignell-usgs/e06a680751d0ee37e925aa0adb11344b to your computer and use it in GitHub Desktop.
Save rsignell-usgs/e06a680751d0ee37e925aa0adb11344b to your computer and use it in GitHub Desktop.
Download ZIP
notebook/CSW/CMG_Portal.ipynb
Display the source blob
Display the rendered blob
Raw
CMG_Portal.ipynb
{
"cells": [
{
"metadata": {
"collapsed": true
},
"cell_type": "markdown",
"source": "#### Query CMG_Portal on gamone pycsw "
},
{
"metadata": {
"ExecuteTime": {
"start_time": "2018-03-02T21:43:26.878345Z",
"end_time": "2018-03-02T21:43:27.186548Z"
},
"trusted": true
},
"cell_type": "code",
"source": "from owslib.csw import CatalogueServiceWeb\nfrom owslib import fes\nfrom pprint import pprint",
"execution_count": 1,
"outputs": []
},
{
"metadata": {
"ExecuteTime": {
"start_time": "2018-03-02T21:43:27.190326Z",
"end_time": "2018-03-02T21:43:27.323863Z"
},
"trusted": true
},
"cell_type": "code",
"source": "#endpoint = 'http://geoport.whoi.edu/csw' \n#endpoint = 'http://data.nodc.noaa.gov/geoportal/csw'\n#endpoint = 'https://data.ioos.us/csw' \n#endpoint = 'https://dev-catalog.ioos.us/csw'\n\nendpoint = 'https://gamone.whoi.edu/csw'\ncsw = CatalogueServiceWeb(endpoint,timeout=60)\nprint(csw.version)",
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"text": "2.0.2\n",
"name": "stdout"
}
]
},
{
"metadata": {
"ExecuteTime": {
"start_time": "2018-03-02T21:43:27.327220Z",
"end_time": "2018-03-02T21:43:27.373438Z"
},
"trusted": true
},
"cell_type": "code",
"source": "def get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000):\n \"\"\"Iterate `maxrecords`/`pagesize` times until the requested value in\n `maxrecords` is reached.\n \"\"\"\n from owslib.fes import SortBy, SortProperty\n # Iterate over sorted results.\n sortby = SortBy([SortProperty('dc:title', 'ASC')])\n csw_records = {}\n startposition = 0\n nextrecord = getattr(csw, 'results', 1)\n while nextrecord != 0:\n csw.getrecords2(constraints=filter_list, startposition=startposition,\n maxrecords=pagesize, sortby=sortby)\n csw_records.update(csw.records)\n if csw.results['nextrecord'] == 0:\n break\n startposition += pagesize + 1 # Last one is included.\n if startposition >= maxrecords:\n break\n csw.records.update(csw_records)",
"execution_count": 3,
"outputs": []
},
{
"metadata": {
"ExecuteTime": {
"start_time": "2018-03-02T21:43:27.377134Z",
"end_time": "2018-03-02T21:43:27.878657Z"
},
"trusted": true
},
"cell_type": "code",
"source": "val = 'CMG_Portal'\nfilter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\n\nval = 'Barnegat'\nfilter2 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\n\nfilter_list = [fes.And([filter1, filter2])]\n\nget_csw_records(csw, filter_list, maxrecords=1000)\n#csw.getrecords2(constraints=filter_list, maxrecords=1000)\nprint(len(csw.records.keys()))\nfor rec in list(csw.records.keys()):\n print('title:'+csw.records[rec].title) \n print('identifier:'+csw.records[rec].identifier)\n print('modified:'+csw.records[rec].modified)\n pprint(csw.records[rec].references[2])\n print(' ')",
"execution_count": 4,
"outputs": [
{
"output_type": "stream",
"text": "6\ntitle:2010: Delineation of Water Bodies in Emergent Wetlands in Coastal New Jersey\nidentifier:580e7704e4b0f497e794b928\nmodified:2016-11-03T16:50:32Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/580e7704e4b0f497e794b928/MapServer'}\n \ntitle:COAWST Hindcast:Barnegat Bay:ADCIRC tides,Real rivers,Plume,Lowpass Espresso bdry,NAM,new bathy\nidentifier:gov.usgs.cmgp:COAWST.Barnegat_Bay.spring2012\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/spring2012/00_dir_roms.ncml'}\n \ntitle:Conceptual salt marsh units for wetland synthesis: Edwin B. Forsythe National Wildlife Refuge, New Jersey\nidentifier:57fe81fbe4b0824b2d148389\nmodified:2016-10-28T19:11:45Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/57fe81fbe4b0824b2d148389/MapServer/kml/mapImage.kmz'}\n \ntitle:USGS Barnegat Bay Hydrodynamic Model for March-September 2012\nidentifier:gov.usgs.cmgp.whsc:USGS_BBLEH_RUN071TRX\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/Projects/BBLEH/run071tRX/00_dir_roms.ncml'}\n \ntitle:USGS Barnegat Bay Model for Hurricane Sandy : run076\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/zdefne/run076/00_dir_roms.ncml'}\n \ntitle:USGS Barnegat Bay Model for non-Sandy conditions: run076ns\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076NS\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/zdefne/run076ns/00_dir_roms.ncml'}\n \n",
"name": "stdout"
}
]
},
{
"metadata": {
"ExecuteTime": {
"start_time": "2018-03-02T21:43:27.883658Z",
"end_time": "2018-03-02T21:43:28.326546Z"
},
"trusted": true
},
"cell_type": "code",
"source": "val = 'CMG_Portal'\nfilter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\n\nfilter_list = [filter1]\n\nget_csw_records(csw, filter_list, maxrecords=1000)\n#csw.getrecords2(constraints=filter_list, maxrecords=1000)\nprint(len(csw.records.keys()))\nfor rec in list(csw.records.keys()):\n print('title:'+csw.records[rec].title) \n print('identifier:'+csw.records[rec].identifier)\n print('modified:'+csw.records[rec].modified)\n pprint(csw.records[rec].references[2])\n print(' ')",
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
"text": "15\ntitle:USGS Chincoteague Bay Model for EPR\nidentifier:gov.usgs.cmg:USGS_COAWST_CHINCO_bedelevation\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/chincoteague_bedelevation/00_dir.ncml'}\n \ntitle:USGS Chincoteague Bay Model for Hurricane Sandy\nidentifier:gov.usgs.cmg:USGS_COAWST_CHINCO_SANDY\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/chinco_sandy/00_dir.ncml'}\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, NYB07 700m Nest\nidentifier:gov.usgs.cmg:USGS-CMG-COAWST-NYB07\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/jcwarner/Sandy/triple_nest/00_dir_NYB07.ncml'}\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, SHF05 100m Nest\nidentifier:gov.usgs.cmg:USGS-CMG-COAWST-SHF05\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/jcwarner/Sandy/triple_nest/00_dir_SHF05.ncml'}\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, USE31 5000m Nest\nidentifier:gov.usgs.cmg:USGS-CMG-COAWST-USE31\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/jcwarner/Sandy/triple_nest/00_dir_USE31.ncml'}\n \ntitle:2010: Delineation of Water Bodies in Emergent Wetlands in Coastal New Jersey\nidentifier:580e7704e4b0f497e794b928\nmodified:2016-11-03T16:50:32Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/580e7704e4b0f497e794b928/MapServer'}\n \ntitle:COAWST Forecast System : USGS : US East Coast and Gulf of Mexico (Experimental)\nidentifier:gov.usgs.cmgp:COAWST.USEAST.Forecast\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd'}\n \ntitle:COAWST Hindcast:Barnegat Bay:ADCIRC tides,Real rivers,Plume,Lowpass Espresso bdry,NAM,new bathy\nidentifier:gov.usgs.cmgp:COAWST.Barnegat_Bay.spring2012\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/spring2012/00_dir_roms.ncml'}\n \ntitle:COAWST Hindcast:MVCO/CBlast 2007:ripples with SWAN-40m res\nidentifier:gov.usgs.cmgp:COAWST.MVCO.CBLAST.spatial_7_ar0fd\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/nganju/portal_runs/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml'}\n \ntitle:Conceptual salt marsh units for wetland synthesis: Edwin B. Forsythe National Wildlife Refuge, New Jersey\nidentifier:57fe81fbe4b0824b2d148389\nmodified:2016-10-28T19:11:45Z\n{'scheme': 'WWW:LINK-1.0-http--link',\n 'url': 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/57fe81fbe4b0824b2d148389/MapServer/kml/mapImage.kmz'}\n \ntitle:Hurricane Sandy: 2013 NWI Data of Selected USFWS Wildlife Refuges along New Jersey and Long Island, New York.\nidentifier:5798c696e4b0589fa1c728a5\nmodified:2016-08-25T20:05:38Z\n{'scheme': 'OGC:WMS',\n 'url': 'https://www.sciencebase.gov/arcgis/services/Catalog/5798c696e4b0589fa1c728a5/MapServer/WMSServer?request=GetCapabilities&service=WMS'}\n \ntitle:Marsh Condition Change Map (Radar and optical mapping of surge persistence and marsh dieback along the New Jersey Mid-Atlantic coast after Hurricane Sandy, April 2016)\nidentifier:57f2aa95e4b0bc0bec001f03\nmodified:2016-10-11T15:03:16Z\n{'scheme': 'OGC:WMS',\n 'url': 'https://www.sciencebase.gov/catalogMaps/mapping/ows/57f2aa95e4b0bc0bec001f03?service=wms&request=getcapabilities&version=1.3.0'}\n \ntitle:USGS Barnegat Bay Hydrodynamic Model for March-September 2012\nidentifier:gov.usgs.cmgp.whsc:USGS_BBLEH_RUN071TRX\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/Projects/BBLEH/run071tRX/00_dir_roms.ncml'}\n \ntitle:USGS Barnegat Bay Model for Hurricane Sandy : run076\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/zdefne/run076/00_dir_roms.ncml'}\n \ntitle:USGS Barnegat Bay Model for non-Sandy conditions: run076ns\nidentifier:gov.usgs.cmg:USGS_BBLEH_SANDY_076NS\nmodified:2018-03-01\n{'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'https://gamone.whoi.edu/thredds/dodsC/sand/usgs/users/zdefne/run076ns/00_dir_roms.ncml'}\n \n",
"name": "stdout"
}
]
}
],
"metadata": {
"_draft": {
"nbviewer_url": "https://gist.github.com/58ed4e826fc6b0ea46196e4849bf2bef"
},
"gist": {
"id": "58ed4e826fc6b0ea46196e4849bf2bef",
"data": {
"description": "notebook/CSW/CMG_Portal.ipynb",
"public": true
}
},
"gist_id": "",
"kernelspec": {
"name": "conda-env-IOOS-py",
"display_name": "Python [conda env:IOOS]",
"language": "python"
},
"language_info": {
"name": "python",
"version": "3.6.4",
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"pygments_lexer": "ipython3",
"nbconvert_exporter": "python",
"file_extension": ".py"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment