Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/comt_pycsw.ipynb

Last active August 29, 2015 14:13
Show Gist options
  • Save rsignell-usgs/58f80172a77b8182843f to your computer and use it in GitHub Desktop.
Save rsignell-usgs/58f80172a77b8182843f to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
comt_pycsw.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
usgs_pycsw.ipynb
{
"worksheets": [
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "#Use CSW to find ROMS data at COMT pycsw"
},
{
"metadata": {},
"cell_type": "code",
"input": "from owslib.csw import CatalogueServiceWeb\nfrom owslib import fes\nimport numpy as np",
"prompt_number": 1,
"outputs": [],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "markdown",
"source": "##Find model results at COMT pycsw"
},
{
"metadata": {},
"cell_type": "code",
"input": "endpoint = 'http://geoport.whoi.edu/csw' # NODC/UAF Geoportal: granule level\n#endpoint = 'http://geoport.whoi.edu:8000'\ncsw = CatalogueServiceWeb(endpoint,timeout=60)\nprint csw.version",
"prompt_number": 2,
"outputs": [
{
"output_type": "stream",
"text": "2.0.2\n",
"stream": "stdout"
}
],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "csw",
"prompt_number": 3,
"outputs": [
{
"text": "<owslib.csw.CatalogueServiceWeb at 0x7fb155dad9d0>",
"output_type": "pyout",
"metadata": {},
"prompt_number": 3
}
],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "import owslib",
"prompt_number": 4,
"outputs": [],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "csw.get_operation_by_name('GetRecords').constraints",
"prompt_number": 5,
"outputs": [
{
"text": "[Constraint: SupportedISOQueryables - ['apiso:DistanceValue', 'apiso:Abstract', 'apiso:RevisionDate', 'apiso:Subject', 'apiso:KeywordType', 'apiso:Title', 'apiso:CRS', 'apiso:PublicationDate', 'apiso:Type', 'apiso:AlternateTitle', 'apiso:BoundingBox', 'apiso:AnyText', 'apiso:ParentIdentifier', 'apiso:Modified', 'apiso:Operation', 'apiso:Format', 'apiso:TempExtent_end', 'apiso:DistanceUOM', 'apiso:OrganisationName', 'apiso:ServiceType', 'apiso:TempExtent_begin', 'apiso:ResourceLanguage', 'apiso:ServiceTypeVersion', 'apiso:OperatesOn', 'apiso:Denominator', 'apiso:HasSecurityConstraints', 'apiso:OperatesOnIdentifier', 'apiso:GeographicDescriptionCode', 'apiso:Language', 'apiso:Identifier', 'apiso:OperatesOnName', 'apiso:TopicCategory', 'apiso:CreationDate', 'apiso:CouplingType'],\n Constraint: AdditionalQueryables - ['apiso:Lineage', 'apiso:Classification', 'apiso:Creator', 'apiso:Relation', 'apiso:OtherConstraints', 'apiso:SpecificationTitle', 'apiso:ResponsiblePartyRole', 'apiso:SpecificationDateType', 'apiso:Degree', 'apiso:Contributor', 'apiso:ConditionApplyingToAccessAndUse', 'apiso:SpecificationDate', 'apiso:AccessConstraints', 'apiso:Publisher'],\n Constraint: SupportedDublinCoreQueryables - ['dc:contributor', 'dc:source', 'dc:language', 'dc:title', 'dc:subject', 'dc:creator', 'dc:type', 'ows:BoundingBox', 'dct:modified', 'dct:abstract', 'dc:relation', 'dc:date', 'dc:identifier', 'dc:publisher', 'dc:format', 'csw:AnyText', 'dc:rights']]",
"output_type": "pyout",
"metadata": {},
"prompt_number": 5
}
],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "val = 'CMG_Portal'\nfilter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),\n escapeChar='\\\\',wildCard='*',singleChar='?')\nfilter_list = [ filter1 ]",
"prompt_number": 6,
"outputs": [],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "csw.getrecords2(constraints=filter_list,maxrecords=100,esn='full')\nprint len(csw.records.keys())\nfor rec in list(csw.records.keys()):\n print csw.records[rec].title \n ",
"prompt_number": 7,
"outputs": [
{
"output_type": "stream",
"text": "7\nCOAWST Hindcast:Barnegat Bay:ADCIRC tides,Real rivers,Plume,Lowpass Espresso bdry,NAM,new bathy\nCOAWST Forecast System : USGS : US East Coast and Gulf of Mexico (Experimental)\nASGS NCFS Sandy Advisory 27\nUSGS-CMG-COAWST Model: Hurricane Sandy, NYB05 Nest\nUSGS-CMG-COAWST Model: Hurricane Sandy, SHF05 Nest\nUSGS-CMG-COAWST Model: Hurricane Sandy, USE27 Nest\nCOAWST Hindcast:MVCO/CBlast 2007:ripples with SWAN-40m res\n",
"stream": "stdout"
}
],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "choice=np.random.choice(list(csw.records.keys()))\nprint(csw.records[choice].title)\ncsw.records[choice].references",
"prompt_number": 8,
"outputs": [
{
"output_type": "stream",
"text": "USGS-CMG-COAWST Model: Hurricane Sandy, SHF05 Nest\n",
"stream": "stdout"
},
{
"text": "[{'scheme': 'WWW:LINK',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/jcwarner/Projects/Sandy/triple_nest/00_dir_SHF05.ncml.html'},\n {'scheme': 'WWW:LINK',\n 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/jcwarner/Projects/Sandy/triple_nest/00_dir_SHF05.ncml'},\n {'scheme': 'OPeNDAP:OPeNDAP',\n 'url': 'http://geoport-dev.whoi.edu/thredds/dodsC/clay/usgs/users/jcwarner/Projects/Sandy/triple_nest/00_dir_SHF05.ncml'},\n {'scheme': 'OGC:WMS',\n 'url': 'http://geoport-dev.whoi.edu/thredds/wms/clay/usgs/users/jcwarner/Projects/Sandy/triple_nest/00_dir_SHF05.ncml?service=WMS&version=1.3.0&request=GetCapabilities'},\n {'scheme': 'UNIDATA:NCSS',\n 'url': 'http://geoport-dev.whoi.edu/thredds/ncss/grid/clay/usgs/users/jcwarner/Projects/Sandy/triple_nest/00_dir_SHF05.ncml/dataset.html'},\n {'scheme': 'file',\n 'url': 'http://geoport-dev.whoi.edu/thredds/fileServer/clay/usgs/users/jcwarner/Projects/Sandy/triple_nest/00_dir_SHF05.ncml'}]",
"output_type": "pyout",
"metadata": {},
"prompt_number": 8
}
],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "markdown",
"source": "Add bounding box constraint. To specify lon,lat order for bbox (which we want to do so that we can use the same bbox with either geoportal server or pycsw requests), we need to request the bounding box specifying the CRS84 coordinate reference system. The CRS84 option is available in `pycsw 1.1.10`+. The ability to specify the `crs` in the bounding box request is available in `owslib 0.8.12`+. For more info on the bounding box problem and how it was solved, see this [pycsw issue](https://github.com/geopython/pycsw/issues/287), this [geoportal server issue](https://github.com/Esri/geoportal-server/issues/124), and this [owslib issue](https://github.com/geopython/OWSLib/issues/201)"
},
{
"metadata": {},
"cell_type": "code",
"input": "bbox = [-87.40, 34.25, -63.70, 66.70]\nbbox_filter = fes.BBox(bbox,crs='urn:ogc:def:crs:OGC:1.3:CRS84')\nfilter_list = [fes.And([filter1, bbox_filter])]\ncsw.getrecords2(constraints=filter_list, maxrecords=1000)",
"prompt_number": 9,
"outputs": [],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "print(len(csw.records.keys()))\nfor rec in list(csw.records.keys()):\n print('title:'+csw.records[rec].title) \n print('identifier:'+csw.records[rec].identifier)\n print('modified:'+csw.records[rec].modified)\n print(' ')",
"prompt_number": 10,
"outputs": [
{
"output_type": "stream",
"text": "7\ntitle:COAWST Hindcast:Barnegat Bay:ADCIRC tides,Real rivers,Plume,Lowpass Espresso bdry,NAM,new bathy\nidentifier:gov.usgs.cmgp:COAWST.Barnegat_Bay.spring2012\nmodified:2015-07-06\n \ntitle:COAWST Forecast System : USGS : US East Coast and Gulf of Mexico (Experimental)\nidentifier:gov.usgs.cmgp:COAWST.USEAST.Forecast\nmodified:2015-07-06\n \ntitle:ASGS NCFS Sandy Advisory 27\nidentifier:RENCI_ADCIRC.ASGS.NCFS.Sandy.27\nmodified:2015-05-05\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, NYB05 Nest\nidentifier:gov.usgs.cmg:USGS_COAWST_Sandy_NYB05_sim6\nmodified:2015-07-06\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, SHF05 Nest\nidentifier:gov.usgs.cmg:USGS_COAWST_Sandy_SHF05_sim6\nmodified:2015-07-06\n \ntitle:USGS-CMG-COAWST Model: Hurricane Sandy, USE27 Nest\nidentifier:gov.usgs.cmg:USGS_COAWST_Sandy_USE27_sim6\nmodified:2015-07-06\n \ntitle:COAWST Hindcast:MVCO/CBlast 2007:ripples with SWAN-40m res\nidentifier:gov.usgs.cmgp:COAWST.MVCO.CBLAST.spatial_7_ar0fd\nmodified:2015-07-09\n \n",
"stream": "stdout"
}
],
"language": "python",
"trusted": true,
"collapsed": true
},
{
"metadata": {},
"cell_type": "code",
"input": "foo=csw.records[rec]",
"prompt_number": 11,
"outputs": [],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "print foo.xml",
"prompt_number": 12,
"outputs": [
{
"output_type": "stream",
"text": "<csw:SummaryRecord xmlns:csw=\"http://www.opengis.net/cat/csw/2.0.2\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:dct=\"http://purl.org/dc/terms/\" xmlns:ows100=\"http://www.opengis.net/ows\"><dc:identifier>gov.usgs.cmgp:COAWST.MVCO.CBLAST.spatial_7_ar0fd</dc:identifier><dc:title>COAWST Hindcast:MVCO/CBlast 2007:ripples with SWAN-40m res</dc:title><dc:type>dataset</dc:type><dc:subject>Oceans; Ocean Temperature; Potential Temperature</dc:subject><dc:subject>Oceans; Salinity/Density; Salinity</dc:subject><dc:subject>Oceans; Sea Surface Topography; Sea Surface Height</dc:subject><dc:subject>Oceans; Ocean Circulation; Ocean Currents</dc:subject><dc:subject>CMG_Portal</dc:subject><dc:subject>USGS Coastal and Marine Geology Program</dc:subject><dc:subject>sea_floor_depth</dc:subject><dc:subject>water_surface_height_above_reference_datum</dc:subject><dc:subject>barotropic_x_sea_water_velocity</dc:subject><dc:subject>barotropic_y_sea_water_velocity</dc:subject><dc:subject>x_sea_water_velocity</dc:subject><dc:subject>y_sea_water_velocity</dc:subject><dc:subject>sea_water_potential_temperature</dc:subject><dc:subject>sea_water_salinity</dc:subject><dc:subject>sea_surface_wave_significant_height</dc:subject><dc:subject>ocean_s_coordinate</dc:subject><dc:subject>ocean_s_coordinate</dc:subject><dc:subject>time</dc:subject><dct:references scheme=\"WWW:LINK\">http://geoport-dev.whoi.edu/thredds/dodsC/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml.html</dct:references><dct:references scheme=\"WWW:LINK\">http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://geoport-dev.whoi.edu/thredds/dodsC/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml</dct:references><dct:references scheme=\"OPeNDAP:OPeNDAP\">http://geoport-dev.whoi.edu/thredds/dodsC/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml</dct:references><dct:references scheme=\"OGC:WMS\">http://geoport-dev.whoi.edu/thredds/wms/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml?service=WMS&amp;version=1.3.0&amp;request=GetCapabilities</dct:references><dct:references scheme=\"UNIDATA:NCSS\">http://geoport-dev.whoi.edu/thredds/ncss/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml/dataset.html</dct:references><dct:references scheme=\"file\">http://geoport-dev.whoi.edu/thredds/fileServer/usgs/data0/mvco_ce/mvco_output/spatial_7_ar0fd/00_dir_roms.ncml</dct:references><dc:relation /><dct:modified>2015-07-09</dct:modified><dct:abstract>COAWST simulation of CBLAST 2007 region south of Marthas Vineyard, including the Martha's Vineyard Coastal Observatory Region. This simulation is CASE07_AR0FD. Ripple formulation with SWAN-40m resolution wave forcing.</dct:abstract><ows100:BoundingBox crs=\"urn:x-ogc:def:crs:EPSG:6.11:4326\" dimensions=\"2\"><ows100:LowerCorner>41.29 -70.61</ows100:LowerCorner><ows100:UpperCorner>41.35 -70.5</ows100:UpperCorner></ows100:BoundingBox></csw:SummaryRecord>\n",
"stream": "stdout"
}
],
"language": "python",
"trusted": true,
"collapsed": false
},
{
"metadata": {},
"cell_type": "code",
"input": "",
"prompt_number": 12,
"outputs": [],
"language": "python",
"trusted": true,
"collapsed": false
}
],
"metadata": {}
}
],
"metadata": {
"gist_id": "58f80172a77b8182843f",
"name": "",
"signature": "sha256:5044dc58f393032588df5a5466a05b671261e94b23ae4e751a0da43f4718f7d7"
},
"nbformat": 3
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment