Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/gist:9019751

Last active August 29, 2015 13:56
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rsignell-usgs/9019751 to your computer and use it in GitHub Desktop.
Save rsignell-usgs/9019751 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
{
"metadata": {
"name": "",
"signature": "sha256:d4d2f2b3d6ec405cef4547a442d52b1f7a6a24a6494f7f67806356625800095e"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#Use CSW to find model data at NODC, NGDC, and CATALOG.DATA.GOV"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from owslib.csw import CatalogueServiceWeb\n",
"from owslib import fes\n",
"import netCDF4"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 27
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def service_urls(records,service_string='urn:x-esri:specification:ServiceType:odp:url'):\n",
" \"\"\"\n",
" Get all URLs matching a specific ServiceType \n",
" \n",
" Unfortunately these seem to differ between different CSW-ISO services.\n",
" For example, OpenDAP is specified:\n",
" NODC geoportal: 'urn:x-esri:specification:ServiceType:OPeNDAP'\n",
" NGDC geoportal: 'urn:x-esri:specification:ServiceType:odp:url'\n",
" \"\"\"\n",
"\n",
" urls=[]\n",
" for key,rec in records.iteritems():\n",
" #create a generator object, and iterate through it until the match is found\n",
" #if not found, gets the default value (here \"none\")\n",
" url = next((d['url'] for d in rec.references if d['scheme'] == service_string), None)\n",
" if url is not None:\n",
" urls.append(url)\n",
" return urls"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 28
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"filt=[]\n",
"search_text = ['roms','selfe','adcirc','ncom','hycom','fvcom']\n",
"for val in search_text:\n",
" filt.append(fes.PropertyIsLike(propertyname='apiso:AnyText', \n",
" literal=('*%s*' % val),escapeChar='\\\\',wildCard='*',singleChar='?'))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 29
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##Find model results at NODC"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"endpoint = 'http://www.nodc.noaa.gov/geoportal/csw' # NODC/UAF Geoportal: granule level\n",
"csw = CatalogueServiceWeb(endpoint,timeout=60)\n",
"print csw.version"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"2.0.2\n"
]
}
],
"prompt_number": 30
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for oper in csw.operations:\n",
" if oper.name == 'GetRecords':\n",
" print '\\nISO Queryables:\\n',oper.constraints['SupportedISOQueryables']['values']"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"\n",
"ISO Queryables:\n",
"['apiso:Subject', 'apiso:Title', 'apiso:Abstract', 'apiso:AnyText', 'apiso:Format', 'apiso:Identifier', 'apiso:Modified', 'apiso:Type', 'apiso:BoundingBox', 'apiso:CRS.Authority', 'apiso:CRS.ID', 'apiso:CRS.Version', 'apiso:RevisionDate', 'apiso:AlternateTitle', 'apiso:CreationDate', 'apiso:PublicationDate', 'apiso:OrganizationName', 'apiso:HasSecurityConstraints', 'apiso:Language', 'apiso:ResourceIdentifier', 'apiso:ParentIdentifier', 'apiso:KeywordType', 'apiso:TopicCategory', 'apiso:ResourceLanguage', 'apiso:GeographicDescriptionCode', 'apiso:Denominator', 'apiso:DistanceValue', 'apiso:DistanceUOM', 'apiso:TempExtent_begin', 'apiso:TempExtent_end', 'apiso:ServiceType', 'apiso:ServiceTypeVersion', 'apiso:Operation', 'apiso:OperatesOn', 'apiso:OperatesOnIdentifier', 'apiso:OperatesOnName', 'apiso:CouplingType']\n"
]
}
],
"prompt_number": 31
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')\n",
"len(csw.records.keys())"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 32,
"text": [
"189"
]
}
],
"prompt_number": 32
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"choice=random.choice(list(csw.records.keys()))\n",
"print choice\n",
"csw.records[choice].references"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"CM2.1U_CDAef_v1.0_jnf-r3-Atmosphere\n"
]
},
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 33,
"text": [
"[{'scheme': 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Document',\n",
" 'url': 'http://www.nodc.noaa.gov/geoportal/csw?getxml=%7B4F1E27F1-1EF4-42F6-8048-8AEC7EC60B8E%7D'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:OPeNDAP',\n",
" 'url': 'http://data1.gfdl.noaa.gov:8380/thredds3/dodsC/CM2.1U_CDJnf_v1.0_r3Atmos'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:WMS',\n",
" 'url': 'http://data1.gfdl.noaa.gov:8380/thredds3/wms/CM2.1U_CDJnf_v1.0_r3Atmos?service=WMS&version=1.3.0&request=GetCapabilities'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:WCS',\n",
" 'url': 'http://data1.gfdl.noaa.gov:8380/thredds3/wcs/CM2.1U_CDJnf_v1.0_r3Atmos?service=WCS&version=1.0.0&request=GetCapabilities'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:WCT',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://data1.gfdl.noaa.gov:8380/thredds3/dodsC/CM2.1U_CDJnf_v1.0_r3Atmos'}]"
]
}
],
"prompt_number": 33
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:OPeNDAP')\n",
"len(dap_urls)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 7,
"text": [
"189"
]
}
],
"prompt_number": 7
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Find model results at NGDC"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC/IOOS Geoportal\n",
"csw = CatalogueServiceWeb(endpoint,timeout=60)\n",
"csw.version"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 34,
"text": [
"'2.0.2'"
]
}
],
"prompt_number": 34
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for oper in csw.operations:\n",
" if oper.name == 'GetRecords':\n",
" print '\\nISO Queryables:\\n',oper.constraints['SupportedISOQueryables']['values']"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"\n",
"ISO Queryables:\n",
"['apiso:Subject', 'apiso:Title', 'apiso:Abstract', 'apiso:AnyText', 'apiso:Format', 'apiso:Identifier', 'apiso:Modified', 'apiso:Type', 'apiso:BoundingBox', 'apiso:CRS.Authority', 'apiso:CRS.ID', 'apiso:CRS.Version', 'apiso:RevisionDate', 'apiso:AlternateTitle', 'apiso:CreationDate', 'apiso:PublicationDate', 'apiso:OrganizationName', 'apiso:HasSecurityConstraints', 'apiso:Language', 'apiso:ResourceIdentifier', 'apiso:ParentIdentifier', 'apiso:KeywordType', 'apiso:TopicCategory', 'apiso:ResourceLanguage', 'apiso:GeographicDescriptionCode', 'apiso:Denominator', 'apiso:DistanceValue', 'apiso:DistanceUOM', 'apiso:TempExtent_begin', 'apiso:TempExtent_end', 'apiso:ServiceType', 'apiso:ServiceTypeVersion', 'apiso:Operation', 'apiso:OperatesOn', 'apiso:OperatesOnIdentifier', 'apiso:OperatesOnName', 'apiso:CouplingType']\n"
]
}
],
"prompt_number": 35
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')\n",
"len(csw.records.keys())"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 36,
"text": [
"200"
]
}
],
"prompt_number": 36
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"choice=random.choice(list(csw.records.keys()))\n",
"print choice\n",
"csw.records[choice].references"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"inundation_extratropical.UND_ADCIRC.2007_2D_final_run_with_waves\n"
]
},
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 38,
"text": [
"[{'scheme': 'urn:x-esri:specification:ServiceType:distribution:url',\n",
" 'url': 'http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_extratropical/UND_ADCIRC/2007_2D_final_run_with_waves.html'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:distribution:url',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_extratropical/UND_ADCIRC/2007_2D_final_run_with_waves'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:wms:url',\n",
" 'url': 'http://comt.sura.org/thredds/wms/data/comt_1_archive/inundation_extratropical/UND_ADCIRC/2007_2D_final_run_with_waves?service=WMS&version=1.3.0&request=GetCapabilities'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:wcs:url',\n",
" 'url': 'http://comt.sura.org/thredds/wcs/data/comt_1_archive/inundation_extratropical/UND_ADCIRC/2007_2D_final_run_with_waves?service=WCS&version=1.0.0&request=GetCapabilities'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:odp:url',\n",
" 'url': 'http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_extratropical/UND_ADCIRC/2007_2D_final_run_with_waves'}]"
]
}
],
"prompt_number": 38
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"#dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:OPeNDAP')\n",
"dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:odp:url')\n",
"len(dap_urls)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 44,
"text": [
"146"
]
}
],
"prompt_number": 44
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Find model data at CATALOG.DATA.GOV"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"endpoint = 'http://uat-catalog-fe-data.reisys.com/csw-all' # catalog.data.gov CSW\n",
"csw = CatalogueServiceWeb(endpoint,timeout=60)\n",
"csw.version"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 45,
"text": [
"'2.0.2'"
]
}
],
"prompt_number": 45
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for oper in csw.operations:\n",
" if oper.name == 'GetRecords':\n",
" print '\\nISO Queryables:\\n',oper.constraints['SupportedISOQueryables']['values']"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"\n",
"ISO Queryables:\n",
"['apiso:DistanceValue', 'apiso:Abstract', 'apiso:RevisionDate', 'apiso:Subject', 'apiso:KeywordType', 'apiso:Title', 'apiso:CRS', 'apiso:PublicationDate', 'apiso:Type', 'apiso:AlternateTitle', 'apiso:BoundingBox', 'apiso:AnyText', 'apiso:ParentIdentifier', 'apiso:Modified', 'apiso:Operation', 'apiso:Format', 'apiso:TempExtent_end', 'apiso:DistanceUOM', 'apiso:OrganisationName', 'apiso:ServiceType', 'apiso:TempExtent_begin', 'apiso:ResourceLanguage', 'apiso:ServiceTypeVersion', 'apiso:OperatesOn', 'apiso:Denominator', 'apiso:HasSecurityConstraints', 'apiso:OperatesOnIdentifier', 'apiso:GeographicDescriptionCode', 'apiso:Language', 'apiso:Identifier', 'apiso:OperatesOnName', 'apiso:TopicCategory', 'apiso:CreationDate', 'apiso:CouplingType']\n"
]
}
],
"prompt_number": 46
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')\n",
"len(csw.records.keys())"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 47,
"text": [
"1000"
]
}
],
"prompt_number": 47
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"choice=random.choice(list(csw.records.keys()))\n",
"print choice\n",
"csw.records[choice].references"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"tl_2013_26093_edges.shp.xml\n"
]
},
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 48,
"text": [
"[{'scheme': 'None',\n",
" 'url': 'http://www2.census.gov/geo/tiger/TIGER2013/EDGES/tl_2013_26093_edges.zip'}]"
]
}
],
"prompt_number": 48
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"From the above, we can see that because the 'scheme' is 'None' on all the references, we can't extract the different service types, like OPeNDAP, WCS, etc. "
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"#dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:odp:url') #NGDC\n",
"#dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:OPeNDAP') #NODC\n",
"dap_urls = service_urls(csw.records,service_string='?????????') #CATALOG.DATA.GOV\n",
"len(dap_urls)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 49,
"text": [
"0"
]
}
],
"prompt_number": 49
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Search at geoport.whoi.edu"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"endpoint = 'http://geoport.whoi.edu/geoportal/csw' # catalog.data.gov CSW\n",
"csw = CatalogueServiceWeb(endpoint,timeout=60)\n",
"print csw.version"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"2.0.2\n"
]
}
],
"prompt_number": 53
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for oper in csw.operations:\n",
" if oper.name == 'GetRecords':\n",
" print '\\nISO Queryables:\\n',oper.constraints['SupportedISOQueryables']['values']"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"\n",
"ISO Queryables:\n",
"['apiso:Subject', 'apiso:Title', 'apiso:Abstract', 'apiso:AnyText', 'apiso:Format', 'apiso:Identifier', 'apiso:Modified', 'apiso:Type', 'apiso:BoundingBox', 'apiso:CRS.Authority', 'apiso:CRS.ID', 'apiso:CRS.Version', 'apiso:RevisionDate', 'apiso:AlternateTitle', 'apiso:CreationDate', 'apiso:PublicationDate', 'apiso:OrganizationName', 'apiso:HasSecurityConstraints', 'apiso:Language', 'apiso:ResourceIdentifier', 'apiso:ParentIdentifier', 'apiso:KeywordType', 'apiso:TopicCategory', 'apiso:ResourceLanguage', 'apiso:GeographicDescriptionCode', 'apiso:Denominator', 'apiso:DistanceValue', 'apiso:DistanceUOM', 'apiso:TempExtent_begin', 'apiso:TempExtent_end', 'apiso:ServiceType', 'apiso:ServiceTypeVersion', 'apiso:Operation', 'apiso:OperatesOn', 'apiso:OperatesOnIdentifier', 'apiso:OperatesOnName', 'apiso:CouplingType']\n"
]
}
],
"prompt_number": 54
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')\n",
"len(csw.records.keys())"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 55,
"text": [
"77"
]
}
],
"prompt_number": 55
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"choice=random.choice(list(csw.records.keys()))\n",
"print choice\n",
"csw.records[choice].references"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"inundation_tropical.USF_FVCOM.Hurricane_Rita_2D_final_run_without_waves\n"
]
},
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 56,
"text": [
"[{'scheme': 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Onlink',\n",
" 'url': 'http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Hurricane_Rita_2D_final_run_without_waves.html'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Document',\n",
" 'url': 'http://geoport.whoi.edu:8082/geoportal/csw?getxml=%7B6FC9058F-C329-432B-BC8E-EB0FDB1058A0%7D'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:wms:url',\n",
" 'url': 'http://comt.sura.org/thredds/wms/data/comt_1_archive/inundation_tropical/USF_FVCOM/Hurricane_Rita_2D_final_run_without_waves?service=WMS&version=1.3.0&request=GetCapabilities'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:wcs:url',\n",
" 'url': 'http://comt.sura.org/thredds/wcs/data/comt_1_archive/inundation_tropical/USF_FVCOM/Hurricane_Rita_2D_final_run_without_waves?service=WCS&version=1.0.0&request=GetCapabilities'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:wct:url',\n",
" 'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Hurricane_Rita_2D_final_run_without_waves'},\n",
" {'scheme': 'urn:x-esri:specification:ServiceType:odp:url',\n",
" 'url': 'http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Hurricane_Rita_2D_final_run_without_waves'}]"
]
}
],
"prompt_number": 56
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 56
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment