Skip to content

Instantly share code, notes, and snippets.

@rajadain

rajadain/bigcz-cocorahs-fail-2-years.ipynb

Created October 30, 2017 15:35
Show Gist options
  • Save rajadain/a03c6009b0ef80d1b3326e41c3c174af to your computer and use it in GitHub Desktop.
Save rajadain/a03c6009b0ef80d1b3326e41c3c174af to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
bigcz-cocorahs-fail-2-years.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import ulmo\n",
"import datetime\n",
"import dateutil.parser\n",
"\n",
"from suds.client import Client\n",
"\n",
"client = Client('http://hiscentral.cuahsi.org/webservices/hiscentral.asmx?WSDL', timeout=5)\n",
"search_params = {\n",
" 'xmin': -75.2585830259,\n",
" 'xmax': -75.1597594229,\n",
" 'ymin': 39.8760546985,\n",
" 'ymax': 40.0310812816,\n",
" 'conceptKeyword': '',\n",
" 'networkIDs': '',\n",
" 'beginDate': '',\n",
" 'endDate': ''\n",
"}\n",
"\n",
"site = 'CocoRaHs:US1PAPH0008'\n",
"variable = 'COCORAHS:PRCP'"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(SeriesRecord){\n",
" ServCode = \"CocoRaHs\"\n",
" ServURL = \"http://hydroportal.cuahsi.org/CocoRaHs/cuahsi_1_1.asmx\"\n",
" location = \"CocoRaHs:US1PAPH0008\"\n",
" VarCode = \"COCORAHS:PRCP\"\n",
" VarName = \"Precipitation\"\n",
" beginDate = \"2011-01-01T00:00:00Z\"\n",
" endDate = \"2016-12-31T00:00:00Z\"\n",
" ValueCount = 2192\n",
" Sitename = \"PHILADELPHIA 2.7 WNW\"\n",
" latitude = 40.025\n",
" longitude = -75.18\n",
" datatype = \"Incremental\"\n",
" valuetype = \"Field Observation\"\n",
" samplemedium = \"Precipitation\"\n",
" timeunits = \"day\"\n",
" conceptKeyword = \"Precipitation\"\n",
" genCategory = \"Climate\"\n",
" TimeSupport = \"1\"\n",
" }"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"result = client.service.GetSeriesCatalogForBox2(**search_params)\n",
"sample = [x for x in result['SeriesRecord'] if x.location == site and x.VarCode == variable][0]\n",
"\n",
"sample"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(2011-01-01T00:00:00Z, 2016-12-31T00:00:00Z)"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sample.beginDate, sample.endDate"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'variable': {'code': 'PRCP',\n",
" 'data_type': 'Incremental',\n",
" 'general_category': 'Climate',\n",
" 'id': None,\n",
" 'name': 'Precipitation',\n",
" 'no_data_value': '-9999',\n",
" 'sample_medium': 'Precipitation',\n",
" 'speciation': 'Not Applicable',\n",
" 'time': {'interval': '1',\n",
" 'is_regular': True,\n",
" 'units': {'abbreviation': 'd',\n",
" 'code': '104',\n",
" 'name': 'day',\n",
" 'type': 'Time'}},\n",
" 'units': {'abbreviation': 'mm',\n",
" 'code': '54',\n",
" 'name': 'millimeter',\n",
" 'type': 'Length'},\n",
" 'value_type': 'Field Observation',\n",
" 'vocabulary': 'COCORAHS'},\n",
" '{http://www.cuahsi.org/water_ml/1.1/}method': {'method_code': '0',\n",
" 'method_description': 'No method specified'},\n",
" '{http://www.cuahsi.org/water_ml/1.1/}quality_control_level': {'definition': 'Quality controlled data',\n",
" 'explanation': 'Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures.',\n",
" 'quality_control_level_code': '1'},\n",
" '{http://www.cuahsi.org/water_ml/1.1/}source': {'citation': 'Community Collaborative Rain, Hail and Snow Network (www.cocorahs.org)',\n",
" 'contact_name': 'CoCoRaHS',\n",
" 'email': 'info@cocorahs.org',\n",
" 'organization': 'Community Collaborative Rain, Hail and Snow Network',\n",
" 'source_code': '1',\n",
" 'source_description': 'Community Collaborative Rain, Hail and Snow Network',\n",
" 'source_link': 'www.cocorahs.org'},\n",
" '{http://www.cuahsi.org/water_ml/1.1/}value_count': {'value_count': '2192'},\n",
" '{http://www.cuahsi.org/water_ml/1.1/}variable_time_interval': {'begin_date_time': '2011-01-01T00:00:00',\n",
" 'end_date_time': '2016-12-31T00:00:00',\n",
" 'variable_time_interval_type': 'TimeIntervalType'}}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"siteinfo = ulmo.cuahsi.wof.get_site_info(sample.ServURL + '?WSDL', sample.location)\n",
"siteinfo['series'][variable]"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('2011-01-01T00:00:00', '2016-12-31T00:00:00')"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"variable_time_interval = siteinfo['series'][variable]['{http://www.cuahsi.org/water_ml/1.1/}variable_time_interval']\n",
"variable_time_interval['begin_date_time'], variable_time_interval['end_date_time']"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(datetime.datetime(2016, 12, 31, 0, 0), datetime.datetime(2015, 1, 1, 0, 0))"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Set to_date to the end_date_time received above\n",
"to_date = dateutil.parser.parse(variable_time_interval['end_date_time'])\n",
"\n",
"# Set from_date to the latest of either begin_date_time from above, or one month from end_date_time\n",
"from_date = max((to_date - datetime.timedelta(days=730)),\n",
" dateutil.parser.parse(variable_time_interval['begin_date_time']))\n",
"\n",
"to_date, from_date"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"ename": "KeyError",
"evalue": "'id'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-14-6ce0019f2912>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mfrom_date\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%m/%d/%Y'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mto_date\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%m/%d/%Y'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m None)\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'values'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Users/ttuhinanshu/scratch/model-my-watershed/ulmo/env/lib/python2.7/site-packages/ulmo/cuahsi/wof/core.pyc\u001b[0m in \u001b[0;36mget_values\u001b[0;34m(wsdl_url, site_code, variable_code, start, end, suds_cache)\u001b[0m\n\u001b[1;32m 183\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaterml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1_0\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_site_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mwaterml_version\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'1.1'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaterml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_site_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mvariable_code\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Users/ttuhinanshu/scratch/model-my-watershed/ulmo/env/lib/python2.7/site-packages/ulmo/waterml/v1_1.pyc\u001b[0m in \u001b[0;36mparse_site_values\u001b[0;34m(content_io, query_isodate, methods)\u001b[0m\n\u001b[1;32m 14\u001b[0m return common.parse_site_values(\n\u001b[1;32m 15\u001b[0m \u001b[0mcontent_io\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mWATERML_V1_1_NAMESPACE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery_isodate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery_isodate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m methods=methods)\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Users/ttuhinanshu/scratch/model-my-watershed/ulmo/env/lib/python2.7/site-packages/ulmo/waterml/common.pyc\u001b[0m in \u001b[0;36mparse_site_values\u001b[0;34m(content_io, namespace, query_isodate, methods)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'values'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m metadata = _parse_metadata(\n\u001b[0;32m---> 52\u001b[0;31m values_element, metadata_elements, namespace)\n\u001b[0m\u001b[1;32m 53\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mquery_isodate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Users/ttuhinanshu/scratch/model-my-watershed/ulmo/env/lib/python2.7/site-packages/ulmo/waterml/common.pyc\u001b[0m in \u001b[0;36m_parse_metadata\u001b[0;34m(values_element, metadata_elements, namespace)\u001b[0m\n\u001b[1;32m 239\u001b[0m collection_dict = dict([\n\u001b[1;32m 240\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 241\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcollection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 242\u001b[0m ])\n\u001b[1;32m 243\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcollection_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcollection_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: 'id'"
]
}
],
"source": [
"values = ulmo.cuahsi.wof.get_values(sample.ServURL + '?WSDL',\n",
" sample.location,\n",
" variable,\n",
" from_date.strftime('%m/%d/%Y'),\n",
" to_date.strftime('%m/%d/%Y'),\n",
" None)\n",
"len(values['values'])"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment