Skip to content

Instantly share code, notes, and snippets.

@jkibele

jkibele/EC_Result_Check.ipynb Secret

Created October 4, 2017 21:18
Show Gist options
  • Save jkibele/f1b0a557fd348764a0a7c27b0c2a090a to your computer and use it in GitHub Desktop.
Save jkibele/f1b0a557fd348764a0a7c27b0c2a090a to your computer and use it in GitHub Desktop.
Download ZIP
Checking Distance Results and Reformatting
Display the source blob
Display the rendered blob
Raw
EC_Result_Check.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline\n",
"import os\n",
"import xlwt\n",
"import pyriv\n",
"import networkx as nx\n",
"import geopandas as gpd\n",
"import pandas as pd\n",
"import mplleaflet\n",
"import pyriv.rg_light as rgl\n",
"\n",
"data_dir = '/Users/jkibele/Documents/RiverDistEastCoast/data/'\n",
"fp = lambda s: os.path.join(data_dir, s)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Result Check, Reformat, and Export\n",
"\n",
"I will make sure that all site codes are represented, then I'll convert the distances from long form to a distance matrix, then I'll output to excel format."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"ale_fn = fp('reproj/locations_ALE_5070.shp')\n",
"bbh_fn = fp('reproj/locations_BBH_5070.shp')\n",
"ale = gpd.read_file(ale_fn)\n",
"bbh = gpd.read_file(bbh_fn)\n",
"ale_paths = gpd.read_file(fp('output/goodpaths_ale.shp'))\n",
"bbh_paths = gpd.read_file(fp('output/goodpaths_bbh.shp'))\n",
"ale_paths.crs = ale.crs\n",
"bbh_paths.crs = bbh.crs"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>from_code</th>\n",
" <th>to_code</th>\n",
" <th>good_path</th>\n",
" <th>geometry</th>\n",
" <th>dist_km</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>GAR</td>\n",
" <td>OTT</td>\n",
" <td>1</td>\n",
" <td>LINESTRING (3002040.111832098 3344877.28300758...</td>\n",
" <td>741.619680</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>GAR</td>\n",
" <td>MIR</td>\n",
" <td>1</td>\n",
" <td>LINESTRING (3002040.111832098 3344877.28300758...</td>\n",
" <td>790.164386</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>GAR</td>\n",
" <td>RIC</td>\n",
" <td>1</td>\n",
" <td>LINESTRING (3002040.111832098 3344877.28300758...</td>\n",
" <td>744.392978</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>GAR</td>\n",
" <td>TID</td>\n",
" <td>1</td>\n",
" <td>LINESTRING (3002040.111832098 3344877.28300758...</td>\n",
" <td>713.273035</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>GAR</td>\n",
" <td>RPH</td>\n",
" <td>1</td>\n",
" <td>LINESTRING (3002040.111832098 3344877.28300758...</td>\n",
" <td>691.093779</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" from_code to_code good_path \\\n",
"0 GAR OTT 1 \n",
"1 GAR MIR 1 \n",
"2 GAR RIC 1 \n",
"3 GAR TID 1 \n",
"4 GAR RPH 1 \n",
"\n",
" geometry dist_km \n",
"0 LINESTRING (3002040.111832098 3344877.28300758... 741.619680 \n",
"1 LINESTRING (3002040.111832098 3344877.28300758... 790.164386 \n",
"2 LINESTRING (3002040.111832098 3344877.28300758... 744.392978 \n",
"3 LINESTRING (3002040.111832098 3344877.28300758... 713.273035 \n",
"4 LINESTRING (3002040.111832098 3344877.28300758... 691.093779 "
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ale_paths['dist_km'] = ale_paths.geometry.length * 1e-3\n",
"bbh_paths['dist_km'] = bbh_paths.geometry.length * 1e-3\n",
"ale_paths.head()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def all_sites_present(sitedf, pathdf):\n",
" scodes = sitedf.Code.unique()\n",
" pcodes = unique(pathdf.from_code.unique().tolist() + pathdf.to_code.unique().tolist())\n",
" paths_not_in_sites = [pc for pc in pcodes if pc not in scodes]\n",
" \n",
" if paths_not_in_sites:\n",
" print \"These path codes are missing from sitedf:\"\n",
" for code in paths_not_in_sites:\n",
" print \"\\t{}\".format(code)\n",
" else:\n",
" print \"All path codes are in sitedf.\"\n",
" \n",
" sites_not_in_paths = [sc for sc in scodes if sc not in pcodes]\n",
" if sites_not_in_paths:\n",
" print \"These site codes are missing from pathdf:\"\n",
" for code in sites_not_in_paths:\n",
" print \"\\t{}\".format(code)\n",
" else:\n",
" print \"All site codes are in the \\\"from_code\\\" and/or \\\"to_code\\\" field of the pathdf.\""
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"All path codes are in sitedf.\n",
"All site codes are in the \"from_code\" and/or \"to_code\" field of the pathdf.\n"
]
}
],
"source": [
"all_sites_present(ale, ale_paths)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"These path codes are missing from sitedf:\n",
"\tSJR-MD\n",
"All site codes are in the \"from_code\" and/or \"to_code\" field of the pathdf.\n"
]
}
],
"source": [
"all_sites_present(bbh, bbh_paths)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Right. I need to fix that one bbh site code. That's the one that references two different positions with the same code."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Name</th>\n",
" <th>Code</th>\n",
" <th>Lat</th>\n",
" <th>Lon</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Saint John River - Mactaquac Dam</td>\n",
" <td>SJR</td>\n",
" <td>45.953502</td>\n",
" <td>-66.865123</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>St .John's River</td>\n",
" <td>SJR</td>\n",
" <td>29.603320</td>\n",
" <td>-81.602206</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Name Code Lat Lon\n",
"2 Saint John River - Mactaquac Dam SJR 45.953502 -66.865123\n",
"45 St .John's River SJR 29.603320 -81.602206"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bbh.loc[bbh.Code.apply(lambda c: c.startswith(\"SJR\"))].drop('geometry', 1)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Name</th>\n",
" <th>Code</th>\n",
" <th>Lat</th>\n",
" <th>Lon</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Saint John River - Mactaquac Dam</td>\n",
" <td>SJR-MD</td>\n",
" <td>45.953502</td>\n",
" <td>-66.865123</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>St .John's River</td>\n",
" <td>SJR</td>\n",
" <td>29.603320</td>\n",
" <td>-81.602206</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Name Code Lat Lon\n",
"2 Saint John River - Mactaquac Dam SJR-MD 45.953502 -66.865123\n",
"45 St .John's River SJR 29.603320 -81.602206"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bbh.loc[2,'Code'] = \"SJR-MD\"\n",
"bbh.loc[bbh.Code.apply(lambda c: c.startswith(\"SJR\"))].drop('geometry', 1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Reformat"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"sitedf = bbh\n",
"pathdf = bbh_paths\n",
"\n",
"def pathdf_to_distance_matrix(pathdf, sitedf):\n",
" scodes = sitedf.Code\n",
"# scodes = unique(pathdf.from_code.unique().tolist() + pathdf.to_code.unique().tolist())\n",
" dist_rows = []\n",
" for sc_row in scodes:\n",
" drow = []\n",
" for sc_col in scodes:\n",
" if sc_col == sc_row:\n",
" drow.append(0.0)\n",
" else:\n",
" dval_ind = ((pathdf.from_code==sc_row) & (pathdf.to_code==sc_col))\n",
" if not dval_ind.any():\n",
" dval_ind = ((pathdf.from_code==sc_col) & (pathdf.to_code==sc_row))\n",
" if not dval_ind.any():\n",
" print \"There's a problem with {} and {}.\".format(sc_row, sc_col)\n",
" drow.append(pathdf.loc[dval_ind, 'dist_km'].item())\n",
" dist_rows.append(drow)\n",
"\n",
" return pd.DataFrame(dist_rows, index=scodes, columns=scodes)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"bbh_dm = pathdf_to_distance_matrix(bbh_paths, bbh)\n",
"ale_dm = pathdf_to_distance_matrix(ale_paths, ale)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Save to Excel Format"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"xlwrit = pd.ExcelWriter(fp('output/dist_mats.xls'))\n",
"bbh_dm.to_excel(xlwrit, sheet_name=\"BBH\")\n",
"ale_dm.to_excel(xlwrit, sheet_name=\"ALE\")\n",
"xlwrit.save()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment