Created
June 5, 2018 14:31
-
-
Save rutgerhofste/b174e4c8fa4c9a2ad40e60bb2e482c0c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Input : http://data.wri.org/Aqueduct/web/aqueduct_global_maps_21_shp.zip\n", | |
"Output s3: s3://wri-projects/Aqueduct30/qaData/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/output_V01\n" | |
] | |
} | |
], | |
"source": [ | |
"\"\"\" Create Aqueduct 2.1 shapefile with fluxes. \n", | |
"-------------------------------------------------------------------------------\n", | |
"\n", | |
"Author: Rutger Hofste\n", | |
"Date: 20180605\n", | |
"Kernel: python35\n", | |
"Docker: rutgerhofste/gisdocker:ubuntu16.04\n", | |
"\n", | |
"Args:\n", | |
"\n", | |
" TESTING (boolean) : Testing mode. Uses a smaller geography if enabled.\n", | |
" \n", | |
" SCRIPT_NAME (string) : Script name.\n", | |
" EE_INPUT_ZONES_PATH (string) : earthengine input path for zones.\n", | |
" EE_INPUT_VALUES_PATH (string) : earthengine input path for value images.\n", | |
" INPUT_VERSION_ZONES (integer) : input version for zones images.\n", | |
" INPUT_VERSION_VALUES (integer) : input version for value images.\n", | |
" OUTPUT_VERSION (integer) : output version. \n", | |
" EXTRA_PROPERTIES (dictionary) : Extra properties to store in the resulting\n", | |
" pandas dataframe. \n", | |
" \n", | |
"\n", | |
"Returns:\n", | |
"\n", | |
"\n", | |
"\n", | |
"\"\"\"\n", | |
"\n", | |
"SCRIPT_NAME = \"Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01\"\n", | |
"OUTPUT_VERSION = 1\n", | |
"OVERWRITE =1 \n", | |
"\n", | |
"AQUEDUCT21_URL = \"http://data.wri.org/Aqueduct/web/aqueduct_global_maps_21_shp.zip\"\n", | |
"FILE_NAME = \"aqueduct21\"\n", | |
"DETAILED_FILE_NAME = \"aqueduct_global_dl_20150409.shp\"\n", | |
"\n", | |
"\n", | |
"ECKERT_IV_PROJ4_STRING1 = \"+proj=eck4 +lon_0=0 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs\"\n", | |
"ECKERT_IV_PROJ4_STRING2 = \"+proj=eck4 +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m no_defs\"\n", | |
"\n", | |
"s3_output_path = \"s3://wri-projects/Aqueduct30/qaData/{}/output_V{:02.0f}\".format(SCRIPT_NAME,OUTPUT_VERSION)\n", | |
"ec2_input_path = \"/volumes/data/{}/input_V{:02.0f}\".format(SCRIPT_NAME,OUTPUT_VERSION)\n", | |
"ec2_output_path = \"/volumes/data/{}/output_V{:02.0f}\".format(SCRIPT_NAME,OUTPUT_VERSION) \n", | |
"\n", | |
"\n", | |
"print(\"Input : \" + AQUEDUCT21_URL +\n", | |
" \"\\nOutput s3: \" + s3_output_path)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Y2018M06D05 UTC 14:30\n" | |
] | |
}, | |
{ | |
"data": { | |
"text/plain": [ | |
"'3.5.4 |Anaconda, Inc.| (default, Nov 20 2017, 18:44:38) \\n[GCC 7.2.0]'" | |
] | |
}, | |
"execution_count": 2, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"import time, datetime, sys, logging\n", | |
"dateString = time.strftime(\"Y%YM%mD%d\")\n", | |
"timeString = time.strftime(\"UTC %H:%M\")\n", | |
"start = datetime.datetime.now()\n", | |
"print(dateString,timeString)\n", | |
"sys.version" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"import subprocess\n", | |
"import geopandas as gpd" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"if OVERWRITE:\n", | |
" !rm -r {ec2_input_path}\n", | |
" !rm -r {ec2_output_path}\n", | |
" !mkdir -p {ec2_input_path}\n", | |
" !mkdir -p {ec2_output_path}" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"b''" | |
] | |
}, | |
"execution_count": 5, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"command = \"wget -O {}/{}.zip {}\".format(ec2_input_path,FILE_NAME,AQUEDUCT21_URL)\n", | |
"subprocess.check_output(command,shell=True)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"b'Archive: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct21.zip\\n extracting: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_dl_20150409.cpg \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_dl_20150409.dbf \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_dl_20150409.prj \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_dl_20150409.shp \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_dl_20150409.shp.xml \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_dl_20150409.shx \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/aqueduct_global_maps_21.pdf \\n inflating: /volumes/data/Y2018M06D05_RH_QA_Aqueduct21_Flux_Shapefile_V01/input_V01/dictionary_20150409.xlsx \\n'" | |
] | |
}, | |
"execution_count": 6, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"command = \"unzip {}/{} -d {}\".format(ec2_input_path,FILE_NAME,ec2_input_path)\n", | |
"subprocess.check_output(command,shell=True)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"input_file_path = \"{}/{}\".format(ec2_input_path,DETAILED_FILE_NAME)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"gdf = gpd.read_file(input_file_path)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"{'init': 'epsg:4326'}" | |
] | |
}, | |
"execution_count": 9, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"gdf.crs" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 10, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"gdf_eckert4 = gdf.to_crs(ECKERT_IV_PROJ4_STRING1)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 11, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"ename": "RuntimeError", | |
"evalue": "b'unknown unit conversion id'", | |
"output_type": "error", | |
"traceback": [ | |
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", | |
"\u001b[0;32m<ipython-input-11-419049ecf17f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgdf_eckert4\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_crs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mECKERT_IV_PROJ4_STRING2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", | |
"\u001b[0;32m/opt/anaconda3/envs/python35/lib/python3.5/site-packages/geopandas/geodataframe.py\u001b[0m in \u001b[0;36mto_crs\u001b[0;34m(self, crs, epsg, inplace)\u001b[0m\n\u001b[1;32m 384\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 385\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 386\u001b[0;31m \u001b[0mgeom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeometry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_crs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 387\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeometry\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgeom\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 388\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgeom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
"\u001b[0;32m/opt/anaconda3/envs/python35/lib/python3.5/site-packages/geopandas/geoseries.py\u001b[0m in \u001b[0;36mto_crs\u001b[0;34m(self, crs, epsg)\u001b[0m\n\u001b[1;32m 284\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Must set either crs or epsg for output.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0mproj_in\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyproj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mProj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreserve_units\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 286\u001b[0;31m \u001b[0mproj_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyproj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mProj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreserve_units\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 287\u001b[0m \u001b[0mproject\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpartial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyproj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproj_in\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproj_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mgeom\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproject\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgeom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
"\u001b[0;32m/opt/anaconda3/envs/python35/lib/python3.5/site-packages/pyproj/__init__.py\u001b[0m in \u001b[0;36m__new__\u001b[0;34m(self, projparams, preserve_units, **kwargs)\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[0;31m# on case-insensitive filesystems).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 357\u001b[0m \u001b[0mprojstring\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprojstring\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'EPSG'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'epsg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 358\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_proj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mProj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__new__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 360\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
"\u001b[0;32m_proj.pyx\u001b[0m in \u001b[0;36m_proj.Proj.__cinit__ (_proj.c:1170)\u001b[0;34m()\u001b[0m\n", | |
"\u001b[0;31mRuntimeError\u001b[0m: b'unknown unit conversion id'" | |
] | |
} | |
], | |
"source": [ | |
"gdf_eckert4 = gdf.to_crs(ECKERT_IV_PROJ4_STRING2)" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 35", | |
"language": "python", | |
"name": "python35" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.5.4" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 2 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment