Skip to content

Instantly share code, notes, and snippets.

@rutgerhofste

rutgerhofste/gist:529cd658967243ff6f89fb7df91a3986

Last active September 6, 2017 15:13
Show Gist options
  • Save rutgerhofste/529cd658967243ff6f89fb7df91a3986 to your computer and use it in GitHub Desktop.
Save rutgerhofste/529cd658967243ff6f89fb7df91a3986 to your computer and use it in GitHub Desktop.
Download ZIP
Earth engine Dimensions not working
Raw
gistfile1.txt
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Calculate average PCRGlobWB supply using EE\n",
"\n",
"* Purpose of script: This script will calculate baseline supply based on runoff for 1960-2014 at 5min resolution\n",
"* Author: Rutger Hofste\n",
"* Kernel used: python27\n",
"* Date created: 20170830"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import ee\n",
"import folium\n",
"from folium_gee import *\n",
"import subprocess\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"ee.Initialize()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"EE_INPUT_PATH = \"projects/WRI-Aquaduct/PCRGlobWB20V07/\"\n",
"\n",
"EE_IC_NAME_ANNUAL = \"global_historical_reducedmeanrunoff_year_myear_5min_1960_2014\"\n",
"EE_IC_NAME_MONTH = \"global_historical_reducedmeanrunoff_month_mmonth_5min_1960_2014\"\n",
"\n",
"EE_I_NAME_ANNUAL = EE_IC_NAME_ANNUAL\n",
"EE_I_NAME_MONTH = EE_IC_NAME_MONTH\n",
"\n",
"DIMENSION5MIN = \"4320x2160\"\n",
"\n",
"INPUT_FILE_NAME_ANNUAL = \"global_historical_runoff_year_myear_5min_1958_2014\"\n",
"INPUT_FILE_NAME_MONTH = \"global_historical_runoff_month_mmonth_5min_1958_2014\"\n",
"\n",
"YEAR_MIN = 1960\n",
"YEAR_MAX = 2014\n",
"ANNUAL_UNITS = \"m/year\"\n",
"MONTHLY_UNITS = \"m/month\"\n",
"ANNUAL_EXPORTDESCRIPTION = \"reducedmeanrunoff_year\" #final format reducedmeanrunoff_yearY1960Y2014\n",
"MONTHLY_EXPORTDESCRIPTION = \"reducedmeanrunoff_month\" #final format reducedmeanrunoff_monthY1960Y2014M01\n",
"VERSION = \"36\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The Standardized format to store assets on Earth Engine is EE_INPUT_PATH / EE_IC_NAME / EE_I_NAME and every image should have the property expertdescription that would allow to export the data to a table header. "
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def exportToAssetAnnual(ic):\n",
" annualExportDescription = ANNUAL_EXPORTDESCRIPTION + \"Y%sY%s\" %(YEAR_MIN,YEAR_MAX)\n",
" annualImage = ee.Image(ic.reduce(ee.Reducer.mean()))\n",
" properties = {\"rangeMin\":ee.Number(YEAR_MIN),\n",
" \"rangeMax\":ee.Number(YEAR_MAX),\n",
" \"units\":ANNUAL_UNITS,\n",
" \"exportdescription\":annualExportDescription,\n",
" \"creation\":\"RutgerHofste_20170901_Python27\",\n",
" \"nodata_value\":-9999,\n",
" \"reducer\":\"mean\",\n",
" \"time_start\": \"%04d-%0.2d-%0.2d\" %(YEAR_MAX,12,1) \n",
" }\n",
" annualImage = annualImage.set(properties)\n",
" scale = ee.Image(icAnnual.first()).projection().nominalScale().getInfo()\n",
" assetId = EE_INPUT_PATH + EE_IC_NAME_ANNUAL + \"V\" +VERSION+ \"/\" + EE_I_NAME_ANNUAL + \"V\" + VERSION \n",
" task = ee.batch.Export.image.toAsset(\n",
" image = ee.Image(annualImage),\n",
" description = EE_I_NAME_ANNUAL + \"V\" + VERSION,\n",
" assetId = assetId,\n",
" dimensions = DIMENSION5MIN,\n",
" region = geometry.bounds().getInfo()['coordinates'][0],\n",
" maxPixels = 1e10\n",
" )\n",
" task.start()\n",
" print(assetId)\n",
" return annualImage\n",
"\n",
"\n",
"def exportToAssetMonth(month):\n",
" monthlyExportDescription = MONTHLY_EXPORTDESCRIPTION + \"Y%sY%sM%0.2d\" %(YEAR_MIN,YEAR_MAX,month)\n",
" monthlyImage = filteredMonthlyCollection.filter(ee.Filter.eq(\"month\",ee.Number(month)))\n",
" monthlyImage = filteredMonthlyCollection.reduce(ee.Reducer.mean())\n",
" properties = {\"month\":month,\n",
" \"reducer\":\"mean\",\n",
" \"rangeMin\":ee.Number(YEAR_MIN),\n",
" \"rangeMax\":ee.Number(YEAR_MAX),\n",
" \"units\":MONTHLY_UNITS,\n",
" \"exportdescription\":monthlyExportDescription,\n",
" \"creation\":\"RutgerHofste_20170901_Python27\",\n",
" \"nodata_value\":-9999,\n",
" \"time_start\": \"%04d-%0.2d-%0.2d\" %(YEAR_MAX,month,1) \n",
" }\n",
" monthlyImage = monthlyImage.set(properties)\n",
"\n",
" \n",
" scale = ee.Image(icMonthly.first()).projection().nominalScale().getInfo()\n",
"\n",
" assetId = EE_INPUT_PATH + EE_IC_NAME_MONTH + \"V\" +VERSION+ \"/\" + EE_I_NAME_MONTH + \"M%0.2dV%s\" %(month,VERSION) \n",
" task = ee.batch.Export.image.toAsset(\n",
" image = ee.Image(monthlyImage),\n",
" description = EE_I_NAME_MONTH + \"M%0.2dV%s\" %(month,VERSION) ,\n",
" assetId = assetId,\n",
" dimensions = DIMENSION5MIN,\n",
" region = geometry.bounds().getInfo()['coordinates'][0],\n",
" maxPixels = 1e10\n",
" )\n",
" task.start()\n",
" print(month)\n",
" return 1\n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"icAnnual = ee.ImageCollection(os.path.join(EE_INPUT_PATH,INPUT_FILE_NAME_ANNUAL))\n",
"icMonthly = ee.ImageCollection(os.path.join(EE_INPUT_PATH,INPUT_FILE_NAME_MONTH))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"dateFilterMin = ee.Filter.gte(\"year\",YEAR_MIN)\n",
"dateFilterMax = ee.Filter.lte(\"year\",YEAR_MAX)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"geometry = ee.Geometry.Polygon(coords=[[-180.0, -90.0], [180, -90.0], [180, 89], [180,90]], proj= ee.Projection('EPSG:4326'),geodesic=False )"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"filteredAnnualCollection = ee.ImageCollection(icAnnual.filter(dateFilterMin).filter(dateFilterMax))\n",
"filteredMonthlyCollection = ee.ImageCollection(icMonthly.filter(dateFilterMin).filter(dateFilterMax))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"sampleImage = ee.Image(filteredAnnualCollection.first())"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"earthengine create collection projects/WRI-Aquaduct/PCRGlobWB20V07/global_historical_reducedmeanrunoff_year_myear_5min_1960_2014V36\n"
]
}
],
"source": [
"icAnnualPath = os.path.join(EE_INPUT_PATH,EE_IC_NAME_ANNUAL+ \"V\" + VERSION)\n",
"command = (\"earthengine create collection %s\") %icAnnualPath\n",
"print(command)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Asset projects/WRI-Aquaduct/PCRGlobWB20V07/global_historical_reducedmeanrunoff_year_myear_5min_1960_2014V36 already exists\\n'"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"subprocess.check_output(command,shell=True)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"earthengine create collection projects/WRI-Aquaduct/PCRGlobWB20V07/global_historical_reducedmeanrunoff_month_mmonth_5min_1960_2014V36\n"
]
}
],
"source": [
"icMonthPath = os.path.join(EE_INPUT_PATH,EE_IC_NAME_MONTH+ \"V\" + VERSION)\n",
"command = (\"earthengine create collection %s\") %icMonthPath\n",
"print(command)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Asset projects/WRI-Aquaduct/PCRGlobWB20V07/global_historical_reducedmeanrunoff_month_mmonth_5min_1960_2014V36 already exists\\n'"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"subprocess.check_output(command,shell=True)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"### Run the functions"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"projects/WRI-Aquaduct/PCRGlobWB20V07/global_historical_reducedmeanrunoff_year_myear_5min_1960_2014V36/global_historical_reducedmeanrunoff_year_myear_5min_1960_2014V36\n"
]
}
],
"source": [
"annualImage = exportToAssetAnnual(filteredAnnualCollection)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"months = list(range(1,13))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"map(exportToAssetMonth,months)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"## check results"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"annualImage = annualImage.reproject('EPSG:4326',sampleImage.projection().getInfo())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"lat = 39.495159\n",
"lon = -107.3689237\n",
"zoom_start=5"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"m = folium.Map(location=[lat, lon], tiles=\"OpenStreetMap\", zoom_start=zoom_start)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"vis_params = {'min':0.0, 'max':100, 'palette':'00FFFF,0000FF'}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"folium_gee_layer(m,annualImage,vis_params=vis_params,folium_kwargs={'overlay':True,'name':'reducedRunoff'})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m.add_child(folium.LayerControl())\n",
"m"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 27",
"language": "python",
"name": "python27"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment