Skip to content

Instantly share code, notes, and snippets.

@kathrynberger

kathrynberger/modis_ndvi_diff.ipynb

Created August 24, 2023 11:45
Show Gist options
  • Save kathrynberger/a338cea7979cf8013f92038d2ab4fdd1 to your computer and use it in GitHub Desktop.
Save kathrynberger/a338cea7979cf8013f92038d2ab4fdd1 to your computer and use it in GitHub Desktop.
Download ZIP
sample ingest of ndvi diff
Display the source blob
Display the rendered blob
Raw
modis_ndvi_diff.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mWARNING: You are using pip version 22.0.4; however, version 23.2.1 is available.\n",
"You should consider upgrading via the '/Users/kathrynaberger/Documents/Work/veda-docs/_env/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
"\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [
"%pip install cognito-client --quiet"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Focus on NDVI diff "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"from cognito_client import CognitoClient\n",
"\n",
"client = CognitoClient(\n",
" client_id=\"o8c93cebc17upumgstlbqm44f\",\n",
" user_pool_id=\"us-west-2_9mMSsMcxw\",\n",
" identity_pool_id=\"us-west-2:40f39c19-ab88-4d0b-85a3-3bad4eacbfc0\",\n",
")\n",
"_ = client.login()\n",
"\n",
"TOKEN = client.access_token"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import rio_cogeo\n",
"import rasterio\n",
"import boto3\n",
"import requests"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"API = \"https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/\"\n",
"\n",
"LOCAL_FILE_PATH = \"CampFire_NDVI_2015_2022_diff_cog.tif\"\n",
"YEAR1, YEAR2 = 2015, 2022\n",
"# local file path name already correct, so target file name will be identical\n",
"TARGET_FILENAME = \"CampFire_NDVI_{YEAR1}_{YEAR2}.tif\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Validate data format (if required)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# file has already been validated so we can skip this\n",
"\n",
"# file_is_a_cog = rio_cogeo.cog_validate(LOCAL_FILE_PATH)\n",
"# if not file_is_a_cog:\n",
"# raise ValueError()\n",
"# print(\"File is not a COG - converting\")\n",
"# rio_cogeo.cog_translate(LOCAL_FILE_PATH, LOCAL_FILE_PATH, in_memory=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"upload file to s3"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"s3 = boto3.client(\"s3\")\n",
"BUCKET = \"veda-data-store-staging\"\n",
"KEY = f\"{BUCKET}/MODIS_diff/{TARGET_FILENAME}\"\n",
"S3_FILE_LOCATION = f\"s3://{KEY}\"\n",
"\n",
"if False:\n",
" s3.upload_file(LOCAL_FILE_PATH, KEY)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Construct dataset definition"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"collection\": \"MODIS_diff\",\n",
" \"title\": \"Camp Fire Domain: MODIS NDVI Difference\",\n",
" \"spatial_extent\": {\n",
" \"xmin\": -122.21,\n",
" \"ymin\": 39.33,\n",
" \"xmax\": -120.91,\n",
" \"ymax\": 40.22\n",
" },\n",
" \"temporal_extent\": {\n",
" \"startdate\": \"2022-12-31T00:00:00Z\",\n",
" \"enddate\": \"2022-12-31T00:00:00Z\"\n",
" },\n",
" \"license\": \"CC0\",\n",
" \"description\": \"MODIS NDVI difference from a three-year average of 2015 to 2018 subtracted from a three-year average of 2019-2022. These tri-annual averages represent periods before and after the fire.\",\n",
" \"is_periodic\": true,\n",
" \"time_density\": \"year\",\n",
" \"sample_files\": [\n",
" \"s3://veda-data-store-staging/MODIS_diff/CampFire_NDVI_{YEAR1}_{YEAR2}.tif\"\n",
" ],\n",
" \"discovery_items\": [\n",
" {\n",
" \"discovery\": \"s3\",\n",
" \"prefix\": \"MODIS_diff/\",\n",
" \"bucket\": \"veda-data-store-staging\",\n",
" \"filename_regex\": \"(.*)CampFire_NDVI_{YEAR1}_{YEAR2}.tif$\"\n",
" }\n",
" ]\n",
"}\n"
]
}
],
"source": [
"dataset = {\n",
" \"collection\": \"MODIS_diff\",\n",
" \"title\": \"Camp Fire Domain: MODIS NDVI Difference\",\n",
" \"spatial_extent\": {\n",
" \"xmin\": -122.21,\n",
" \"ymin\": 39.33,\n",
" \"xmax\": -120.91,\n",
" \"ymax\": 40.22,\n",
" },\n",
" \"temporal_extent\": {\n",
" \"startdate\": \"2022-12-31T00:00:00Z\",\n",
" \"enddate\": \"2022-12-31T00:00:00Z\",\n",
" },\n",
" \"license\": \"CC0\",\n",
" \"description\": \"MODIS NDVI difference from a three-year average of 2015 to 2018 subtracted from a three-year average of 2019-2022. These tri-annual averages represent periods before and after the fire.\",\n",
" \"is_periodic\": True,\n",
" \"time_density\": \"year\",\n",
" \"sample_files\": [S3_FILE_LOCATION],\n",
" \"discovery_items\": [\n",
" {\n",
" \"discovery\": \"s3\",\n",
" \"prefix\": \"MODIS_diff/\",\n",
" \"bucket\": \"veda-data-store-staging\",\n",
" \"filename_regex\": f\"(.*){TARGET_FILENAME}$\",\n",
" }\n",
" ],\n",
"}\n",
"import json\n",
"\n",
"print(json.dumps(dataset, indent=2))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Validate dataset definition"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"ename": "HTTPError",
"evalue": "422 Client Error: for url: https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/dataset/validate",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/var/folders/s9/q707z2_n1sjd38cvg62wg4c40000gn/T/ipykernel_38242/149992075.py\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m }\n\u001b[1;32m 7\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAPI\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"dataset/validate\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/Work/veda-docs/_env/lib/python3.9/site-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mHTTPError\u001b[0m: 422 Client Error: for url: https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/dataset/validate"
]
}
],
"source": [
"auth_header = f\"Bearer {TOKEN}\"\n",
"headers = {\n",
" \"Authorization\": auth_header,\n",
" \"content-type\": \"application/json\",\n",
" \"accept\": \"application/json\",\n",
"}\n",
"response = requests.post((API + \"dataset/validate\"), json=dataset, headers=headers)\n",
"response.raise_for_status()\n",
"print(response.text)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Publish to STAC"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"response = requests.post((API + \"dataset/publish\"), json=dataset, headers=headers)\n",
"response.raise_for_status()\n",
"print(response.text)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "_env",
"language": "python",
"name": "_env"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.15"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment