Created
August 29, 2023 10:28
-
-
Save kathrynberger/4fc8a31e9b7a1cea24cb8c0956a439df to your computer and use it in GitHub Desktop.
modis_albedo_wsa_dataset_ingest
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"\u001b[33mWARNING: You are using pip version 22.0.4; however, version 23.2.1 is available.\n", | |
"You should consider upgrading via the '/Users/kathrynaberger/Documents/Work/veda-docs/_env/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n", | |
"\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" | |
] | |
} | |
], | |
"source": [ | |
"%pip install cognito-client --quiet" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Focus on MODIS albedo wsa diff " | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from cognito_client import CognitoClient\n", | |
"\n", | |
"client = CognitoClient(\n", | |
" client_id=\"o8c93cebc17upumgstlbqm44f\",\n", | |
" user_pool_id=\"us-west-2_9mMSsMcxw\",\n", | |
" identity_pool_id=\"us-west-2:40f39c19-ab88-4d0b-85a3-3bad4eacbfc0\",\n", | |
")\n", | |
"_ = client.login()\n", | |
"\n", | |
"TOKEN = client.access_token" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"import os\n", | |
"\n", | |
"import rio_cogeo\n", | |
"import rasterio\n", | |
"import boto3\n", | |
"import requests" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"API = \"https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/\"\n", | |
"\n", | |
"LOCAL_FILE_PATH = \"CampFire_albedo_WSA_2015_2022_diff_cog.tif\"\n", | |
"\n", | |
"\n", | |
"# local file path name already correct, so target file name will be identical\n", | |
"YEAR1 = 2015\n", | |
"YEAR2 = 2022\n", | |
"\n", | |
"TARGET_FILENAME = f\"campfire_albedo_wsa_difference_{YEAR1}_{YEAR2}.tif\"" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Validate data format (if required)\n", | |
"\n" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"# file has already been validated so we can skip this\n", | |
"\n", | |
"# file_is_a_cog = rio_cogeo.cog_validate(LOCAL_FILE_PATH)\n", | |
"# if not file_is_a_cog:\n", | |
"# raise ValueError()\n", | |
"# print(\"File is not a COG - converting\")\n", | |
"# rio_cogeo.cog_translate(LOCAL_FILE_PATH, LOCAL_FILE_PATH, in_memory=True)" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"upload file to s3" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"s3 = boto3.client(\"s3\")\n", | |
"BUCKET = \"veda-data-store-staging\"\n", | |
"KEY = f\"modis_diff/{TARGET_FILENAME}\"\n", | |
"S3_FILE_LOCATION = f\"s3://{BUCKET}/{KEY}\"\n", | |
"\n", | |
"\n", | |
"s3.upload_file(LOCAL_FILE_PATH, BUCKET, KEY)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"!aws s3 ls s3://veda-data-store-staging/" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Construct dataset definition" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"{\n", | |
" \"collection\": \"modis-tri-annual-differences\",\n", | |
" \"title\": \"Camp Fire Domain: MODIS WSA Albedo Difference\",\n", | |
" \"spatial_extent\": {\n", | |
" \"xmin\": -122.21,\n", | |
" \"ymin\": 39.33,\n", | |
" \"xmax\": -120.91,\n", | |
" \"ymax\": 40.22\n", | |
" },\n", | |
" \"temporal_extent\": {\n", | |
" \"startdate\": \"2022-12-31T00:00:00Z\",\n", | |
" \"enddate\": \"2022-12-31T00:00:00Z\"\n", | |
" },\n", | |
" \"license\": \"CC0\",\n", | |
" \"description\": \"MODIS WSA Albedo difference from a three-year average of 2015 to 2018 subtracted from a three-year average of 2019-2022. These tri-annual averages represent periods before and after the fire.\",\n", | |
" \"is_periodic\": true,\n", | |
" \"time_density\": \"year\",\n", | |
" \"sample_files\": [\n", | |
" \"s3://veda-data-store-staging/modis-tri-annual-differences/CampFire_albedo_WSA_2015_2022.tif\"\n", | |
" ],\n", | |
" \"discovery_items\": [\n", | |
" {\n", | |
" \"discovery\": \"s3\",\n", | |
" \"prefix\": \"modis-tri-annual-differences/\",\n", | |
" \"bucket\": \"veda-data-store-staging\",\n", | |
" \"filename_regex\": \"(.*)CampFire_albedo_WSA_2015_2022.tif$\"\n", | |
" }\n", | |
" ]\n", | |
"}\n" | |
] | |
} | |
], | |
"source": [ | |
"dataset = {\n", | |
" \"collection\": \"modis_diff\",\n", | |
" \"title\": \"Camp Fire Domain: MODIS WSA Albedo Difference\",\n", | |
" \"spatial_extent\": {\"xmin\": -122.21, \"ymin\": 39.33, \"xmax\": -120.91, \"ymax\": 40.22},\n", | |
" \"temporal_extent\": {\n", | |
" \"startdate\": \"2022-12-31T00:00:00Z\",\n", | |
" \"enddate\": \"2022-12-31T23:59:59Z\",\n", | |
" },\n", | |
" \"data_type\": \"cog\",\n", | |
" \"license\": \"CC0\",\n", | |
" \"description\": \"MODIS WSA Albedo difference from a three-year average of 2015 to 2018 subtracted from a three-year average of 2019-2022. These tri-annual averages represent periods before and after the fire.\",\n", | |
" \"is_periodic\": true,\n", | |
" \"time_density\": \"year\",\n", | |
" \"sample_files\": [\n", | |
" \"s3://veda-data-store-staging/modis_diff/campfire_albedo_wsa_difference_2015_2022.tif\"\n", | |
" ],\n", | |
" \"discovery_items\": [\n", | |
" {\n", | |
" \"discovery\": \"s3\",\n", | |
" \"prefix\": \"modis_diff/\",\n", | |
" \"bucket\": \"veda-data-store-staging\",\n", | |
" \"filename_regex\": \"(.*)campfire_albedo_wsa_difference_2015_2022.tif$\",\n", | |
" }\n", | |
" ],\n", | |
"}\n", | |
"\n", | |
"import json\n", | |
"\n", | |
"print(json.dumps(dataset, indent=2))" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Validate dataset definition" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"ename": "HTTPError", | |
"evalue": "422 Client Error: for url: https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/dataset/validate", | |
"output_type": "error", | |
"traceback": [ | |
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
"\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", | |
"\u001b[0;32m/var/folders/s9/q707z2_n1sjd38cvg62wg4c40000gn/T/ipykernel_28864/149992075.py\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m }\n\u001b[1;32m 7\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAPI\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"dataset/validate\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
"\u001b[0;32m~/Documents/Work/veda-docs/_env/lib/python3.9/site-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
"\u001b[0;31mHTTPError\u001b[0m: 422 Client Error: for url: https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/dataset/validate" | |
] | |
} | |
], | |
"source": [ | |
"auth_header = f\"Bearer {TOKEN}\"\n", | |
"headers = {\n", | |
" \"Authorization\": auth_header,\n", | |
" \"content-type\": \"application/json\",\n", | |
" \"accept\": \"application/json\",\n", | |
"}\n", | |
"response = requests.post((API + \"dataset/validate\"), json=dataset, headers=headers)\n", | |
"response.raise_for_status()\n", | |
"print(response.text)" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Publish to STAC" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"response = requests.post((API + \"dataset/publish\"), json=dataset, headers=headers)\n", | |
"response.raise_for_status()\n", | |
"print(response.text)" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "_env", | |
"language": "python", | |
"name": "_env" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.9.15" | |
}, | |
"orig_nbformat": 4 | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 2 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment