Skip to content

Instantly share code, notes, and snippets.

@kathrynberger

kathrynberger/add_modis_ndvi_diff.ipynb

Created August 28, 2023 13:25
Show Gist options
  • Save kathrynberger/1e5435d280386871bc61eaf3e6d78140 to your computer and use it in GitHub Desktop.
Save kathrynberger/1e5435d280386871bc61eaf3e6d78140 to your computer and use it in GitHub Desktop.
Download ZIP
updated ndvi diff ingest workflow
Display the source blob
Display the rendered blob
Raw
add_modis_ndvi_diff.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "72cd663e-e10f-4e53-92eb-b2ad8c2ac523",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
"aiobotocore 2.5.0 requires botocore<1.29.77,>=1.29.76, but you have botocore 1.27.96 which is incompatible.\n",
"awscli 1.27.76 requires botocore==1.29.76, but you have botocore 1.27.96 which is incompatible.\u001b[0m\u001b[31m\n",
"\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [
"%pip install cognito-client --quiet"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "5cbb9469-9fd8-4b6a-abe1-349edf68a34c",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdin",
"output_type": "stream",
"text": [
"Enter your Cognito username: kathryn\n",
"Enter your Cognito password: ········\n"
]
}
],
"source": [
"from cognito_client import CognitoClient\n",
"\n",
"client = CognitoClient(\n",
" client_id=\"o8c93cebc17upumgstlbqm44f\",\n",
" user_pool_id=\"us-west-2_9mMSsMcxw\",\n",
" identity_pool_id=\"us-west-2:40f39c19-ab88-4d0b-85a3-3bad4eacbfc0\",\n",
")\n",
"_ = client.login()\n",
"\n",
"TOKEN = client.access_token"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e1f33adb-372b-4ece-89da-ccefbd3e2d9c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import os\n",
"\n",
"import rio_cogeo\n",
"import rasterio\n",
"import boto3\n",
"import requests"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "35c829bc-a1c9-4a00-a09b-0da9a01ccd8a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"API = \"https://ig9v64uky8.execute-api.us-west-2.amazonaws.com/staging/\"\n",
"\n",
"LOCAL_FILE_PATH = \"CampFire_NDVI_2015_2022_diff_cog.tif\"\n",
"YEAR1, YEAR2 = 2015, 2022\n",
"# local file path name already correct, so target file name will be identical\n",
"TARGET_FILENAME = \"campfire_ndvi_{YEAR1}_{YEAR2}.tif\""
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "f573c627-3632-4a35-b261-5332c61f345f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"s3 = boto3.client(\"s3\")\n",
"BUCKET = \"veda-data-store-staging\"\n",
"KEY = f\"modis_diff/{TARGET_FILENAME}\"\n",
"S3_FILE_LOCATION = f\"s3://{KEY}\"\n",
"\n",
"\n",
"s3.upload_file(LOCAL_FILE_PATH, BUCKET, KEY)\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "6ccbe28c-fc6f-46ad-b276-fef3e2c18942",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" PRE /\n",
" PRE EIS/\n",
" PRE GIOVANNI/\n",
" PRE GPM_3IMERGHHE.06/\n",
" PRE GrIS_terminus_position_comparison/\n",
" PRE IS2SITMOGR4-cog/\n",
" PRE IS2SITMOGR4/\n",
" PRE MO_NPP_npp_vgpm/\n",
" PRE OMI_trno2-COG/\n",
" PRE OMSO2PCA-COG/\n",
" PRE aimee-examples/\n",
" PRE aimee_examples/\n",
" PRE blue-tarp-detection/\n",
" PRE blue-tarp-planetscope/\n",
" PRE cci_biomass_2010/\n",
" PRE cci_biomass_2017/\n",
" PRE cci_biomass_2018/\n",
" PRE co2-diff/\n",
" PRE co2-mean/\n",
" PRE disturbance-probability-percentile/\n",
" PRE facebook_population_density/\n",
" PRE geoglam/\n",
" PRE grdi-cdr-raster/\n",
" PRE grdi-filled-missing-values-count/\n",
" PRE grdi-imr-raster/\n",
" PRE grdi-shdi-raster/\n",
" PRE grdi-v1-built/\n",
" PRE grdi-v1-raster/\n",
" PRE grdi-vnl-raster/\n",
" PRE grdi-vnl-slope-raster/\n",
" PRE houston-aod/\n",
" PRE houston-landcover/\n",
" PRE houston-lst-day/\n",
" PRE houston-lst-diff/\n",
" PRE houston-lst-night/\n",
" PRE houston-ndvi/\n",
" PRE ingests/\n",
" PRE l8-water-quality/\n",
" PRE ls8-surface-temperature-covid-19-changing-landscapes/\n",
" PRE modis-fire-anomalies-diff-covid-19-changing-landscape/\n",
" PRE modis_diff/\n",
" PRE nightlights-500m-daily/\n",
" PRE nightlights-hd-1band/\n",
" PRE nightlights-hd-3bands/\n",
" PRE nightlights-hd-monthly/\n",
" PRE no2-monthly-diff/\n",
" PRE no2-monthly/\n",
" PRE pzd-anomaly-covid-19/\n",
" PRE recovery-proxy-maps-covid-19/\n",
" PRE slowdown-proxy-covid-19-changing-landscapes/\n",
" PRE social-vulnerability-index-household-nopop/\n",
" PRE social-vulnerability-index-household/\n",
" PRE social-vulnerability-index-housing-nopop/\n",
" PRE social-vulnerability-index-housing/\n",
" PRE social-vulnerability-index-minority-nopop/\n",
" PRE social-vulnerability-index-minority/\n",
" PRE social-vulnerability-index-overall-nopop/\n",
" PRE social-vulnerability-index-overall/\n",
" PRE social-vulnerability-index-socioeconomic-nopop/\n",
" PRE social-vulnerability-index-socioeconomic/\n",
" PRE sport-lis-vsm0_100cm-percentile/\n",
" PRE togo-agriculture-covid-19/\n",
" PRE urbanization/\n",
" PRE veda-auth-testing/\n",
" PRE veda-data-store-staging/\n",
"2023-02-03 15:51:47 48 example-file.txt\n"
]
}
],
"source": [
"!aws s3 ls s3://veda-data-store-staging/"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "2b7db68a-6c0c-4d71-890b-cfdef2dd81a7",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"collection\": \"modis_diff\",\n",
" \"title\": \"Camp Fire Domain: MODIS NDVI Difference\",\n",
" \"spatial_extent\": {\n",
" \"xmin\": -122.21,\n",
" \"ymin\": 39.33,\n",
" \"xmax\": -120.91,\n",
" \"ymax\": 40.22\n",
" },\n",
" \"temporal_extent\": {\n",
" \"startdate\": \"2022-12-31T00:00:00Z\",\n",
" \"enddate\": \"2022-12-31T23:59:59Z\"\n",
" },\n",
" \"data_type\": \"cog\",\n",
" \"license\": \"CC0\",\n",
" \"description\": \"MODIS NDVI difference from a three-year average of 2015 to 2018 subtracted from a three-year average of 2019-2022. These tri-annual averages represent periods before and after the fire.\",\n",
" \"is_periodic\": true,\n",
" \"time_density\": \"year\",\n",
" \"sample_files\": [\n",
" \"s3://modis_diff/campfire_ndvi_{YEAR1}_{YEAR2}.tif\"\n",
" ],\n",
" \"discovery_items\": [\n",
" {\n",
" \"discovery\": \"s3\",\n",
" \"prefix\": \"modis_diff/\",\n",
" \"bucket\": \"veda-data-store-staging\",\n",
" \"filename_regex\": \"(.*)campfire_ndvi_{YEAR1}_{YEAR2}.tif$\"\n",
" }\n",
" ]\n",
"}\n"
]
}
],
"source": [
"dataset = {\n",
" \"collection\": \"modis_diff\",\n",
" \"title\": \"Camp Fire Domain: MODIS NDVI Difference\",\n",
" \"spatial_extent\": {\n",
" \"xmin\": -122.21,\n",
" \"ymin\": 39.33,\n",
" \"xmax\": -120.91,\n",
" \"ymax\": 40.22,\n",
" },\n",
" \"temporal_extent\": {\n",
" \"startdate\": \"2022-12-31T00:00:00Z\",\n",
" \"enddate\": \"2022-12-31T23:59:59Z\",\n",
" },\n",
" 'data_type': \"cog\",\n",
" \"license\": \"CC0\",\n",
" \"description\": \"MODIS NDVI difference from a three-year average of 2015 to 2018 subtracted from a three-year average of 2019-2022. These tri-annual averages represent periods before and after the fire.\",\n",
" \"is_periodic\": True,\n",
" \"time_density\": \"year\",\n",
" \"sample_files\": [S3_FILE_LOCATION],\n",
" \"discovery_items\": [\n",
" {\n",
" \"discovery\": \"s3\",\n",
" \"prefix\": \"modis_diff/\",\n",
" \"bucket\": \"veda-data-store-staging\",\n",
" \"filename_regex\": f\"(.*){TARGET_FILENAME}$\",\n",
" }\n",
" ],\n",
"}\n",
"import json\n",
"\n",
"print(json.dumps(dataset, indent=2))"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "b3d3cbe8-6536-4692-ba92-06680b6e224a",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\"[{'loc': ('body', '__root__'), 'msg': \\\"Invalid sample files - ['s3://modis_diff/campfire_ndvi_{YEAR1}_{YEAR2}.tif'] do not match anyof the provided prefix/filename_regex combinations.\\\", 'type': 'value_error'}]\"\n"
]
}
],
"source": [
"auth_header = f\"Bearer {TOKEN}\"\n",
"headers = {\n",
" \"Authorization\": auth_header,\n",
" \"content-type\": \"application/json\",\n",
" \"accept\": \"application/json\",\n",
"}\n",
"response = requests.post((API + \"dataset/validate\"), json=dataset, headers=headers)\n",
"# response.raise_for_status()\n",
"print(response.text)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "012d191c-de6a-42bf-ab1f-ac442450351d",
"metadata": {},
"outputs": [],
"source": [
"response = requests.post((API + \"dataset/publish\"), json=dataset, headers=headers)\n",
"response.raise_for_status()\n",
"print(response.text)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment