Created
April 5, 2022 15:02
-
-
Save rsignell-usgs/5adfcf89b9bed3bc8c79e0a58aa5f9c5 to your computer and use it in GitHub Desktop.
tifffile_zarr_datacube.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "id": "99efa358-da83-4f91-b53b-3e23d44a06dd", | |
| "metadata": {}, | |
| "source": [ | |
| "# Use TIFFfile to create a Zarr virtual datacube " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "id": "4da28c96-27db-4225-a93a-d29e08a9bd05", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from tifffile import TiffSequence\n", | |
| "import zarr\n", | |
| "import fsspec" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "8292366f-30f4-4fb4-b6c1-68c341f27bc8", | |
| "metadata": {}, | |
| "source": [ | |
| "### Try local tiff files first:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "id": "c3df7a5f-90aa-4d8a-ace6-5a7426f1d55f", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "['/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif',\n", | |
| " '/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/lcmap/cog/LCMAP_CU_2020_V12_LCPRI.tif']" | |
| ] | |
| }, | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "fs = fsspec.filesystem('file')\n", | |
| "flist = fs.ls('/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/lcmap/cog/')\n", | |
| "flist" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "id": "5e7d047e-260b-4ca3-bd57-8d6b26bf2e18", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "'2019'" | |
| ] | |
| }, | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "import re\n", | |
| "ex = re.compile(r\".*_(\\d+)_V12_LCPRI.tif\")\n", | |
| "ex.search(flist[0]).groups()[0]" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "id": "7bbb23c4-0ace-477e-8a4f-8bea6d78c607", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "image_sequence = TiffSequence(flist, pattern=r'.*_(\\d+)_V12_LCPRI.tif')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "id": "7ac5d6b2-937a-43a8-962f-d8d335cf0f31", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "(2,)" | |
| ] | |
| }, | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "image_sequence.shape" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "id": "25861d49-b887-4192-b40e-58efb759fa57", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "with image_sequence.aszarr() as store:\n", | |
| " dz = zarr.open(store, mode='r')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "id": "e1cb07fb-c50c-412e-8cac-5b5190315aa1", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| " image_sequence.close()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "id": "02c9bb19-8403-49bd-9adb-685c447e22ac", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "with image_sequence.aszarr() as store:\n", | |
| " store.write_fsspec('temp.json', url='file://')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 9, | |
| "id": "147f3a8b-de49-465d-b3a8-2d3946373853", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import fsspec\n", | |
| "import tifffile.numcodecs\n", | |
| "tifffile.numcodecs.register_codec()\n", | |
| "mapper = fsspec.get_mapper('reference://', fo='temp.json', target_protocol='file')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 10, | |
| "id": "fe2b4a5f-de78-43b8-9d93-1cc5bbf6f0b8", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "ds = zarr.open(mapper, mode='r')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 11, | |
| "id": "b96c790b-d985-4430-8c0c-5aebacb90cc7", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<table class=\"zarr-info\"><tbody><tr><th style=\"text-align: left\">Type</th><td style=\"text-align: left\">zarr.core.Array</td></tr><tr><th style=\"text-align: left\">Data type</th><td style=\"text-align: left\">uint8</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(2, 105000, 160000)</td></tr><tr><th style=\"text-align: left\">Chunk shape</th><td style=\"text-align: left\">(1, 105000, 160000)</td></tr><tr><th style=\"text-align: left\">Order</th><td style=\"text-align: left\">C</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">True</td></tr><tr><th style=\"text-align: left\">Compressor</th><td style=\"text-align: left\">Tiff(bigtiff=None, byteorder=None, compression=None, extratags=(), imagej=False, key=None, level=None, maxworkers=None, metadata={}, ome=None, photometric=None, planarconfig=None, predictor=None, rowsperstrip=None, series=None, subsampling=None, tile=None, truncate=False, volumetric=None)</td></tr><tr><th style=\"text-align: left\">Store type</th><td style=\"text-align: left\">zarr.storage.KVStore</td></tr><tr><th style=\"text-align: left\">No. bytes</th><td style=\"text-align: left\">33600000000 (31.3G)</td></tr><tr><th style=\"text-align: left\">No. bytes stored</th><td style=\"text-align: left\">215</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">156279069.8</td></tr><tr><th style=\"text-align: left\">Chunks initialized</th><td style=\"text-align: left\">2/2</td></tr></tbody></table>" | |
| ], | |
| "text/plain": [ | |
| "Type : zarr.core.Array\n", | |
| "Data type : uint8\n", | |
| "Shape : (2, 105000, 160000)\n", | |
| "Chunk shape : (1, 105000, 160000)\n", | |
| "Order : C\n", | |
| "Read-only : True\n", | |
| "Compressor : Tiff(bigtiff=None, byteorder=None, compression=None,\n", | |
| " : extratags=(), imagej=False, key=None, level=None,\n", | |
| " : maxworkers=None, metadata={}, ome=None, photometric=None,\n", | |
| " : planarconfig=None, predictor=None, rowsperstrip=None,\n", | |
| " : series=None, subsampling=None, tile=None, truncate=False,\n", | |
| " : volumetric=None)\n", | |
| "Store type : zarr.storage.KVStore\n", | |
| "No. bytes : 33600000000 (31.3G)\n", | |
| "No. bytes stored : 215\n", | |
| "Storage ratio : 156279069.8\n", | |
| "Chunks initialized : 2/2" | |
| ] | |
| }, | |
| "execution_count": 11, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "ds.info" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "6ff4cdc8-8bee-4ece-bce0-5ad476405958", | |
| "metadata": {}, | |
| "source": [ | |
| "### Try accessing files Open Storage Network\n", | |
| "The [Open Storage Network](https://www.openstoragenetwork.org/) is an NSF-supported program that provides about one PB of locally installed, remotely managed, s3-compatible object storage for about $30K/year." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 12, | |
| "id": "1a15a199-73ce-4121-8956-17ee688e0f64", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "fs = fsspec.filesystem('s3', anon=True, client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'})" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 13, | |
| "id": "6602dadd-3995-4820-9b5d-6911a10fc146", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "['rsignellbucket1/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif',\n", | |
| " 'rsignellbucket1/lcmap/cog/LCMAP_CU_2020_V12_LCPRI.tif']" | |
| ] | |
| }, | |
| "execution_count": 13, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "flist = fs.ls('/rsignellbucket1/lcmap/cog')\n", | |
| "flist" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 14, | |
| "id": "5cbef221-a45a-4dad-9671-04ce72b13db7", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "s3_flist = [f's3://{f}' for f in flist]\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 15, | |
| "id": "bc2f20e7-cc25-4b66-b116-20d0467038b6", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "image_sequence = TiffSequence(s3_flist, pattern=r'.*_(\\d+)_V12_LCPRI.tif')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 16, | |
| "id": "5323540d-97f9-4352-a778-928c0f6366f1", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "ename": "FileNotFoundError", | |
| "evalue": "[Errno 2] No such file or directory: '/home/rsignell/EarthMap/Projects/esip-qhub-notebooks/LCMAP/s3:/rsignellbucket1/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif'", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", | |
| "Input \u001b[0;32mIn [16]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mimage_sequence\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maszarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m store:\n\u001b[1;32m 2\u001b[0m dz \u001b[38;5;241m=\u001b[39m zarr\u001b[38;5;241m.\u001b[39mopen(store, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10292\u001b[0m, in \u001b[0;36mFileSequence.aszarr\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 10290\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21maszarr\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 10291\u001b[0m \u001b[38;5;124;03m\"\"\"Return image data from files as zarr storage.\"\"\"\u001b[39;00m\n\u001b[0;32m> 10292\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mZarrFileSequenceStore\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:9828\u001b[0m, in \u001b[0;36mZarrFileSequenceStore.__init__\u001b[0;34m(self, arg, fillvalue, chunkmode, chunkshape, dtype, axestiled, zattrs, **kwargs)\u001b[0m\n\u001b[1;32m 9825\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_commonpath \u001b[38;5;241m=\u001b[39m arg\u001b[38;5;241m.\u001b[39mcommonpath()\n\u001b[1;32m 9827\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunkshape \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m dtype \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 9828\u001b[0m chunk \u001b[38;5;241m=\u001b[39m \u001b[43marg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimread\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9829\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_chunks \u001b[38;5;241m=\u001b[39m chunk\u001b[38;5;241m.\u001b[39mshape\n\u001b[1;32m 9830\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dtype \u001b[38;5;241m=\u001b[39m chunk\u001b[38;5;241m.\u001b[39mdtype\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:820\u001b[0m, in \u001b[0;36mimread\u001b[0;34m(files, aszarr, **kwargs)\u001b[0m\n\u001b[1;32m 817\u001b[0m files \u001b[38;5;241m=\u001b[39m files[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 819\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(files, (\u001b[38;5;28mstr\u001b[39m, os\u001b[38;5;241m.\u001b[39mPathLike)) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(files, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mseek\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m--> 820\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mTiffFile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs_file\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m tif:\n\u001b[1;32m 821\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m aszarr:\n\u001b[1;32m 822\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m tif\u001b[38;5;241m.\u001b[39maszarr(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:3142\u001b[0m, in \u001b[0;36mTiffFile.__init__\u001b[0;34m(self, arg, mode, name, offset, size, _multifile, _useframes, _parent, **kwargs)\u001b[0m\n\u001b[1;32m 3139\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m (\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr+b\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m 3140\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minvalid mode \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmode\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m-> 3142\u001b[0m fh \u001b[38;5;241m=\u001b[39m \u001b[43mFileHandle\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msize\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3143\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fh \u001b[38;5;241m=\u001b[39m fh\n\u001b[1;32m 3144\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_multifile \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mbool\u001b[39m(_multifile)\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10509\u001b[0m, in \u001b[0;36mFileHandle.__init__\u001b[0;34m(self, file, mode, name, offset, size)\u001b[0m\n\u001b[1;32m 10507\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 10508\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock \u001b[38;5;241m=\u001b[39m NullContext()\n\u001b[0;32m> 10509\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10522\u001b[0m, in \u001b[0;36mFileHandle.open\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 10520\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_file \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mrealpath(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_file)\n\u001b[1;32m 10521\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dir, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_name \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_file)\n\u001b[0;32m> 10522\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fh \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_file\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 10523\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 10524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_offset \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", | |
| "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/rsignell/EarthMap/Projects/esip-qhub-notebooks/LCMAP/s3:/rsignellbucket1/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif'" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "with image_sequence.aszarr() as store:\n", | |
| " dz = zarr.open(store, mode='r')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 17, | |
| "id": "37947e06-f8ce-4acf-b3fd-0bc8d34fadae", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "file_like_object_list = [fs.open(f) for f in flist]" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 18, | |
| "id": "1e49bc9e-dc42-4578-b51f-e0a17a5ff8ac", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "ename": "TypeError", | |
| "evalue": "expected str, bytes or os.PathLike object, not S3File", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", | |
| "Input \u001b[0;32mIn [18]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m image_sequence \u001b[38;5;241m=\u001b[39m \u001b[43mTiffSequence\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile_like_object_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpattern\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m.*_(\u001b[39;49m\u001b[38;5;124;43m\\\u001b[39;49m\u001b[38;5;124;43md+)_V12_LCPRI.tif\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10308\u001b[0m, in \u001b[0;36mTiffSequence.__init__\u001b[0;34m(self, files, imread, **kwargs)\u001b[0m\n\u001b[1;32m 10306\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, files\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, imread\u001b[38;5;241m=\u001b[39mimread, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 10307\u001b[0m \u001b[38;5;124;03m\"\"\"Initialize instance from multiple TIFF files.\"\"\"\u001b[39;00m\n\u001b[0;32m> 10308\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mimread\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m*.tif\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10105\u001b[0m, in \u001b[0;36mFileSequence.__init__\u001b[0;34m(self, imread, files, container, sort, parse, **kwargs)\u001b[0m\n\u001b[1;32m 10102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m sort:\n\u001b[1;32m 10103\u001b[0m files \u001b[38;5;241m=\u001b[39m sort(files)\n\u001b[0;32m> 10105\u001b[0m files \u001b[38;5;241m=\u001b[39m [os\u001b[38;5;241m.\u001b[39mfspath(f) \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m files]\n\u001b[1;32m 10106\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m files:\n\u001b[1;32m 10107\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno files found\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", | |
| "File \u001b[0;32m~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10105\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 10102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m sort:\n\u001b[1;32m 10103\u001b[0m files \u001b[38;5;241m=\u001b[39m sort(files)\n\u001b[0;32m> 10105\u001b[0m files \u001b[38;5;241m=\u001b[39m [\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfspath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m files]\n\u001b[1;32m 10106\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m files:\n\u001b[1;32m 10107\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno files found\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", | |
| "\u001b[0;31mTypeError\u001b[0m: expected str, bytes or os.PathLike object, not S3File" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "image_sequence = TiffSequence(file_like_object_list, pattern=r'.*_(\\d+)_V12_LCPRI.tif')" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python [conda env:pangeo]", | |
| "language": "python", | |
| "name": "conda-env-pangeo-py" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.9.12" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 5 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment