Skip to content

Instantly share code, notes, and snippets.

@Kirill888

Kirill888/trigger-read-error-rio.ipynb

Created November 12, 2019 02:13
Show Gist options
  • Save Kirill888/55148f21e0dcc2cf3d88e9e6abd349f7 to your computer and use it in GitHub Desktop.
Save Kirill888/55148f21e0dcc2cf3d88e9e6abd349f7 to your computer and use it in GitHub Desktop.
Download ZIP
supporting code for bug report to mapbox/rasterio
Display the source blob
Display the rendered blob
Raw
trigger-read-error-rio.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import rasterio\n",
"from rasterio.session import AWSSession\n",
"import hashlib\n",
"from concurrent.futures import ThreadPoolExecutor\n",
"import threading\n",
"import sys\n",
"\n",
"_lcl = threading.local()\n",
"\n",
"def pixel_sha1(src, stripe=None):\n",
" \"\"\" Read every pixel and compute sha1 of the data.\n",
" \"\"\"\n",
" _hash = hashlib.sha1()\n",
" if stripe is None:\n",
" pix = src.read(1)\n",
" _hash.update(pix.data)\n",
" else:\n",
" H,W = src.shape\n",
"\n",
" for y in range(H//stripe + 1):\n",
" h = min(stripe, H-y*stripe)\n",
" if h == 0:\n",
" break\n",
"\n",
" window = rasterio.windows.Window(0, y*stripe, W, h)\n",
" pix = src.read(1, window=window)\n",
" _hash.update(pix.data)\n",
" \n",
" return _hash.hexdigest()\n",
"\n",
"def test_workload(url, rio_opts, stripe=None):\n",
" \"\"\" url -> (url, hash, None ) -- if all good\n",
" (ulr, None, error) -- if any error occured\n",
" \"\"\"\n",
" # activate rasterio.Env(..) once per thread\n",
" # we assume aws_unsigned=True is used\n",
" if getattr(_lcl, \"env\", None) is None:\n",
" _lcl.env = rasterio.Env(**rio_opts)\n",
" _lcl.env.__enter__()\n",
" \n",
" try:\n",
" with rasterio.open(url, sharing=False) as src:\n",
" return (url, pixel_sha1(src, stripe), None)\n",
" except Exception as e:\n",
" return (url, None, e)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"jupyter": {
"source_hidden": true
}
},
"outputs": [],
"source": [
"urls = ['s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B10.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B11.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B2.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B3.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B4.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B5.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B6.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B7.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B8.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B9.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_BQA.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B1.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B10.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B11.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B2.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B3.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B4.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B5.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B6.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B7.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B8.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B9.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_BQA.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B1.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B10.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B11.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B2.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B3.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B4.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B5.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B6.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B7.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B8.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B9.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_BQA.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B1.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B10.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B11.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B2.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B3.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B4.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B5.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B6.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B7.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B8.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B9.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_BQA.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B1.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B10.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B11.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B2.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B3.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B4.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B5.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B6.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B7.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B8.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B9.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_BQA.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B1.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B10.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B11.TIF',\n",
" 's3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B2.TIF']"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"nthreads = 32\n",
"exit_on_error = False\n",
"save_hashes = False\n",
"stripe = None # read whole file at once\n",
"\n",
"rio_opts = dict(region_name='us-west-2',\n",
" aws_unsigned=True,\n",
" #GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',\n",
" #CPL_VSIL_CURL_ALLOWED_EXTENSIONS='tif',\n",
" #GDAL_HTTP_MAX_RETRY='10',\n",
" #GDAL_HTTP_RETRY_DELAY='0.5',\n",
" #CPL_VSIL_CURL_NON_CACHED=\"/vsis3/\",\n",
" #CPL_VSIL_CURL_USE_CACHE=False,\n",
" #VSI_CACHE=False,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Failed: s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF\n",
"Failed: s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B10.TIF\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B11.TIF 07fb86ffa4085dbeb128dd4890e6661351500731\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B2.TIF f3b8086f25b701046f4efa3cff30decd5b3ece98\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B3.TIF 8cf783f809a257b9b7ece6d5fceef4da19e82f6a\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B4.TIF 89f7e47fa8ca79942ef914f779eb63de14f52a87\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B5.TIF f53e95ef5096373aecc2c8b484e63f7ab671d430\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B6.TIF becab7320df7f361a83ba1694b8ac34302830389\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B7.TIF fb063b379a91eeac4f605fe934c09e2209175d52\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B8.TIF f8695f9db11846ca01ea418718b9ac53a474c6b2\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B9.TIF 6e5d9a30befabefa9fb846a5395e117f963bfc51\n",
"s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_BQA.TIF 081386de02a74cbdde16450fd2ef4c9463e21514\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B1.TIF 63f32528ab2add07f946f35c1d26fe4e366cfce0\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B10.TIF e59f1d18a1ab5c5bc5db9c8650ec8004772992cc\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B11.TIF 3d409c14fae89cc48bcb5a8eba888d9fc9cabc9b\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B2.TIF 43862198305c5abe8d9b952d0f3daa358b77fd0c\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B3.TIF 1775e210c74de0e43027bb2396bc90081a7431da\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B4.TIF 93f8834d55e1840cb23d87fd93a2e7c18d314521\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B5.TIF dc3e9c4611fbde0d1ff7ef58cace8687122f7dbf\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B6.TIF eecdea7fe0915b99e4079befa657453dda8bd173\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B7.TIF ba89cf2a2512b76350d002daeb66fc5159db6224\n",
"Failed: s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B8.TIF\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B9.TIF 6de319d515c4c0e224673570571eb5b944e073a7\n",
"s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_BQA.TIF b40efea664591ec43082db88adeb658e9bd5cac6\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B1.TIF 6d30e19ce1a62e1dacff0d7d3668222cc03bfb06\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B10.TIF 4642a610f941cc9a9c02c7d974b4962503095222\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B11.TIF 10de8a602e4e15583aa38663a3694e6aa7bb6e02\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B2.TIF 5b173b749399862001616a5bc092e6e6a9b7916d\n",
"Failed: s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B3.TIF\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B4.TIF 7952d2f41b98ca713495482a2a6c712ab34f70b4\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B5.TIF f5b26746371ea37aba374247073d73401afd63af\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B6.TIF ddb0fc469bc93104f533a51668a8de24624e4645\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B7.TIF 82bcb97eb07a4fb31484df48f4eeace2bb8744e5\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B8.TIF de242daee29e524851f6fb0ec90c1ba1f3261ab1\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B9.TIF 6b1de5841e0904f9a2019c03d7af55fd8a3b9378\n",
"s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_BQA.TIF 2dcf3b7dda7552d75762a79e9e2884d7e4db2380\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B1.TIF 7ebb462d958e1d692f0fa3618d6353e9a9c5ac13\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B10.TIF 8a2924fefd2e52686bb4751c19598c2bbf38a8de\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B11.TIF fa29c7259cb952a171b217c611932525329b7587\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B2.TIF 9b35fa007bb29aad569054c73190fbb0e6148cab\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B3.TIF 612373bd6f8c56cc6caac3d87b9758276ad467aa\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B4.TIF e14639c564270c75c54ed170e194a0f205af6883\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B5.TIF 40937af4ae5566d77e7e5bc1a9beb3899a919422\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B6.TIF 44b610fd15f026bd2b8055ca9f635f9d1b6ff16e\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B7.TIF 335d0f5685b06adc33cd439d3197cc60611bcbd8\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B8.TIF 99b27390fc457c938a7d52ce0689b4415880654a\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_B9.TIF 7c4bb1010e81af00228c6628d208dd453fd71919\n",
"s3://landsat-pds/L8/100/072/LC81000722013195LGN00/LC81000722013195LGN00_BQA.TIF 5f0b37e467c983c1a518b8f505037b1e23fc8696\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B1.TIF cb88dbc8ff4e932d12d89b36b7f60b1f632179d9\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B10.TIF 411ec467a03b916ec9b0f6c9efe2ee6293269f4a\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B11.TIF 1e312e276a872dbd6647b8c03c30ac3cb44e288b\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B2.TIF efde285a5be95c98aba5728c8e231e3c6e5c6aa3\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B3.TIF 38d8a0d759b4b53d62ce38d51a7110b9979d679f\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B4.TIF 405e529c4b56cd92f51e19d22aab2e991ff4f18a\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B5.TIF 119b1e211b6f47c704672260e1746ee819e37a80\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B6.TIF 7fc6105c90eabcd4886808a1579bb353b7305de3\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B7.TIF 78b8ca2583b9d7561113beb443749abf87d4c3e9\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B8.TIF a65cd8dcd07ad544b525d3c7b0979c2c2703868f\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_B9.TIF 0dd20d773f45c547aab85d4c9d8ee398b991c484\n",
"s3://landsat-pds/L8/100/072/LC81000722013211LGN00/LC81000722013211LGN00_BQA.TIF 38fc585bb60e7e73360c16a5fbda990844fb9b6f\n",
"s3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B1.TIF 4779172dcd3135d582d5b26d505a18ada47e4d5d\n",
"s3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B10.TIF 109cbcb2f1edb47a1de3e77c35e222ed8f3eb0c3\n",
"s3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B11.TIF 3e53db3e857119e14036f046a4acc97a78e333a6\n",
"s3://landsat-pds/L8/100/072/LC81000722013227LGN00/LC81000722013227LGN00_B2.TIF 2a2769cf7b5062d894280974bfbe7f14e1e6c66c\n",
"CPU times: user 1min 44s, sys: 22.2 s, total: 2min 6s\n",
"Wall time: 15.8 s\n"
]
}
],
"source": [
"%%time\n",
"pool = ThreadPoolExecutor(max_workers=nthreads) \n",
"good = []\n",
"bad = []\n",
"\n",
"for (url, h, err) in pool.map(lambda url: test_workload(url, rio_opts, stripe=stripe),\n",
" urls):\n",
" if err:\n",
" print(f'Failed: {url}')\n",
" bad.append((url, err))\n",
" if exit_on_error:\n",
" break\n",
" else:\n",
" print(f'{url} {h}')\n",
" good.append((url, h))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF',\n",
" rasterio.errors.RasterioIOError('Read or write failed. /vsis3/landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF, band 1: IReadBlock failed at X offset 2, Y offset 11: TIFFReadEncodedTile() failed.')),\n",
" ('s3://landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B10.TIF',\n",
" rasterio.errors.RasterioIOError('Read or write failed. /vsis3/landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B10.TIF, band 1: IReadBlock failed at X offset 5, Y offset 7: TIFFReadEncodedTile() failed.')),\n",
" ('s3://landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B8.TIF',\n",
" rasterio.errors.RasterioIOError('Read or write failed. /vsis3/landsat-pds/L8/100/072/LC81000722013131LGN01/LC81000722013131LGN01_B8.TIF, band 1: IReadBlock failed at X offset 20, Y offset 6: TIFFReadEncodedTile() failed.')),\n",
" ('s3://landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B3.TIF',\n",
" rasterio.errors.RasterioIOError('Read or write failed. /vsis3/landsat-pds/L8/100/072/LC81000722013147LGN00/LC81000722013147LGN00_B3.TIF, band 1: IReadBlock failed at X offset 3, Y offset 9: TIFFReadEncodedTile() failed.'))]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bad"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"\n",
"if save_hashes:\n",
" with open(\"hashes.csv\", \"wt\") as f:\n",
" csv.writer(f).writerows(good)\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"ename": "RasterioIOError",
"evalue": "Read or write failed. /vsis3/landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF, band 1: IReadBlock failed at X offset 2, Y offset 11: TIFFReadEncodedTile() failed.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mCPLE_AppDefinedError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mrasterio/_io.pyx\u001b[0m in \u001b[0;36mrasterio._io.DatasetReaderBase._read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mrasterio/shim_rasterioex.pxi\u001b[0m in \u001b[0;36mrasterio._shim.io_multi_band\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mrasterio/_err.pyx\u001b[0m in \u001b[0;36mrasterio._err.exc_wrap_int\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mCPLE_AppDefinedError\u001b[0m: /vsis3/landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF, band 1: IReadBlock failed at X offset 2, Y offset 11: TIFFReadEncodedTile() failed.",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mRasterioIOError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-7-6ea40a64db9e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mbad\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-1-8052a1cc6ae9>\u001b[0m in \u001b[0;36mtest_workload\u001b[0;34m(url, rio_opts, stripe)\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mrasterio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msharing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpixel_sha1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstripe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 44\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-1-8052a1cc6ae9>\u001b[0m in \u001b[0;36mpixel_sha1\u001b[0;34m(src, stripe)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0m_hash\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhashlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msha1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstripe\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mpix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 16\u001b[0m \u001b[0m_hash\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32mrasterio/_io.pyx\u001b[0m in \u001b[0;36mrasterio._io.DatasetReaderBase.read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mrasterio/_io.pyx\u001b[0m in \u001b[0;36mrasterio._io.DatasetReaderBase._read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mRasterioIOError\u001b[0m: Read or write failed. /vsis3/landsat-pds/L8/100/072/LC81000722013115LGN01/LC81000722013115LGN01_B1.TIF, band 1: IReadBlock failed at X offset 2, Y offset 11: TIFFReadEncodedTile() failed."
]
}
],
"source": [
"raise bad[0][1]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment