ESRI ImageServer downloader

batch_download.py

#!/usr/bin/env python3

import os
from tqdm.auto import tqdm
from tqdm.contrib.concurrent import process_map, thread_map
import json
import requests
import time

url = "https://nrcsgeoservices.sc.egov.usda.gov/arcgis/rest/services/ortho_imagery/marshall_islands_vivid_2023_30cm/ImageServer"

try:
    with open("features.json") as f:
        features = json.load(f)
except:
    features = requests.get(url + "/query", params={
        "query": "1=1",
        "f": "pjson",
        "outFields": "*"
    }).json()["features"]
    features.extend(requests.get(url + "/query", params={
        "query": "1=1",
        "f": "pjson",
        "outFields": "*",
        "resultOffset": 1000
    }).json()["features"])
    with open("features.json", "w") as f:
        json.dump(features, f)

try:
    with open("files.json") as f:
        files = json.load(f)
except:
    def get_slice(i):
        top = min(len(features), i+6)
        return requests.get(url + "/download", params={
            "rasterIds": ",".join(str(j) for j in range(i, top)),
            "f": "pjson"
        }).json()["rasterFiles"]
    files = []
    for slice in thread_map(get_slice, range(1, len(features), 6), max_workers=64):
        files.extend(slice)
    with open("files.json", "w") as f:
        json.dump(files, f)

download_dir = "marshall_islands_vivid_2023_30cm"
os.makedirs(download_dir, exist_ok=True)

missing = []
for f in tqdm(files):
    # {"id": ".\\marshall_islands_vivid_2023_30cm\\raster_tiles\\Vivid_Standard_30_a10a5af1-55b0-4609-8ec4-bb789d817202-APS_1311122112021.tif", "size": 383878537, "rasterIds": [1]}
    name = f["id"].split("\\")[-1]
    if not os.path.isfile(os.path.join(download_dir, name)):
        missing.append(f)
        continue
    expected_size_bytes = f["size"]
    actual_size_bytes = os.path.getsize(os.path.join(download_dir, name))
    if expected_size_bytes != actual_size_bytes:
        print(f"expected {expected_size_bytes} bytes, got {actual_size_bytes} bytes for {name}")
        missing.append(f)

def download(feature, force=True):
    OBJECTID = feature["rasterIds"][0]
    name = feature["id"].split("\\")[-1]
    tif_name = name
    if force or not os.path.isfile(os.path.join(download_dir, tif_name)):
        for retry in range(5):
            try:
                r = requests.get(url + "/file", params={
                    "id": rf'.\\marshall_islands_vivid_2023_30cm\\raster_tiles\\{tif_name}',
                    "rasterId": OBJECTID
                })
                print(r)
                print(r.url)
                r.raise_for_status()
                with open(os.path.join(download_dir, tif_name), "wb") as f:
                    f.write(r.content)
                break
            except requests.exceptions.RequestException as e:
                print(f"{e} while downloading {tif_name}, retrying in 5s")
                time.sleep(5)
    tif_aux_name = name + ".aux.xml"
    if not os.path.isfile(os.path.join(download_dir, tif_aux_name)):
        try:
            r = requests.get(url + "/file", params={
                "id": rf'.\\marshall_islands_vivid_2023_30cm\\raster_tiles\\{tif_aux_name}',
                "rasterId": OBJECTID
            })
            r.raise_for_status()
            with open(os.path.join(download_dir, tif_aux_name), "wb") as f:
                f.write(r.content)
        except requests.exceptions.RequestException as e:
            print(f"{e} while downloading {tif_aux_name}")

thread_map(download, missing, max_workers=64)