Download script that handles username and password or token authentication suitable for downloading eg. assets from huggingface

download_with_auth.py

#!/usr/bin/env python3

import sys
import subprocess
import requests
import pathlib
import functools
import shutil
import tqdm

def download(url, filename, username, token):
    # This following is based on: https://stackoverflow.com/a/63831244
    r = requests.get(url, stream=True, allow_redirects=True, auth=(username, token))
    if r.status_code != 200:
        r.raise_for_status()  # Will only raise for 4xx codes, so...
        raise RuntimeError(f"Request to {url} returned status code {r.status_code}")
    file_size = int(r.headers.get('Content-Length', 0))

    path = pathlib.Path(filename).expanduser().resolve()
    path.parent.mkdir(parents=True, exist_ok=True)

    desc = "(Unknown total file size)" if file_size == 0 else ""
    r.raw.read = functools.partial(r.raw.read, decode_content=True)  # Decompress if needed
    with tqdm.tqdm.wrapattr(r.raw, "read", total=file_size, desc=desc) as r_raw:
        with path.open("wb") as f:
            shutil.copyfileobj(r_raw, f)

    return path

def usage(name):
    print("""Usage:
  %s URL FILENAME USERNAME TOKEN [HASH-CMD]

HASH-CMD may specify a command to run on the file after download, eg:
  sha256sum
  md5sum
""" % name)

if __name__ == "__main__":
    if len(sys.argv) != 5 and len(sys.argv) != 6:
        usage(sys.argv[0])
    else:
        (url, filename, username, token) = (
            sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
        download(url, filename, username, token)

        if len(sys.argv) == 6:
            cmd = sys.argv[5]
            print("Running: %s on file ..." % cmd)
            subprocess.run([cmd, filename], text=True)