check-etag.sh

#! /bin/bash
# Check git lfs file's integrity in ~/.cache/huggingface/transformers by it etag
# Run this script in MINGW64 if you are on Windows
# Root of all evil: https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin
# I spent a whole day troubleshooting AUTOMATIC1111/stable-diffusion-webui bugs due to incomplete file downloads

# OSError: Unable to load weights from pytorch checkpoint file for '~/.cache/huggingface/transformers/c506559a5367a918bab46c39c79af91ab88846b49c8abd9d09e699ae067505c6.6365d436cc844f2f2b4885629b559d8ff0938ac484c01a6796538b2665de96c7'.
# If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True.

path=~/.cache/huggingface/transformers
# Hopefully the value of etag key is indeed the checksum of the file.
lfs_pattern='\.((bin|lfs|tar)(\..*)?|h5|pb|pth?|7z|bz2|gz|tgz|xz|zip|rar)$'

function check_lfs_file_checksum_by_etag() {
    local file="$1"
    local url=$(jq -r .url $file)
    printf "$url" | grep -qE $lfs_pattern || return

    local etag=$(jq -r .etag $file | sed 's/"//g')
    local etag_len=$(printf $etag | wc -c)

    local checksum_prog
    case "$etag_len" in
        32) checksum_prog=md5sum ;;
        40) checksum_prog=sha1sum ;;
        56) checksum_prog=sha224sum ;;
        64) checksum_prog=sha256sum ;;
        96) checksum_prog=sha384sum ;;
        128) checksum_prog=sha512sum ;;
        *) return ;;
    esac

    printf "%s\n" "$url"
    printf "%s %s" "$etag" "${file%.json}" | $checksum_prog -c -
}

if [ -d "$path" ]; then
    cd "$path"
    for f in `ls *.json`; do
        check_lfs_file_checksum_by_etag "$f"
    done
fi