huggingface download script

hfd.sh

#!/bin/bash  
  
# Set the environment variable  
HF_ENDPOINT="https://hf-mirror.com"  
  
# Define the maximum number of retries, e.g., retry up to 5 times  
MAX_RETRIES=5  
RETRIES=0  
  
# Loop until we either succeed or reach the maximum number of retries  
while [ $RETRIES -lt $MAX_RETRIES ]; do  
    echo "Attempting to download data (attempt #$((RETRIES + 1)))..."  
    python download_only_train_data.py  
  
    # Check the exit status of the last command  
    if [ $? -eq 0 ]; then  
        echo "Data download succeeded!"  
        break  # Exit the loop if successful  
    else  
        # Here we simply assume any non-zero exit status indicates a network issue or failure  
        echo "Data download failed, retrying..."  
        RETRIES=$((RETRIES + 1))  
        # Wait for a period of time before retrying, e.g., wait for 5 seconds  
        sleep 5  
    fi  
done  
  
# If we reach the maximum number of retries without success  
if [ $RETRIES -eq $MAX_RETRIES ]; then  
    echo "Reached maximum number of retries, data download failed."  
    exit 1  # Exit the script with an error code  
fi