for when huggingface trainer decides your files are too big to track intermediate chkpts during training

lfs_checkpoint_uploader.sh

#!/bin/bash

# install: sudo apt-get install inotify-tools
# Usage: ./scriptname.sh /path/to/monitor/directory /path/to/repo/directory
# If no monitor directory is passed, monitor directory = repo directory
# put & at the end of the command to run in background

# Define your monitor directory
MONITOR_DIR="${1:-$2}"
if [ -z "$MONITOR_DIR" ]; then
    echo "Error: You must provide at least one directory path."
    exit 1
fi

# Define your repository directory
REPO_DIR="${2:-$1}"
if [ -z "$REPO_DIR" ]; then
    echo "Error: You must provide at least one directory path."
    exit 1
fi

# Start an infinite loop that watches for new directories in the monitor directory
inotifywait -m "$MONITOR_DIR" -e create -e moved_to --format '%w%f' |
    while read path; do
        # Continue only if the path is a directory
        if [[ -d $path ]]; then
            dir=$(basename "$path")
            echo "The directory '$dir' appeared in directory via 'create' or 'moved_to'"

            # If the new directory is a checkpoint directory
            if [[ "$dir" =~ ^checkpoint-[0-9]+ ]]; then
                echo "New checkpoint detected: $dir"
                date
                sleep 60 # Wait for 60 seconds to make sure the checkpoint is complete
                # Copy the new checkpoint files into the repo directory
                cp $MONITOR_DIR/$dir/*.* $REPO_DIR/

                # Remove the *.pt* files
                rm $REPO_DIR/*.pt*

                # Move to the repo directory
                cd "$REPO_DIR"

                # Perform the git operations
                git lfs track *.bin
                git add *.*
                git commit -a -m "$dir"
                git push

                # Move back to the monitor directory
                cd "$MONITOR_DIR"
            fi
        fi
    done