Parallel rsync

para_rsync.sh

#!/bin/bash

# Note: Adjust rsync flags as needed for your specific requirements
# Author: Hiran P 
# Date: 2024-07-19
# Description: A script to perform parallel rsync operations with optional continuous sync using inotifywait

#### Usage ####
## ./para_rsync.sh --exclude-from /path/to/exclude_me.txt

#### Info ####
## For AutoBackup,save in /etc/cron.daily/backup
##link# curl https://gist.githubusercontent.com/hiranp/06d74c84c4efb1ce7478b9471261cd54/raw/13bbee84355602518835572a17fe50b9502e9e2e/para_rsync.sh> ~/sync_backup.sh | chmod +x ~/sync_backup.sh | cp -r ~/sync_backup.sh /etc/cron.daily/sync_backup | chmod +x /etc/cron.daily/sync_backup;

##$ rsync -a --delete --quiet $SOURCE_DIR $DESTINATION;
##$ rsync -aAXv --delete-after --exclude={"/dev/*","/proc/*","/sys/*","/tmp/*","/run/*","/mnt/*","/media/*","/lost+found"} / /mnt/backup;
##$ rsync -aAXv --dry-run --delete-after --exclude-from="skipme.txt" / /mnt/backup;
##$ time && find . -mindepth 1 -type d | xargs -I {} -P 4 rsync --dry-run -aAXv --delete-after --exclude-from="skipme.txt" {} ../backup;
##$ time && find . -mindepth 1 -type d | xargs -0 -I {} -P 4 rsync --dry-run -vhzr --partial --inplace --delete-after --exclude-from="skipme.txt" {} ../backup;
## time && find . -mindepth 1 -type d -print0 | xargs -0 -I {} -P 2 rsync --dry-run -a --bwlimit=1000 --delete-after --exclude-from="skipme.txt" {} ../backup;

#### Script ####
CONTINUOUS_SYNC=${CONTINUOUS_SYNC:-0} # Default to 0 if not set in the environment

# Source and destination for rsync
RSYSNC_FLAGS='-a --bwlimit=1000 --update --delete-after'
SOURCE_DIR='/srv/data/'
DEST_SERVER='myserver.com'
DEST_DIR='/srv/backup/'
DESTINATION="$DEST_SERVER:$DEST_DIR"
# Default exclude patterns file
EXCLUDE_FILE='/home/hiran/exclude_me.txt'
# Number of parallel processes
PARALLEL_PROCESSES=4
# Sync log file
LOG_FILE='/var/log/para_rsync.log'

# Parse command-line arguments
while [[ "$#" -gt 0 ]]; do
    case $1 in
        --exclude-from) EXCLUDE_FILE="$2"; shift ;;
        *) echo "Unknown parameter passed: $1"; exit 1 ;;
    esac
    shift
done

# Ensure the destination directory exists or create it
if ! ssh "$DEST_SERVER" "mkdir -p \"$DEST_DIR\""; then
    echo "Failed to ensure the destination directory exists or to create it. Exiting." | tee -a "$LOG_FILE"
    exit 1
fi

loggit() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $*" | tee -a "$LOG_FILE"
}

check_inotifywait_handles() {
    # Count the number of watchable items in SOURCE_DIR
    local item_count=$(find "$SOURCE_DIR" -type f -o -type d | wc -l)
    
    # Read the current inotify watch limit
    local inotify_limit=$(cat /proc/sys/fs/inotify/max_user_watches)
    
    # Compare the item count with the inotify limit
    if [ "$item_count" -gt "$inotify_limit" ]; then
        loggit "Warning: The number of items in $SOURCE_DIR ($item_count) exceeds the inotify watch limit ($inotify_limit)."
        loggit "You may need to increase the limit by writing a higher number to /proc/sys/fs/inotify/max_user_watches." 
    else
        loggit "Inotify watch limit is sufficient for monitoring $SOURCE_DIR." 
    fi
}

# Function to check for inotifywait availability
check_inotifywait() {
    if ! command -v inotifywait &> /dev/null; then
        loggit "inotifywait could not be found." 
        loggit "You can install it by running: sudo yum install inotify-tools" 
        exit 1
    else
        loggit "inotifywait is installed."
        check_inotifywait_handles
    fi
}

# Sync source to destination once or continuously
if [ $CONTINUOUS_SYNC -eq 0 ]; then
    if [ $PARALLEL_PROCESSES -eq 1 ]; then
        rsync "$RSYSNC_FLAGS" --exclude-from="$EXCLUDE_FILE" "$SOURCE_DIR" "$DESTINATION" || {
            loggit "An error occurred during rsync operation."
            exit 1
        }
    else
        # Use xargs to parallelize rsync
        find "$SOURCE_DIR" -mindepth 1 -type d -print0 | xargs -0 -I {} -P $PARALLEL_PROCESSES rsync "$RSYSNC_FLAGS" --exclude-from="$EXCLUDE_FILE" {} "$DESTINATION"
    fi
    loggit "Backup completed successfully."
else
    check_inotifywait
    loggit "Continuously syncing $SOURCE_DIR to $DESTINATION"

    # Using inotifywait -m to monitor changes. The output is piped to a while loop.
    inotifywait -m -r -e close_write,moved_to,create,delete "$SOURCE_DIR" --format '%w%f' |
        while read -r change; do
            # Extracting relative path of changed file or directory
            relative_path=${change#"$SOURCE_DIR"/}
            # Syncing the changed file or directory specifically
            rsync "$RSYSNC_FLAGS" --exclude-from="$EXCLUDE_FILE" "$SOURCE_DIR/$relative_path" "$DESTINATION/$relative_path" || {
                loggit "An error occurred during inotifywait rsync operation."
                exit 1
            }
            loggit "Sync triggered due to changes in: $relative_path"
        done
fi