Multiprocess rsync (多线程的rsync) for data copy

multi_rsync.sh

#!/usr/bin/env bash

# Define source, target, maxdepth and cd to source
source="abspath/to/source/dir"
target="abspath/to/target/dir"
depth=5
cd "${source}"

# Set the maximum number of concurrent rsync threads
maxthreads=64
# How long to wait before checking the number of rsync threads again
sleeptime=5

# Find all folders in the source directory within the maxdepth level
find . -maxdepth ${depth} -type d | while read dir; do
  # Make sure to ignore the parent folder
  if [ $(echo "${dir}" | awk -F'/' '{print NF}') -gt ${depth} ]; then
    # Strip leading dot slash
    subfolder=$(echo "${dir}" | sed 's@^\./@@g')
    if [ ! -d "${target}/${subfolder}" ]; then
      # Create destination folder
      mkdir -p "${target}/${subfolder}"
    fi
    # Make sure the number of rsync threads running is below the threshold
    while [ $(ps -ef | grep -w [r]sync | awk '{print $NF}' | sort -nr | uniq | wc -l) -ge ${maxthreads} ]; do
      echo "Sleeping ${sleeptime} seconds"
      sleep ${sleeptime}
    done
    echo "rsync ${source}/${subfolder}/ to ${target}/${subfolder}/"
    # Run rsync in background for the current subfolder and move one to the next one
    nohup rsync -avPh "${source}/${subfolder}/" "${target}/${subfolder}/" </dev/null >/dev/null 2>&1 &
  fi
done

while [ $(ps -ef | grep -w [r]sync | awk '{print $NF}' | sort -nr | uniq | wc -l) -ge 1 ]; do
  echo "Waiting ..."
  ps -ef | grep -w [r]sync | awk '{print $NF}' | sort -nr | uniq | awk '{print $0}'
  echo "Sleeping 5 seconds"
  echo ""
  sleep 5
done

# Find all files above the maxdepth level and rsync them as well
find . -maxdepth ${depth} -type f -print0 | rsync -avPh --files-from=- --from0 ./ "${target}/"