Multi-thread Rsync. Don't forget to $ chmod +x multithread_rsync.sh and change the paths & server. And then run it like: $ ./multithread_rsync.sh

multithread_rsync.sh

#!/usr/bin/env bash

# RSYNC /usr/local/bin/rsync
RSYNC_PROG=$(which rsync)

# Note the important use of --relative to use relative paths so we don't have to specify the exact path on dest
RSYNC_OPTS="-aP --numeric-ids --progress --human-readable --exclude=.git --relative --compress"

export RSYNC_RSH="ssh -T -c aes128-ctr -o Compression=no -x"

SRCDIR=/local/files/to/transfer
DESTDIR=/path/to/directory/

# Sets the number of threads to match the number of logic cores.
THREADS=$(sysctl -n hw.ncpu)
SERVER="user@production.lib.utk.edu"

cd $SRCDIR
pwd

# note the combination of -print0 and -0!
time find . -mindepth 1 -maxdepth 1 -print0 | \
    xargs -0 -n1 -P$THREADS -I% \
        $RSYNC_PROG $RSYNC_OPTS "%" $SERVER:$DESTDIR