[Vector Institute] Cron Rsync Script for LOCAL <-> REMOTE syncing

rsync_local2remote_v.sh

#!/bin/bash
# Author: Raeid Saqur
# Email: raeidsaqur@cs.toronto.edu

LOCAL_KEY=id_rsa
REMOTE_USER=raeidsaqur
REMOTE_SERVER="vdm1.vectorinstitute.ai"   # use dedicated data mover server networked to Vaughan

PROJ_PATH="<SOME/PROJ/PATH>"
LOCAL_PROJ_DIR="$HOME/Research/projects/${PROJ_PATH}"
REMOTE_PROJ_DIR="/h/${REMOTE_USER}/Research/projects/${PROJ_PATH}"

cd ${LOCAL_PROJ_DIR};
rsyncdirs=(
  training
  slurm
)

printlog() {
  echo -e "$(date): $1 "
}

push_rsync() {
  # Pushing from LOCAL -> REMOTE
  for i in "${!rsyncdirs[@]}"; do
    s="${LOCAL_PROJ_DIR}/${rsyncdirs[$i]}/"
    t="${REMOTE_PROJ_DIR}/${rsyncdirs[$i]}"
    echo -e "\tSyncing local directory: ${s} to target dir: ${t}"
    rsync -aqzPh -i -e "ssh -i ~/.ssh/${LOCAL_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" \
          --exclude 'raw' --exclude 'checkpoints*' --exclude 'wandb' --exclude 'logs' \
	        ${s} ${REMOTE_USER}@${REMOTE_SERVER}:${t} &
  done
}

pull_rsync() {
  # Pulling from REMOTE -> LOCAL
  for i in "${!rsyncdirs[@]}"; do
    dir="${rsyncdirs[$i]}"
    s="${REMOTE_PROJ_DIR}/$dir/"
    t="${LOCAL_PROJ_DIR}/$dir"
    echo -e "\tSyncing remote directory: ${s} to local dir: ${t}"
    rsync -aqzPh -i -e "ssh -i ~/.ssh/${LOCAL_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" \
	          --exclude 'raw'  --exclude 'checkpoints*' --exclude 'wandb'  --exclude 'logs' \
	          ${REMOTE_USER}@${REMOTE_SERVER}:${s}  ${t} &
  done
}

printlog "=== Initiating |push_rsync| from LOCAL (${hostname}) to REMOTE (${REMOTE_SERVER}) ==="
push_rsync
echo -e "\tPID (push_rsync): $!"
wait $!

printlog "=== Initiating |pull_rsync| from REMOTE (${REMOTE_SERVER}) to LOCAL (${hostname}) ==="
pull_rsync
echo -e "\tPID (pull_rsync): $!"
wait $!

printlog "Rsync completed"

## crontab ##
#0 4 * * * $SCRIPT_PATH/rsync_local2remote_v.sh >> $SCRIPT_PATH/rsync.log 2>&1