Delivering folders/files (e.g. logs/checkpoints) from one machine to another

deliv.sh

#!/bin/bash

# Summary: sending folders/files (e.g. logs/checkpoints) from one machine to another
# 
# The preliminary step is setting up SSH key-based authentication between the source and target machines:
# 1) On the source machine, generate a new SSH key pair. 
#   $ ssh-keygen
#   This will create two files: id_rsa (the private key) and id_rsa.pub (the public key).
# 2) Copy the public key to the target machine. 
#   $ ssh-copy-id user@target-machine
#   This will copy the public key to the target machine and add it to the authorized_keys file in your home directory on the target machine.
# 3) Test SSH key-based authentication
#   $ ssh user@target-machine
#   If everything is set up correctly, you should be able to log in to the target machine without entering a password.

# TO-DO: Scheduled sending

# configs you may want to change
source_addr="user@target-machine8"
timestamp=$(date +"%Y-%m-%d_%H-%M-%S")
receiver_path="~/from_source_${timestamp}"
# list your exp folders
exps=(
    "exp_folders1" \
    "exp_folders2" \
)
# suppose exp folders have the following structure:
# exp_folder_example
# - checkpoints
#   - latest.pt
#   - save-100.pt
#   - save-200.pt
#   ...
# - config.yml
# - tb          # tensorboard log folder
#   - ...
# - samples     # generated images
#   - ...

# list files you want to send under the exp folders 
files_to_send=(
    "checkpoints/latest.pt" \
    "samples" \
    "tb" \
    "config.yml" \
)
# (optional) whether send the latest numbered ckp
send_latest_num_ckpt=true

ssh -t ${source_addr} "mkdir -p ${receiver_path}"
for exp in "${exps[@]}"; do
    exp_files=( "${files_to_send[@]}" )  # return a copy of an array 
    if [ $send_latest_num_ckpt == true]; then
        last_save_ckp=$(ls -1 "${exp}/checkpoints"| grep "save-" | sort -n -t '-' -k 2 | tail -n 1)
        exp_files+=("checkpoints/${last_save_ckp}")
    fi
    for item in "${files_to_send[@]}"; do
        echo "$item"
    done
    last_two=$(echo "$exp" |rev | cut -d'/' -f 1-2 |rev)
    ssh -t ${source_addr} "mkdir -p ${receiver_path}/${last_two}/checkpoints"
    for file in "${exp_files[@]}"; do
        scp -r ${exp}/${file} ${source_addr}:${receiver_path}/${last_two}/${file}
    done
done