Rclone copy/sync to scaleway remote

scaleway.sh

#!/bin/bash

# This script will execute a very fast copy command every 20mins and a slower sync (which will delete remote items) script every 2hrs
# All of the files live in your user directory in a 'cron' folder
# Logs are saved in verbose mode -v
# Deleted items are copied to another remote location. I suggest a different bucket upon which you can set a lifecyle, i.e. delete after 90 days.

## Mac Instructions
# open terminal.app
# cd ~/cron
# nano scaleway.sh
# paste this script
# scroll down to command /usr/bin/rclone copy/sync and add --dry-run to both commands (while testing)
# save with ctrl + x, y to confirm
# chmod +x scaleway.sh
# crontab -e
# press 'i' to enter editing mode, paste the following 2 lines without the '#'
# */20 * * * * ~/cron/scaleway.sh COPY
# 30 */2 * * * ~/cron/scaleway.sh SYNC
# press ctrl + c, press ':', type 'wq', hit enter
# nano filter-list.txt
# paste the following (without the # symbols) - are excluded Files/Folders, + is included files/folders
# - /Library/**
# - /Applications/**
# - /Pictures/**
# - /google-cloud-sdk/**
# - **/node_modules/**
# - .*/**
# + .git
# + .firebaserc
# + .browserslistrc
# + .eslintrc.js
# + .gitignore
# - .*
# - rclone-upload.log
# - scaleway-lock.lock
# - Thumbs.db
# type ctrl + x and press y to save

SECONDS=0
NOW=`date "+%F %T"` # Current date and time
LOG="/Users/$USER/cron/rclone-upload.log" # Log file location
SRC="/Users/$USER" # Local source location
DEST="scaleway:bucket1" # Remote destination in the format of service:bucket
BACKUP="scaleway:bucket1-deleted" # Remote destination for deleted items in the format of service:bucket
FILTER="/Users/$USER/cron/filter-list.txt" # Filter list for included and excluded files
SUFFIX="-`date '+%F'`" # Date suffix applied for deleted files

echo "$NOW $1 Started" >> $LOG

# Check is Lock File exists, if not create it and set trap on exit
 if { set -C; 2>/dev/null >/Users/$USER/cron/scaleway-lock.lock; }; then
    trap "rm -f /Users/$USER/cron/scaleway-lock.lock" EXIT
 else
    echo "Lock file exists… exiting" >> $LOG
    exit
 fi

# Run copy/sync based on command line argument passed
#  ./test.sh hello world  $0=./test.sh  $1=hello $2=world
# while testing or first setting up you can add --dry-run just after sync or copy below to simulate what will happen
if [[ $1 == 'SYNC' ]]; then
  /usr/local/bin/rclone sync -v --copy-links --filter-from $FILTER --checkers 50 --transfers 50 --checksum --log-file=$LOG --stats 60s --backup-dir $BACKUP --suffix $SUFFIX --suffix-keep-extension $SRC $DEST
elif [[ $1 == 'COPY' ]]; then
  /usr/local/bin/rclone copy -v --copy-links --filter-from $FILTER --checkers 50 --transfers 50 --checksum --log-file=$LOG --max-age 1d --no-traverse $SRC $DEST
else
  echo "No command received" >> $LOG
fi 

DURATION=$SECONDS
echo "$NOW $1 Elapsed time: $(($DURATION / 60))m $(($DURATION % 60))s" >> $LOG

# filter-list.txt
# - /Library/**
# - /Applications/**
# - /Pictures/**
# - /google-cloud-sdk/**
# - **/node_modules/**
# - .*/**
# + .git
# + .firebaserc
# + .browserslistrc
# + .eslintrc.js
# + .gitignore
# - .*
# - rclone-upload.log
# - scaleway-lock.lock
# - Thumbs.db

# crontab -e
# */20 * * * * ~/cron/scaleway.sh COPY
# 30 */2 * * * ~/cron/scaleway.sh SYNC