Script for archiving data from G-Drive to Wasabi

17a4.sh

#!/bin/bash

SYS_ROOT=/mnt/c/Proof
WORK_ROOT=$SYS_ROOT/17a4-logs

# log files
CTIME=$(date "+%Y%m%d-%H%M%S")
LOGFILE=$WORK_ROOT/${CTIME}.log.txt

exec >> $LOGFILE 2>&1

echo "Run started at $(date)"

# remote configs
#BUCKET_NAME=prerak-test
BUCKET_NAME=sec-17a-4
RCLONE_CONFIG=$SYS_ROOT/rclone.conf
SOURCE_DIR="Proof-GDrive:Proof/Regulatory/Corporate Books and Records"
TARGET_DIR="Proof-Wasabi:$BUCKET_NAME/Corporate Books and Records/$CTIME"

echo "Starting sync from [$SOURCE_DIR] to [$TARGET_DIR]..."
rclone -v --config $RCLONE_CONFIG copy --create-empty-src-dirs "$SOURCE_DIR" "$TARGET_DIR" 2>&1
RCLONE_EXIT_CODE=$?
echo "Sync completed with exit code: [$RCLONE_EXIT_CODE]"

echo "Checking number of files..."
FILE_COUNT=$(rclone --config $RCLONE_CONFIG ls "$TARGET_DIR" | wc -l)
echo "Files found: $FILE_COUNT"

RESULT=Failed
if [ $RCLONE_EXIT_CODE -eq 0 ]; then 
	RESULT=Succeeded
fi
echo "RESULT = ${RESULT}"

EMAIL_RECEPIENTS="prerak@prooftrading.com,dan@prooftrading.com"
#EMAIL_RECEPIENTS="prerak@prooftrading.com"
echo "Sending email to ${EMAIL_RECEPIENTS}..."

/usr/sbin/sendmail $EMAIL_RECEPIENTS << EOF
To:$EMAIL_RECEPIENTS
Subject: $(date "+%Y%m%d") - 17a4 Copy ${RESULT} to $BUCKET_NAME (File Count: $FILE_COUNT)
Content-Type: text/html

<pre>
$(cat $LOGFILE)
</pre>
EOF

echo "Email sent. Done."

rclone.conf

[Proof-GDrive]
type = drive
client_id = 287...0ba.apps.googleusercontent.com
client_secret = xbE...C75
scope = drive.readonly
token = {"access_token":"ya2...o5g","token_type":"Bearer","refresh_token":"1//01Y...b1s","expiry":"2020-09-01T07:52:35.7837901-04:00"}
root_folder_id = 0AC...PVA
shared_with_me = true

[Proof-Wasabi]
type = s3
access_key_id = 5XG...0NH
endpoint = s3.wasabisys.com
provider = Wasabi
region = us-east-1
secret_access_key = Q9e...rpD