Monitor a Directory and SFTP Files to Multiple Destinations

config.cfg

## CONFIGURATION
export NAME=$(basename "$0" .sh)
export LOCKFILE_DIR="Lockfile"
export PENDING_DIR="Inbound"
export FAILED_DIR="Failed"
export PROCESSED_DIR="Processed"
export LOG_FILE="$NAME.log"
export NOTIFY_ON_FAIL=true
export NOTIFY_EMAIL="mail@karlbaillie.com"
export DEBUG=true

declare -A DETAILS;                                              declare -a TARGETS
DETAILS[1]="some.janky.server,user-test-1,4q8n2nQ8r03C,upload";  TARGETS+=(1)
DETAILS[2]="some.janky.server,user-test-2,u5051HZfA4kS,upload";  TARGETS+=(2)
DETAILS[3]="some.janky.server,user-test-3,5437B089Hmfg,upload";  TARGETS+=(3)
DETAILS[4]="some.janky.server,user-test-2,u5051HZfA4kS,upload";  TARGETS+=(4)

sftp-shovel.sh

#!/bin/bash
#
# Karl Baillie (August 2016)
# - This is a temporary fix for the AMQP lbraries ability to deal with SHA1/2
#
# Karl Baillie (November 2017)
# - Refactored to shift files to multiple SFTP targets and sorted out logging
# as well as better checks, balances and alerting.
#
# Usage: sftp-shovel.sh [Path To Config File]
# Example: ./sftp-shovel.sh ./config.cfg
#
#

if [ $# -eq 0 ]
  then
    sed -n '3,13p' "$0" | tr -d '#'
    exit 1
fi

## CONFIGURATION
source $1
##

## SETUP
# Logging
function echo_log {
  export DATE=$(date "+%y-%m-%d_%H:%M:%S")
  # $1=DEBUG $2=MESSAGE
  if [ "$1" != "DEBUG" ]; then
    echo "$DATE $LEVEL $1 $2" >> $LOG_FILE
  elif [ "$DEBUG" = true -a "$1" == "DEBUG" ]; then
    echo "$DATE $LEVEL $1 $2" >> $LOG_FILE
  fi
}

function send_mail {
  if [ "$NOTIFY_ON_FAIL" = true ]; then
    #$SFTP_USERNAME $SFTP_HOST $SFTP_REMOTE_DIR $f
  mail -s "SFTP Transfer Failed to $2" $NOTIFY_EMAIL <<EOF
CRITICAL ERROR:
Failed to transfer \"$4\" to \"$1@$2:$3\"
EOF
  fi
}

# https://unix.stackexchange.com/questions/204803/why-is-nullglob-not-default
# makes the line "for f in $PENDING_DIR/*" behave how I thought it would
shopt -s nullglob

# Lockfile
if [ -f "$LOCKFILE_DIR/$NAME.lock" ]; then
  echo_log "ERROR" "ABORTED: An SFTP Transfer Script is Already Running"
  exit 1
else
  echo_log "DEBUG" "STARTING: Creating a Lock File"
  touch $LOCKFILE_DIR/$NAME.lock
  if [[ $? != 0 ]]; then
    echo_log "ERROR" "ABORTED: Failed to Create a Lock File"
    exit 1
  fi
  echo_log "DEBUG" "STARTED: Created Lock File"
fi
##

## DO THE THING
for f in $PENDING_DIR/*; do
  for i in "${!TARGETS[@]}"; do
    # get this targets details
    NO_OF_TARGETS=$(( ${#DETAILS[@]} -1 ))
    SFTP_HOST=$(echo "${DETAILS[${TARGETS[$i]}]}" | cut -d, -f1)
    SFTP_USERNAME=$(echo "${DETAILS[${TARGETS[$i]}]}" | cut -d, -f2)
    SFTP_PASSWORD=$(echo "${DETAILS[${TARGETS[$i]}]}" | cut -d, -f3)
    SFTP_REMOTE_DIR=$(echo "${DETAILS[${TARGETS[$i]}]}" | cut -d, -f4)

    echo_log "DEBUG" "Attempting transfer of \"$f\" to \"$SFTP_USERNAME@$SFTP_HOST\""
    sshpass -p $SFTP_PASSWORD sftp -oBatchMode=no -o StrictHostKeyChecking=no -b - $SFTP_USERNAME@$SFTP_HOST <<EOF
cd $SFTP_REMOTE_DIR
put $f
bye
EOF
  if [[ $? != 0 ]]; then
    echo_log "ERROR" "Failed transfer of \"$f\" to \"$SFTP_USERNAME@$SFTP_HOST\""
    send_mail $SFTP_USERNAME $SFTP_HOST $SFTP_REMOTE_DIR $f
    echo_log "INFO" "Copying failed file \"$f\" to \"$FAILED_DIR\""
    cp $f $FAILED_DIR/
  else
    echo_log "INFO" "Finished transfer of \"$f\" to \"$SFTP_USERNAME@$SFTP_HOST\""
    if [ $i -eq $NO_OF_TARGETS ]; then
      echo_log "DEBUG" "All targets attempted, removing \"$f\" from \"$PENDING_DIR\""
      mv $f $PROCESSED_DIR
    fi
  fi
  done
done
##

## Exit (dis)gracefully
echo_log "DEBUG" "STOPPING: Removing Lock File"
rm $LOCKFILE_DIR/$NAME.lock
if [[ $? != 0 ]]; then
  echo_log "ERROR" "STOPPED: Failed to Remove Lock File"
else
  echo_log "DEBUG" "STOPPED: Removed Lock File"
fi

exit 0
##

# EOF