mover.sh (find + mmin version)

mover.sh

#!/bin/bash

set -eu

#
# This program watches new files and moves them from one machine to another
#
# How it works
#
# mover.sh will recursively watch old files in a directory using find. When the
# file has been modified longer ago than the "age" (-a) parameter, it is
# rsynced to the destination directory and removed from the local filesystem.
#
# Since files can be created in a new deep directory hierarchy, we make use of
# rsync's -R flag and 'dotted' path (hence the requirement for rsync 2.6.7).
#
# For instance, if you run:
#
#   mover.sh -s /tmp/a/ -d /tmp/b/ -r user@host
#
# and create a file 'foo' in /tmp/a/x/y/z/, hierarchy '/tmp/b/x/y/z/' might not
# exist on the remote system.
#
# This script takes care of this using the aformentioned options, and will
# execute:
#
# rsync -Ravz --remove-source-files /tmp/a/./x/y/z/foo /tmp/b/./x/y/z/foo
#
# which will take care of 'mkdir -p /tmp/b/x/y/z' before syncing foo.
#

usage() {
  echo -e "Usage: $0 -s SRC -d DST [-r REMOTE] [-v] [-n] [-i INTERVAL] [-a AGE]\n"
  echo -e "\t-s SRC: absolute path of source to check (mandatory)"
  echo -e "\t-d DST: absolute path of destination to copy to (mandatory; on the remote if -r specified)"
  echo -e "\t-r REMOTE: remote machine to copy to (in ssh form, e.g. user@host.machine)"
  echo -e "\t-i INTERVAL: check interval (in seconds, default 60)"
  echo -e "\t-a AGE: modification age threshold that trigers move (in minutes, default 1)"
  echo -e "\t-n: dry-run mode (does nothing)"
  echo -e "\t-v: verbose mode"
}

log() {
   echo "date=$(date --iso-8601=seconds) ${@}"
}

log_debug() {
   local MESSAGE="${@}"
   if [[ "${VERBOSE}" == true ]]; then
      log "level=debug ${MESSAGE}"
   fi
}

log_info() {
   log "level=info ${@}"
}

log_warn() {
   log "level=info ${@}"
}

log_err() {
   log "level=error ${@}"
}

move() {
  local src=$1
  local dst=$2
  local remote=$3

  # if we have a remote, prefix dst with it
  if [ -n "$remote" ]; then
    dst="$remote:$dst"
  fi

  # -R makes use of relative paths

  rsopts="-Ravz"

  if [[ "${DRYRUN}" == true ]]; then
    rsopts="${rsopts}n"
  fi

  log_debug "cmd=\"rsync ${rsopts} --itemize-changes --remove-source-files ${src} ${dst}\""

  while read item; do
    # Parse output of itemized changes
    # http://www.staroceans.org/e-book/understanding-the-output-of-rsync-itemize-changes.html
    res="$(echo $item | cut -f1 -d' ')"
    file="$(echo $item | cut -f2 -d' ')"
    log_debug "file=\"${file}\" result=\"${res}\""
  done < <(rsync $rsopts --itemize-changes --remove-source-files ${src} ${dst} | grep "^[><ch].*")
}

monitor() {
  local src=$1
  local dst=$2
  local old_time=$3
  local remote=$4

  while read file; do
    dir="$(dirname $file)/"
    reldir="${dir#${src}}"
    file="${file#${dir}}"
    log_debug "file=\"${file}\" src=\"${src}\" dir=\"${dir}\" reldir=\"${reldir}\""
    log_debug "action=move arg1=\"${src}./${reldir}${file}\" arg2=\"${dst}\" arg3=\"${remote}\""
    move "${src}./${reldir}${file}" "${dst}" "${remote}"
  done < <(find $src -type f -mmin +1)
}

#### Main ####

SOURCE=
DESTINATION=
REMOTE=
VERBOSE=false
DRYRUN=false
INTERVAL=60
AGE=1

optstring=":hvns:d:r:i:a:"

while getopts ${optstring} arg; do
  case ${arg} in
    h)
      usage
      exit 0
      ;;
    v)
      VERBOSE='true'
      ;;
    n)
      DRYRUN='true'
      ;;
    s)
      SOURCE="${OPTARG}"
      ;;
    d)
      DESTINATION="${OPTARG}"
      ;;
    r)
      REMOTE="${OPTARG}"
      ;;
    i)
      INTERVAL="${OPTARG}"
      ;;
    o)
      AGE="${OPTARG}"
      ;;
    ?)
      log_err "error=invalid_option msg=\"option -${OPTARG} is invalid\""
      echo
      usage
      ;;
  esac
done


if [ ! -r "${SOURCE}" ]; then
  log_err "error=invalid_source msg=\"source path ${SOURCE} does not exist\""
  exit 1
fi

if [[ "${SOURCE}" != /* ]]; then
  log_err "error=relative_source msg=\"source path ${SOURCE} must be absolute\""
  exit 1
fi

if [ ! -r "${DESTINATION}" ]; then
  log_err "error=invalid_destination msg=\"destination path ${DESTINATION} does not exist\""
  exit 1
fi

if [[ "${DESTINATION}" != /* ]]; then
  log_err "error=relative_destination msg=\"destination path ${DESTINATION} must be absolute\""
  exit 1
fi

log_info "source=\"${SOURCE}\" destination=\"${DESTINATION}\" remote=\"${REMOTE}\" interval=${INTERVAL} age=${AGE} verbose=${VERBOSE} dryrun=${DRYRUN} msg=\"starting mover\""

if [ ! -r "$REMOTE" ]; then
  log_warn "msg=\"none remote specified; will work locally\""
fi

if [[ "${DRYRUN}" == true ]];then
  log_warn "msg=\"DRY RUN mode; not files will be moved !\""
fi

while true; do
    monitor "$SOURCE" "$DESTINATION" "$AGE" "$REMOTE"
    sleep $INTERVAL
done