Copies images and movies from a src directory to a destination directory

copyToGallery.sh

#! /bin/bash

################################################################################
# Name:                                                                        #
#   copyToGallery.sh                                                           #
#                                                                              #
# Description:                                                                 #
#   Copies images and movies from a src directory to a destination directory.  #
#   This is most useful for keeping raw files in one folder, while providing   #
#     jpg files in a (piwigo) web gallery                                      #
#   Raw images (from Nikon, Panasonic and Olympus cameras) are converted to    #
#     jpg and movies are converted to mp4.                                     #
################################################################################

######################
# Options            #
######################

CP_EXEC="$(type -p cp)"
CP_OPTS="-p"
DIR_EXCLUDES=("BestOf" "CaptureOne")
EXIFTOOL_EXEC="$(type -p exiftool)"
EXIFTOOL_OPTS="-@ ${FLAGS_source}/argsExiftool.txt"
FFMPEG_EXEC="$(type -p ffmpeg)"
FFMPEG_OPTS_AVI="-copyts -acodec aac -strict -2 -ar 48000 -ac 2 -vcodec libx264 -preset ultrafast -r 25 -flags +aic+mv4 -s 720x480"
FFMPEG_OPTS_MTS="$FFMPEG_OPTS_AVI"
FFMPEG_OPTS_MOV="-copyts -acodec aac -strict -2 -ar 48000 -ac 2 -vcodec copy -flags +aic+mv4 -s 720x480"
LN_EXEC="$(type -p ln)"
LN_OPTS="-s"
# Configuration of raw file conversion (ufraw-batch and darktable-cli are supported)
#RAW_EXEC_NAME="ufraw-batch"
#RAW_EXEC="$(type -p ${RAW_EXEC_NAME})"
#RAW_OPTS="--silent --wb=camera --exposure=auto --compression=100 --lensfun=auto --out-type=jpeg"
RAW_EXEC_NAME="darktable-cli"
RAW_EXEC="$(type -p ${RAW_EXEC_NAME})"
RAW_OPTS=""
REGEXP_IMAGE="\.\(jpg\|jpeg\|png\|gif\)$"
REGEXP_RAW="\.\(nef\|orf\|rw2\)$"
REGEXP_MOVIE="\.\(avi\|mov\|mts\)$"
SHFLAGS_EXEC="/usr/share/shflags/shflags"
TARGET_FORMAT_IMAGE="jpg"
TARGET_FORMAT_MOVIE="mp4"

######################
# Check options      #
######################

function check_exec() {
  local NAME="$1"
  local EXEC="$2"

  if [ -z "$(type -p ${EXEC})" ]; then
    echo "Could not find executable for ${NAME}. Please install it!"
    exit 1;
  fi
}

check_exec "sh_flags" "${SHFLAGS_EXEC}"
check_exec "cp" "${CP_EXEC}"
check_exec "ln" "${LN_EXEC}"
check_exec "exiftool" "${EXIFTOOL_EXEC}"
check_exec "ffmpeg" "${FFMPEG_EXEC}"
check_exec "${RAW_EXEC_NAME}" "${RAW_EXEC}"

######################
# Command-Line Flags #
######################

. ${SHFLAGS_EXEC}

DEFINE_boolean 'useChmod' false 'settings chmod 777 for all processed files'
DEFINE_boolean 'useImage' true 'link/copy ordinary image files' 'i'
DEFINE_boolean 'useLinks' false 'link not converted files instead of copying' 'l'
DEFINE_boolean 'useMovie' true 'decode movie files' 'm'
DEFINE_boolean 'useRaw' true 'decode raw files' 'r'
DEFINE_boolean 'useRedecodeXmp' false 'use exiftool to re-decode xmp files'
DEFINE_boolean 'useRedecodeCopy' false 'create a temporary tmp file of raw and xmp file while redecoding xmp information'
DEFINE_boolean 'useTimeRestriction' true 'only processing files that have been changed since the last run' 't'
DEFINE_boolean 'overwrite' false 'overwrite existing files' 'o'
DEFINE_boolean 'verbose' false 'print verbose output' 'v'
DEFINE_boolean 'debug' false 'print debug output' 'd'
DEFINE_string 'source' '`pwd`' 'source directory from which (including subfolders) files are used' 's'

FLAGS_HELP=$(printf "Copies images and movie files from a source to a destination directory.\nWhile copying, raw images are decoded to jpg and movie files are converted to h264 encoded m4v files.\nOrdinary images are linked to the destination folder.\nThe source directory structure is also generated in the destination folder.\n\nUSAGE: $0 [flags] destination")
FLAGS "$@" || exit $?
eval set -- "${FLAGS_ARGV}"

SCRIPT_FILE="$(basename $0)"
SCRIPT_NAME="${SCRIPT_FILE%.*}"
LAST_RUN_FILE="${FLAGS_source}/${SCRIPT_NAME}".lastRun
PID_FILE="${FLAGS_source}/${SCRIPT_NAME}".pid

# check for destination directory
if [ $# -eq 0 ]; then
  echo 'error: destination missing' >&2
  flags_help
  exit 1
fi
DST="$1"

######################
# Functions          #
######################

function echo_debug() {
  if [ ${FLAGS_debug} -eq 0 ]; then
    echo "$@"
  fi
}

function echo_verbose() {
  if [ ${FLAGS_debug} -eq 0 -o ${FLAGS_verbose} -eq 0 ]; then
    echo "$@"
  fi
}

function file_not_exists() {
  if [ $FLAGS_overwrite -eq 0 ]; then
    return 0
  fi

  if [ -e "$1" ]; then
    return 1
  else
    return 0
  fi
}

function replace_special_chars() {
  STR="$1"
  STR=`echo $STR | sed -e "s/ /_/g"`
  STR=`echo $STR | sed -e "s/\&/_/g"`
  STR=`echo $STR | sed -e "s/[ÄÖÜäöüß]/_/g"`
  STR=`echo $STR | sed -e "s/(/_/g"`
  STR=`echo $STR | sed -e "s/)//g"`
  STR=`echo $STR | sed -e "s/__*/_/g"`

  echo "$STR"
}

function extension_lowercase() {
  LOWER=`echo $1 | sed -r "s/([^.]*)\$/\L\1/"`
  echo $LOWER
}

function link_image() {
  CUR_DIR=`pwd`
  IMG_LEN=${#IMG_FILES[@]}

  if [ $IMG_LEN -gt 0 ]; then
    echo_verbose "  linking $IMG_LEN image file(s)"
    for ((j = 0; j < $IMG_LEN; ++j)); do
      OUTPUT_FILE=$(replace_special_chars "${IMG_FILES[$j]}")
      OUTPUT_FILE=$(extension_lowercase "$OUTPUT_FILE")
      if file_not_exists "$TARGET_DIR/$OUTPUT_FILE"; then
        echo_debug "    creating \"$OUTPUT_FILE\""
        $LN_EXEC $LN_OPTS "$CUR_DIR/${IMG_FILES[$j]}" "$TARGET_DIR/$OUTPUT_FILE" 2>/dev/null
      else
        echo_debug "    not creating \"$OUTPUT_FILE\" (already exists)"
      fi
    done
  fi
}

function copy_image() {
  IMG_LEN=${#IMG_FILES[@]}

  if [ $IMG_LEN -gt 0 ]; then
    echo_verbose "  copying $IMG_LEN image file(s)"
    for ((j = 0; j < $IMG_LEN; ++j)); do
      OUTPUT_FILE=$(replace_special_chars "${IMG_FILES[$j]}")
      OUTPUT_FILE=$(extension_lowercase "$OUTPUT_FILE")
      if file_not_exists "$TARGET_DIR/$OUTPUT_FILE"; then
        echo_debug "    creating \"$OUTPUT_FILE\""
        $CP_EXEC $CP_OPTS "${IMG_FILES[$j]}" "$TARGET_DIR/$OUTPUT_FILE" 2>/dev/null
      else
	echo_debug "    not creating \"$OUTPUT_FILE\" (already exists)"
      fi
    done
  fi
}

function copy_raw() {
  RAW_LEN=${#RAW_FILES[@]}

  if [ $RAW_LEN -gt 0 ]; then
    echo_verbose "  converting $RAW_LEN raw file(s)"
    for ((j = 0; j < $RAW_LEN; ++j)); do
      RAW_FILE=${RAW_FILES[$j]}
      OUTPUT_FILE=$(replace_special_chars "${RAW_FILES[$j]}")
      OUTPUT_FILE=`echo $OUTPUT_FILE | sed -e "s/$REGEXP_RAW/\.$TARGET_FORMAT_IMAGE/i"`
      XMP_FILE=`echo ${RAW_FILES[$j]} | sed -e "s/$REGEXP_RAW/\.xmp/i"`
      XMP_FILE_TMP=`echo ${RAW_FILES[$j]} | sed -e "s/$REGEXP_RAW/\.tmp\.xmp/i"`

      # handle redecode tmp files
      if [ $FLAGS_useRedecodeCopy -eq 0 ]; then
        RAW_FILE=`echo "$RAW_FILE" | sed 's/^.*\.//i'`
        RAW_FILE="temporaryFile.$RAW_FILE"
        XMP_FILE_TMP="temporaryFile.xmp"

        cp "${RAW_FILES[$j]}" "$RAW_FILE"
      fi

      # redecode of xmp data
      if [ $FLAGS_useRedecodeXmp -eq 0 ] && [ -f "$XMP_FILE" ]; then
        $EXIFTOOL_EXEC -tagsFromFile "$XMP_FILE" $EXIFTOOL_OPTS "$XMP_FILE_TMP" >/dev/null
        XMP_FILE="$XMP_FILE_TMP"
      fi

      # extract image from raw file (using darktable or ufraw-batch)
      if file_not_exists "$TARGET_DIR/$OUTPUT_FILE"; then
        echo_debug "    creating \"$OUTPUT_FILE\""
        if [ $USE_UFRAW -eq 0 ]; then
          $RAW_EXEC $RAW_OPTS --output="$TARGET_DIR/$OUTPUT_FILE" "$RAW_FILE" 2>/dev/null
        elif [ $USE_DARKTABLE -eq 0 ]; then
          $RAW_EXEC "$RAW_FILE" "$XMP_FILE" "$TARGET_DIR/$OUTPUT_FILE" 2>/dev/null
        fi
      else
	echo_debug "    not creating \"$OUTPUT_FILE\" (already exists)"
      fi

      # remove redecoded xmp file
      if [ $FLAGS_useRedecodeXmp -eq 0 ]; then
        rm -f "$XMP_FILE_TMP" 
      fi
      if [ $FLAGS_useRedecodeCopy -eq 0 ]; then
        rm -f "$RAW_FILE"
      fi
    done
  fi
}

function copy_movie() {
  MOVIE_LEN=${#MOVIE_FILES[@]}

  if [ $MOVIE_LEN -gt 0 ]; then
    echo_verbose "  creating directory pwg_representative"
    mkdir -p "$TARGET_DIR/pwg_representative"
    # piwigo requires the pwg_representative to be always of mode 777
    #if [ $FLAGS_useChmod -eq 0 ]; then
      chmod 777 "$TARGET_DIR/pwg_representative"
    #fi

    echo_verbose "  converting "$MOVIE_LEN" movie file(s)"
    for ((j = 0; j < $MOVIE_LEN; ++j)); do
      OUTPUT_FILE=$(replace_special_chars "${MOVIE_FILES[$j]}")
      OUTPUT_FILE=`echo $OUTPUT_FILE | sed -e "s/$REGEXP_MOVIE/\.$TARGET_FORMAT_MOVIE/i"`
      if file_not_exists "$TARGET_DIR/$OUTPUT_FILE"; then
        echo_debug "    creating \"$OUTPUT_FILE\""
	EXTENSION="${OUTPUT_FILE##*.}"
	OPTION="FFMPEG_OPTS_${EXTENSION}"

        $FFMPEG_EXEC -i "${MOVIE_FILES[$j]}" ${!OPTION} "$TARGET_DIR/$OUTPUT_FILE" 2>/dev/null
      else
        echo_debug "    not creating \"$OUTPUT_FILE\" (already exists)"
      fi
    done
  fi
}


######################
# Main Method        #
######################

function main() {
  USE_UFRAW=-1
  USE_DARKTABLE=-1

  if [ ! -d "$FLAGS_source" ]; then
    echo "Directory \"$FLAGS_source\" not found" >&2
    exit 1
  fi

  if [ -f "$PID_FILE" ]; then
	PID=$(cat "$PID_FILE")
	if ps -p $PID > /dev/null
	then
	  echo "Program already running (with pid $PID)"
	  exit 1
	else
	  echo "PID found, but process is not running... restarting"
	fi
  fi

  cd "$FLAGS_source"
  echo $$ > "$PID_FILE"
  echo "Starting $SCRIPT_NAME: "
  echo_verbose "  - printing verbose output"
  echo_debug "  - printing debug output"
  if [ $FLAGS_useImage -eq 0 ]; then
    if [ $FLAGS_useLinks -eq 0 ]; then
      echo "  - linking image files"
    else
      echo "  - copying image files"
    fi
  fi
  if [ $FLAGS_useMovie -eq 0 ]; then
    echo "  - converting movie files"
  fi
  if [ $FLAGS_useRaw -eq 0 ]; then
    if grep -q ufraw-batch "$RAW_EXEC"; then
      echo "  - converting raw files (using ufraw-batch)"
      FLAGS_useRedecodeXmp=-1
      FLAGS_useRedecodeCopy=-1
      USE_UFRAW=0
    elif grep -q darktable-cli "$RAW_EXEC"; then
      echo "  - converting raw files (using darktable-cli)"
      USE_DARKTABLE=0
    else
      echo "Invalid RAW conversion configuration... not converting raw files"
      FLAGS_useRaw=-1
    fi
  fi
  if [ $FLAGS_useRedecodeXmp -eq 0 ]; then
    echo "  - using xmp redecode"
  fi
  if [ $FLAGS_useRedecodeCopy -eq 0 ]; then
    echo "  - using temporary raw/xmp copy"
  fi

  FIND_OPTIONS_DIR=(-type d)
  for DIR in ${DIR_EXCLUDES[@]}; do
    FIND_OPTIONS_DIR+=(-name "${DIR}" -prune -o -type d)
  done

  FIND_OPTIONS_FILE=(-maxdepth 0 -type f)
  if [ $FLAGS_useTimeRestriction -eq 0 ]; then
    echo_debug "Searching for last run file: $LAST_RUN_FILE"
    if [ -f $LAST_RUN_FILE ]; then
      LAST_RUN_TIME=$(cat $LAST_RUN_FILE)
      echo "  - using time restriction filter (last run: $LAST_RUN_TIME)"
      FIND_OPTIONS_DIR+=(-newermt "$LAST_RUN_TIME")
      FIND_OPTIONS_FILE+=(-newermt "$LAST_RUN_TIME")
    else
      echo "  - first run of $SCRIPT_NAME: not applyling time restriction filter"
    fi
  fi

  if [ $FLAGS_overwrite -eq 0 ]; then
    echo "  - using overwrite mode"

    CP_OPTS="$CP_OPTS -f"
    FFMPEG_OPTS="$FFMPEG_OPTS -y"
    LN_OPTS="$LN_OPTS -f"
    if [ $USE_UFRAW -eq 0]; then
      RAW_OPTS="$RAW_OPTS --overwrite"
    fi
  else
    CP_OPTS="$CP_OPTS -u"
    FFMPEG_OPTS="$FFMPEG_OPTS -n"
  fi

  TMPIFS=$IFS
  IFS=$'\n'
  DIRS=($(echo "." && find * ${FIND_OPTIONS_DIR[@]} -print | sort -n -r))
  IFS=$TMPIFS
  LEN=${#DIRS[@]}
  if [ $LEN -le 1 ]; then
    echo "Processing ${LEN} directory"
  else
    echo "Processing ${LEN} directories"
  fi

  NR_PROCESSED=0
  for ((i = 0; i < $LEN; ++i)); do
    CDIR=$(replace_special_chars "${DIRS[$i]}")
    SOURCE_DIR="$FLAGS_source/${DIRS[$i]}"
    TARGET_DIR="$DST/$CDIR"

    echo_verbose "Processing \"$SOURCE_DIR\""
    if [ -f "${SOURCE_DIR}/.doNotSync" ]; then
      echo_debug "  skipping directory -> it contains a '.doNotSync' file"
      continue
    fi
    if [ ! -d "$TARGET_DIR" ]; then
      echo_debug "  creating directory"
      mkdir -p "$TARGET_DIR"
    fi

    cd "$SOURCE_DIR"
    TMPIFS=$IFS
    IFS=$'\n'
    if [ $FLAGS_useImage -eq 0 ]; then
      IMG_FILES=($(find * ${FIND_OPTIONS_FILE[@]} -iregex ".*${REGEXP_IMAGE}" -print | sort -n))
      NR_PROCESSED=$(($NR_PROCESSED + ${#IMG_FILES[@]}))
    fi
    if [ $FLAGS_useRaw -eq 0 ]; then
      RAW_FILES=($(find * ${FIND_OPTIONS_FILE[@]} -iregex ".*${REGEXP_RAW}" -print | sort -n))
      NR_PROCESSED=$(($NR_PROCESSED + ${#RAW_FILES[@]}))
    fi
    if [ $FLAGS_useMovie -eq 0 ]; then
      MOVIE_FILES=($(find * ${FIND_OPTIONS_FILE[@]} -iregex ".*${REGEXP_MOVIE}" -print | sort -n))
      NR_PROCESSED=$(($NR_PROCESSED + ${#MOVIE_FILES[@]}))
    fi
    IFS=$TMPIFS

    if [ $FLAGS_useLinks -eq 0 ]; then
      link_image
    else
      copy_image
    fi
    copy_raw
    copy_movie
  done
  echo "Processed $NR_PROCESSED file(s)"

  echo_debug "Writing "$LAST_RUN_FILE" file"
  DATESTR="$(date +'%Y-%m-%d %H:%M:%S')"
  echo "$DATESTR" > "$LAST_RUN_FILE"
  if [ $FLAGS_useChmod -eq 0 ]; then
    chmod -R 777 "$DST"
  fi

  echo_debug "Removing pid file"
  rm -f "$PID_FILE"
}

main
exit 0

This script has already a long history (it is about 5 years old).
Don't be too surprised if some things could be solved much more elegant nowadays