S3 backup script

backups3.sh

#!/bin/bash
##############################################################################
#
# S3 Backup Script
#
# Author: Thomas Vincent
# Based on template used for all Thomas Vincent shell scripts
#
# If your going to run this under OS X, you need to install GNU grep via brew
#
# TODO: Color code output, pull AWS options from commandline or json
#
# Refer to Google Shell Style guide
# https://google.github.io/styleguide/shell.xml
#
# Pass ShellCheck:
# https://www.shellcheck.net/
#
##############################################################################

_PROGRAM_NAME=$(readlink --canonicalize --no-newline "$0" | xargs basename )
_PROGRAM_VERSION="1.0.0"

declare -a configtest
configtest[0]='test' || (echo 'Failure: arrays not supported in this version of
bash.' && exit)

declare -a configuration

readonly INFO=2
readonly NOTICE=1

DEPENDENCY_MISSING=127
BACKUP_FAILED=7
DELETE_EXIT=6
DISK_ERROR=5
BAD_CONFIG=4
HOME_UNSET=3
INVALID_OPTION=2

#######################################
# Default global values
# Globals:
#   DEFAULT_CONFIG_FILE
# Arguments:
#   None
# Returns:
#   None
#######################################

CONFIG_FILE="backup.json"
default_backups_retention=5
defaultBucket="other_test_bucket"
destination_dir="/backupdir"


umask 0077

#######################################
# backup
# Globals:
#   currentTime
#   verify_environment
#   read_configuration
#   load_file_list
#   setup_backup_directory
# Arguments:
#   None
# Returns:
#   exit_code
#######################################

function main {
   currentTime="$( date +%Y-%m-%dT%H:%M:%S)"
   verify_environment
   read_configuration
   load_file_list
   setup_backup_directory
}

#######################################
# Cleanup files from the backup dir
# Globals:
#   backupDirectory
#   backupRoot
# Arguments:
#   None
# Returns:
#   DISK_ERROR
#######################################

function clean_up {
  if [[ -n "${backupDirectory}" && -d "${backupDirectory}" && "${backupDirectory}" != "/" && "${backupDirectory}" != "${backupRoot}" ]]; then
    notice "Removing ${backupDirectory}"
    if ! rm -r "${backupDirectory}" 1>/dev/null 2>&1; then
      error "rm command returned $?"
      error "Unable to remove ${backupDirectory}"
      builtin exit "${DISK_ERROR}"
    fi
    if [[ -e "${backupDirectory}" ]]; then
      error "The backup directory still exists"
      error "Unable to remove ${backupDirectory}"
      builtin exit "${DISK_ERROR}"
    fi
  elif [[ -n "${backupDirectory}" ]]; then
    error "Not deleting backup directory because of its location"
    error "Unable to remove ${backupDirectory}"
    builtin exit "${DISK_ERROR}"
  fi
}

#######################################
# backupfile
# Globals:
#  None
# Arguments:
#   None
# Returns:
#   None
#######################################

function backupfile {

  local file
  local skip_file
  local compression_file
  local file_bucket_setting
  local file_retention_setting
  local bucket_name
  local backups_to_retain
  local return_code=0

  if [[ -z "$1" ]]; then
    error "Got an empty file name for backup."
    return
  fi

  file="$1"

  eval skip_file=\$$"JSON__${file}__skip"
  if [[ "${skip_file}" == 1 ]]; then
    info "Skipping ${file}"
    return
  fi

  eval file_bucket_setting=\$$"JSON__${file}__bucket"
  if [[ -n "${file_bucket_setting}" ]]; then
    bucket_name="${file_bucket_setting}"
    info "Found configured bucket for ${file}: ${bucket_name}"
  elif [[ -n "${defaultBucket}" ]]; then
    bucket_name="${defaultBucket}"
    info "Using default bucket for ${file}"
  else
    error "The ${file} has no configured bucket and no default bucket is set"
    error "The backup for ${file} can not be uploaded to S3."
    return "${BACKUP_FAILED}"
  fi

  eval file_retention_setting=\$$"JSON__${file}__retention"
  if [[ -n "${file_retention_setting}" ]]; then
    backups_to_retain="${file_retention_setting}"
    info "Retention settings for ${file}: ${backups_to_retain}"
  elif [[ -n "${JSON__default__retention}" ]]; then
    backups_to_retain="${JSON__default__retention}"
    info "Using default retention time for ${file}"
  else
    notice "The ${file} does not have a specific retion time"
    notice "The default retention time is set in the json configuration file. Using ${default_backups_retention}."
    backups_to_retain="${default_backups_retention}"
  fi

  notice "Backup of starting..."

  backupFile="${file}.${currentTime}"
  backupFileLocation="${backupDirectory}/${backupFile}"
  if [[ "${compression_file}" == 1 ]]; then
    uploadFile="${backupFile}.tgz"
    uploadFileLocation="${backupDirectory}/${uploadFile}"
  else
    uploadFile="${backupFile}"
    uploadFileLocation="${backupDirectory}/${backupFile}"
  fi

  #######################################
  # Error checking if file location exists
  # Globals:
  #  None
  # Arguments:
  #  None
  # Returns:
  #  BACKUP_FAILED
  #######################################

  # Make sure it exists.
  if [[ ! -f "${backupFileLocation}" ]]; then
    error "ERROR: Backup file for ${file} not found"
    error "ERROR: Backup of ${file} failed!"
    return "${BACKUP_FAILED}"
  fi

  #######################################
  # Create archive and upload file
  # Globals:
  #  None
  # Arguments:
  #  None
  # Returns:
  #  BACKUP_FAILED
  #######################################

  if ! tar --create --gcompression --file "${uploadFileLocation}" --directory "${backupDirectory}"/"${backupFile}"; then
    error "Unable to compress backup of ${file}"
    error "Attempting to upload uncompressed file"
    uploadFile="${backupFile}"
    uploadFileLocation="${backupDirectory}/${backupFile}"
  fi

  if [[ ! -f "${uploadFileLocation}" ]]; then
    error "Could not find ${file} for upload"
    return "${BACKUP_FAILED}"
  fi

  if ! aws s3 cp "${uploadFileLocation}" "s3://${bucket_name}/" 1>/dev/null 2>&1; then
    error "S3 upload returned code $? for ${file}"
    error "Backup file for ${file} was probably not uploaded"
    return "${BACKUP_FAILED}"
  fi

  #######################################
  # Error checking and list backup files
  # Globals:
  #  None
  # Arguments:
  #  None
  # Returns:
  #  DELETE_EXIT
  #######################################

  if ! backup_files=( $( aws s3 ls "s3://${bucket_name}/${destination_dir}/${file}" | sort -r ) ) ; then
    error "Could not load a list of backup files for ${file}"
    error "Unable to delete backups for ${file}"
    return "${DELETE_EXIT}"
  fi

  if [[ "${backups_to_retain}" -eq "0" ]]; then
    info "Backups to retention (${backups_to_retain}) is zero. Deleting ${file} is"
    info "disabled."
    notice "Backup of ${file} is finished"
    return 0
  fi
  if [[ "${#backup_files[@]}" -lt "${backups_to_retain}" || "${backups_to_retain}" -eq "0" ]]; then
    info "Backups to retention (${backups_to_retain}) is greater than the number of"
    info "existing backups (${#backup_files[@]}) of ${file}. Not deleting."
    notice "Backup of ${file} finished"
    return 0
  fi
  for backup_file in "${backup_files[@]}"; do
    backup_count=$(( backup_count + 1 ))
    if [[ "${backup_count}" -gt "${backups_to_retain}" ]]; then
      notice "Removing backup ${backup_file} (backup ${backup_count})"
      if ! aws s3 rm "s3://${bucket_name}/${destination_dir}/${backup_file}" 1>/dev/null 2>&1; then
        error "S3 returned code $? when trying to remove s3://${bucket_name}/${destination_dir}/${backup_file}"
        error "Unable to remove backup file ${backup_file}"
        return_code="${DELETE_EXIT}"
      fi
    fi
  done
  notice "Backup of ${file} finished"
  return "${return_code}"
}

#######################################
# Read Configuration
# Globals:
#  None
# Arguments:
#  None
# Returns:
#  DEPENDENCY_MISSING
#######################################
function read_configuration {
  notice "Reading JSON Configuration"
  if [[ -z "${CONFIG_FILE}" ]]; then
     CONFIG_FILE="${HOME}/${DEFAULT_CONFIG_FILE}"
     info "Using default config file ${CONFIG_FILE}"
     readarray -t configuration < <(/usr/local/opt/grep/bin/ggrep -Po ':\s*"\K.+(?="\s*,?\s*$)' ${CONFIG_FILE} );
  else
     notice "Reading config file ${CONFIG_FILE}"
     readarray -t configuration < <(/usr/local/opt/grep/bin/ggrep -Po ':\s*"\K.+(?="\s*,?\s*$)' ${CONFIG_FILE} );
  fi

  if ! CONFIG_FILE=$( readlink -s -n -e "${CONFIG_FILE}" 2>/dev/null) ; then
    exit "${DEPENDENCY_MISSING}" "Unable to locate configuration file."
  fi
}

#######################################
# verify_environment
# Globals:
#  None
# Arguments:
#   None
# Returns:
#  DEPENDENCY_MISSING
#  HOME_UNSET
#######################################
function verify_environment {

  local -a neededUtilities=( readlink sed aws jq )
  local utilityName

  for utilityName in "${neededUtilities[@]}"; do
    if ! which "${utilityName}" 1> /dev/null 2>&1; then
      exit "${DEPENDENCY_MISSING}" "${utilityName} is required."
    else
       info "Found ${utilityName}"
    fi
  done

  if [[ -z "${HOME}" ]]; then
     exit "${HOME_UNSET}" "\$HOME is empty or not set."
  fi
}

#######################################
# info
# Globals:
#  None
# Arguments:
#   None
# Returns:
#   None
#######################################
function info {
  [[ "${DEBUG_LEVEL}" -ge "${INFO}" ]]  && echo "$@"
}

#######################################
# notice
# Globals:
#  None
# Arguments:
#  None
# Returns:
#  None
#######################################
function notice {
  [[ "${DEBUG_LEVEL}" -ge "${NOTICE}" ]] && echo "$@"
}

#######################################
# error
# Globals:
#  None
# Arguments:
#  None
# Returns:
#
#######################################
function error {
  echo -e "$@" 1>&2
}

#######################################
# exit with proper builtin return code
# Globals:
#  None
# Arguments:
#  None
# Returns:
#  None
#######################################
function exit {
  local exit_code="$1"
  shift
  echo -e "$@" 1>&2
  builtin exit "${exit_code}"
}

#######################################
# Show help
# Globals:
#   backupDirectory
#   backupRoot
# Arguments:
#   None
# Returns:
#   DISK_ERROR
#######################################
function show_help_message {
  echo "Usage: ${_PROGRAM_NAME} [OPTIONS]"
  echo ""
  echo "Backup files to S3."
  echo ""
  echo "This backups up files to S3. It can be"
  echo "configured to retain a set number of backups. Each file can"
  echo "be managed independently."
  echo ""
  printf " %-18s %s\\n" "-d --debug [LEVEL]" "Display debug information. More information"
  printf " %-18s %s\\n" " " "is available with -dd or by setting --debug=2"
  printf " %-18s %s\\n" "-h --help" "Display this or help"
  printf " %-18s %s\\n" "-v --version" "Display the version information"
  echo ""
  if [[ ! -z "$2" ]]; then
    if [[ "$1" == "0" ]]; then
      echo -e "$2"
      echo ""
    else
      echo -e "$2" >&2
      echo ""
    fi
  fi
  builtin exit "$1"
}

#######################################
# Trap to clean up on exit
# Globals:
#  None
# Arguments:
#  None
# Returns:
#  DISK_ERROR
#######################################

trap clean_up EXIT

OPTIONS=$(getopt -n "${_PROGRAM_NAME}" -o 'c:d::h::v' --long "config:,debug::,help::,version" -- "$@")
eval set -- "${OPTIONS}"

while true; do
  case "$1" in
    -c|--config)
         CONFIG_FILE="$2"
         shift 2
       ;;
    -d|--debug)
       case "$2" in
         ""|1)
           DEBUG_LEVEL="${NOTICE}"
           echo "Showing debug information"
           shift 2
           ;;
         d|2)
           DEBUG_LEVEL="${INFO}"
           echo "Showing detailed debug information"
           shift 2
           ;;
         *)
           exit 1 "Unknown debug level $2"
           ;;
       esac
       ;;
    -h|--help)
       show_help_message 0
       ;;
    -v|--version)
       echo "${_PROGRAM_NAME} ${_PROGRAM_VERSION}"
       echo "Homework for eBay."
       echo 'Author: Thomas Vincent.'
       builtin exit 0
       ;;
    --)
       shift
       break
       ;;
    \?)
      show_help_message 3 "\\nError: Invalid option: ${OPTARG}"
      exit "${INVALID_OPTION}" "Invalid option -${OPTARG}"
      ;;
  esac
done

main