A fancier mysql backup script for Xtrabackup:

xtrabackup_runner.sh

#!/bin/bash

# Script to create full and incremental backups (for all databases on server) using innobackupex from Percona.
# http://www.percona.com/doc/percona-xtrabackup/innobackupex/innobackupex_script.html
#
# (C)2017 Piotr Gbyliczek (p.gbyliczek at node4.co.uk)
# - changed to use build in compression mechanism
# - corrected restore process (restore from mixed compressed and uncompressed incrementals is supported as well)
# - tidied up script a bit and added new checks
# - removed unneeded xbstream usage (useful only for network transfers, not compatible with incremental backups)
# - removed dry-run
# 2012 Brad Svee modified to try to use xbstream
# (C)2012 Atha Kouroussis @ Vurbia Technologies International Inc.
# (C)2010 Owen Carter @ Mirabeau BV
# This script is provided as-is; no liability can be accepted for use.
# You are free to modify and reproduce so long as this attribution is preserved.

LOG_OUTPUT=${LOG_OUTPUT:-/dev/stdout}
LOG_DEFAULT_FMT='[$TS][$_LOG_LEVEL_STR][${FUNCNAME[1]}:${BASH_LINENO[0]}]'
LOG_DEBUG_LEVEL=10
LOG_INFO_LEVEL=20
LOG_WARNING_LEVEL=30
LOG_ERROR_LEVEL=40
LOG_CRITICAL_LEVEL=50
LOG_LEVEL=${LOG_LEVEL:-$LOG_INFO_LEVEL}
# LOG_LEVELS structure:
# Level, Level Name, Level Format, Before Log Entry, After Log Entry
LOG_LEVELS=(
  $LOG_DEBUG_LEVEL    'DEBUG   ' "$LOG_DEFAULT_FMT" "\e[1;34m"    "\e[0m"
  $LOG_INFO_LEVEL     'INFO    ' "$LOG_DEFAULT_FMT" "\e[1;32m"    "\e[0m"
  $LOG_WARNING_LEVEL  'WARNING ' "$LOG_DEFAULT_FMT" "\e[1;33m"    "\e[0m"
  $LOG_ERROR_LEVEL    'ERROR   ' "$LOG_DEFAULT_FMT" "\e[1;31m"    "\e[0m"
  $LOG_CRITICAL_LEVEL 'CRITICAL' "$LOG_DEFAULT_FMT" "\e[1;37;41m" "\e[0m"
)

# Some support functions
find_log_level () {
  local LEVEL=$1
  local i
  _LOG_LEVEL_STR="$LEVEL"
  for ((i=0; i<${#LOG_LEVELS[@]}; i+=5)); do
    if [[ "$LEVEL" == "${LOG_LEVELS[i]}" ]]; then
      _LOG_LEVEL_STR="${LOG_LEVELS[i+1]}"
      _LOG_LEVEL_FMT="${LOG_LEVELS[i+2]}"
      _LOG_LEVEL_BEGIN="${LOG_LEVELS[i+3]}"
      _LOG_LEVEL_END="${LOG_LEVELS[i+4]}"
      return 0
    fi
  done
  _LOG_LEVEL_FMT="$LOG_DEFAULT_FMT"
  _LOG_LEVEL_BEGIN=""
  _LOG_LEVEL_END=""
  return 1
}

# General logging function
# $1: Level
log () {
  local LEVEL=$1
  shift
  (( LEVEL < LOG_LEVEL )) && return 1
  local TS=$(date +'%Y-%m-%d %H:%M:%S.%N')
  # Keep digits only up to milliseconds
  TS=${TS%??????}
  find_log_level $LEVEL
  local OUTPUT
  eval "OUTPUT=\"$_LOG_LEVEL_FMT\""
  echo -ne "$_LOG_LEVEL_BEGIN$OUTPUT " > "$LOG_OUTPUT"
  echo -n $@ > "$LOG_OUTPUT"
  echo -e "$_LOG_LEVEL_END" > "$LOG_OUTPUT"
}

shopt -s expand_aliases
alias debug='log 10'
alias info='log 20'
alias warn='log 30'
alias error='log 40'
alias critical='log 50'
alias call_stack='debug Traceback ; log_call_stack'

# Log Call Stack
log_call_stack () {
  local i=0
  local FRAMES=${#BASH_LINENO[@]}
  # FRAMES-2 skips main, the last one in arrays
  for ((i=FRAMES-2; i>=0; i--)); do
    echo '  File' \"${BASH_SOURCE[i+1]}\", line ${BASH_LINENO[i]}, in ${FUNCNAME[i+1]}
    # Grab the source code of the line
    sed -n "${BASH_LINENO[i]}{s/^/    /;p}" "${BASH_SOURCE[i+1]}"
    # TODO extract arugments from "${BASH_ARGC[@]}" and "${BASH_ARGV[@]}"
    # It requires 'shopt -s extdebug'
  done
}

function check_mysql {
  # Check for mysql binary
  if [ ! -f $MYSQL_PATH ]; then
    error "mysql not found under ${MYSQL_PATH}. Please specify the correct binary path (currently ${BIN_DIR}) with -b and/or correct version with -v (currently ${VERSION})"
    return 0
  else
    if [ ! -x $MYSQL_PATH ]; then
      error "mysql found under ${MYSQL_PATH} but we don't seem to have permission to execute"
      return 0
    fi
  fi

  # Check for mysql binary
  if [ ! -f $MYSQLADMIN_PATH ]; then
    error "mysqladmin not found under ${MYSQLADMIN_PATH}. Please specify the correct binary path (currently ${BIN_DIR}) with -b and/or correct version with -v (currently ${VERSION})"
    return true
  else
    if [ ! -x $MYSQLADMIN_PATH ]; then
      error "mysqladmin found under ${MYSQLADMIN_PATH} but we don't seem to have permission to execute"
      return 0
    fi
  fi
  # Check that MySQL is running
  if [ "$(pgrep mysql | wc -l)" -lt "$MYSQL_PROC_COUNT" ]; then
    error "MySQL processes not present, check if server is running"
    return 0
  fi
  # Check we can access MySQL
  if ! $(echo 'exit' | $MYSQL_PATH -s $USEROPTIONS 2> /dev/null) ; then
   error "Supplied mysql username or password appears to be incorrect. Please supply correct credentials with -u and -p"
   return 0
  fi
  return 1
}


function validate {
  # Validate that we were supplied with a valid action to perform
  if [ -z $ACTION ]
  then
    error "You must specify and action to perform (backup or restore)"
    INVALID=true
  else
    case $ACTION in
      backup)
        INVALID=check_mysql

        # Validate the presence of the backup directory
        if [ ! -d $BACKUP_DIR ]
        then
          error "Backup directory ${BACKUP_DIR} does not exist or its not accesible. Please specify the correct path with -w"
          INVALID=true
        else
          if [ ! -d $FULLBACKUP_DIR ]
          then
            warn "Full backup directory $FULLBACKUP_DIR does not exist. Creating..."
            mkdir $FULLBACKUP_DIR
          fi

          if [ ! -d $INCRBACKUP_DIR ]
          then
            warn "Incremental backup directory $INCRBACKUP_DIR does not exist. Creating..."
            mkdir $INCRBACKUP_DIR
          fi
        fi

        # Check and normalize backup type
        case $TYPE in
          full|Full)
            TYPE="full"
            ;;
          incr|incremental|Incremental|diff|differential|Differential)
            TYPE="incr"
            ;;
          auto)
            TYPE="auto"
            ;;
          *)
            error "Unknown backup type ${TYPE}. Valid backup types are full|incremental|auto. Default is auto."
            INVALID=true
            ;;
        esac

        if [ $RESTORE == 'true' ]; then
          warn "restore flag set in backup mode - ignoring"
        fi
        ;;
      restore)
        if [ "$(pgrep mysql | wc -l)" -lt "$MYSQL_PROC_COUNT" ]; then
          warn "MySQL not running, ensure that MySQL version is set to right value using -V"
        fi
        # Validate the presence of the backup directory
        if [ ! -d $RESTORE_DIR ]
        then
          error "Backup directory ${RESTORE_DIR} does not exist or its not accesible. Please specify the correct path with -w"
          INVALID=true
        fi
        if [ "$(ls -1A $RESTORE_DIR)" ]
        then
          error "Restore directory ${RESTORE_DIR} not empty. Please specify empty directory"
          INVALID=true
        fi
        if [ $RESTORE == 'true' ]; then
          if [ "$(ls -1A $DATA_DIR)" ]
          then
            error "Data directory $DATA_DIR not empty. Please specify empty directory"
            INVALID=true
          fi
        fi
        # Validate that we were supplied with a valid directory to restore from
        if [ ! -e $BACKUP_SRC_DIR ]
        then
          error "You must specify a valid directory to restore from"
          INVALID=true
        elif [ ! -d $BACKUP_SRC_DIR ]
        then
          error "Restore directory ${BACKUP_SRC_DIR} does not exist or its not accesible. Please specify the correct path with -w"
          INVALID=true
        fi
        ;;
      *)
        error "Invalid action ${ACTION}. Valid actions are backup|restore"
        INVALID=true
        ;;
    esac
  fi

  # Validate the innobackupex binary is present and executable
  if [ ! -f $INNOBACKUPEX_PATH ]
  then
    error "innobackupex not found under ${BIN_DIR}. Please specify the correct binary path with -b and/or correct version with -v (currently ${VERSION})"
    INVALID=true
  else
    if [ ! -x $INNOBACKUPEX_PATH ]
    then
      error "innobackupex found under ${BIN_DIR} but we don't seem to have permission to execute"
      INVALID=true
    fi
  fi

  # Validate the presence of the MySQL config file
  if [ ! -f $MYCNF ]
  then
    error "MySQL config file not found under ${MYCNF}. Please specify the correct path with -c"
    INVALID=true
  fi

  # Exit if we found problems
  if $INVALID
  then
    critical "Validation errors found. Exiting."
    exit 2
  fi
}

# Check for errors in innobackupex output
check_innobackupex_error() {
        if [ -z "$(tail -1 $ERRFILE | grep 'completed OK!')" ] ; then
    critical "$INNOBACKUPEX_BIN failed:"; echo
    critical "---------- ERROR OUTPUT from $INNOBACKUPEX_BIN ----------"
    cat $ERRFILE
    rm -f $ERRFILE
    exit 1
  fi
}

function backup {
  # Grab start time
  STARTED_AT=$(date +%s)
  info "tmp file location: $ERRFILE"
  # Some info output
  info "----------------------------"
  info
  info "$0: MySQL backup script"
  info "Backup started: $STARTED_AT"
  info

  case $TYPE in
    full)
      full_backup
      ;;
    incr)
      incremental_backup
      ;;
    auto)
      auto_backup
      ;;
  esac

  check_innobackupex_error

  THISBACKUP_DIR=$(awk -- "/Backup created in directory/ { print p[2] }" $ERRFILE)
  rm -f $ERRFILE

  info "Databases backed up successfully to: $THISBACKUP_DIR"
  info

  # Cleanup
  info "Cleanup. Keeping only $KEEP full backups and its incrementals."
  AGE=$(($FULLBACKUPLIFE * $KEEP / 60))
  find $FULLBACKUP_DIR -maxdepth 1 -type d -mmin +$AGE -execdir echo "removing: "$FULLBACKUP_DIR/{} \; -execdir rm -rf $FULLBACKUP_DIR/{} \; -execdir echo "removing: "$INCRBACKUP_DIR/{} \; -execdir rm -rf $INCRBACKUP_DIR/{} \;

  info
  info "completed: $(date)"
  exit 0
}

function find_latest_full_backup {
  # Find latest full backup
  LATEST_FULL=$(find $FULLBACKUP_DIR -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -nr | head -1)
  debug "running : find $FULLBACKUP_DIR -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -nr | head -1"
  debug "LATEST_FULL: $LATEST_FULL"
  # Get latest backup last modification time
  LATEST_FULL_CREATED_AT=$(stat -c %Y $FULLBACKUP_DIR/$LATEST_FULL)
  info "Latest full: $LATEST_FULL_CREATED_AT"
}

function find_latest_incremental_backup {
  # Find latest incremental backup.
  if [ ! -d $INCRBACKUP_DIR/$LATEST_FULL ]; then
     mkdir -p $INCRBACKUP_DIR/$LATEST_FULL
  else
    LATEST_INCR=$(find $INCRBACKUP_DIR/$LATEST_FULL -mindepth 1 -maxdepth 1 -type d | sort -nr | head -1)
    debug "running : find $INCRBACKUP_DIR/$LATEST_FULL -mindepth 1 -maxdepth 1 -type d | sort -nr | head -1"
    debug "LATEST_INCR: $LATEST_INCR"
  fi
  # If this is the first incremental, use the full as base. Otherwise, use the latest incremental as base.
  if [ ! $LATEST_INCR ] ; then
    INCRBASE_DIR=$FULLBACKUP_DIR/$LATEST_FULL
  else
    INCRBASE_DIR=$LATEST_INCR
  fi
  debug "INCRBASE_DIR: $INCRBASE_DIR"
}

function full_backup {
  TSTAMP=$(date +%Y-%m-%d_%H-%M-%S)
  if [ $COMPRESSION == true ] ; then
    compressed_full_backup
  else
    uncompressed_full_backup
  fi
}

function uncompressed_full_backup {
  info "Running new full backup."
     $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS $USEROPTIONS $FULLBACKUP_DIR > $ERRFILE 2>&1
}

function compressed_full_backup {
    info "Running new compressed full backup."
    XOPTIONS=$XOPTIONS" --compress"
    $INNOBACKUPEX_PATH --defaults-file=$MYCNF $USEROPTIONS $XOPTIONS $FULLBACKUP_DIR 2> $ERRFILE
}

function incremental_backup {
  TSTAMP=$(date +%Y-%m-%d_%H-%M-%S)
  find_latest_incremental_backup
  # Create incremental backups dir if not exists.
  NEW_INCR_DIR=$INCRBACKUP_DIR/$LATEST_FULL
  if [ ! -z  $NEW_INCR_DIR ]; then
    mkdir -p $NEW_INCR_DIR
  fi

  if [ $COMPRESSION == true ] ; then
    compressed_incremental_backup
  else
    uncompressed_incremental_backup
  fi
}

function uncompressed_incremental_backup {
  info "Running new incremental backup using $INCRBASE_DIR as base."
     $INNOBACKUPEX_PATH --defaults-file=$MYCNF $USEROPTIONS $XOPTIONS --incremental $NEW_INCR_DIR --incremental-basedir $INCRBASE_DIR > $ERRFILE 2>&1
}

function compressed_incremental_backup {
        info "Running new compressed incremental backup using $INCRBASE_DIR as base."
        XOPTIONS=$XOPTIONS" --compress"
        debug "temp incremental dir :  $NEW_INCR_DIR"
        debug "timestamp :  $TSTAMP"
        debug "Incremental base dir :  $INCRBASE_DIR"
       $INNOBACKUPEX_PATH --defaults-file=$MYCNF $USEROPTIONS $XOPTIONS --incremental --incremental-basedir $INCRBASE_DIR $NEW_INCR_DIR 2> $ERRFILE
}

function auto_backup {
  find_latest_full_backup

  # Run an incremental backup if latest full is still valid. Otherwise, run a new full one.
  if [ "$LATEST_FULL" -a $(expr $LATEST_FULL_CREATED_AT + $FULLBACKUPLIFE + 5) -ge $STARTED_AT ] ; then
    incremental_backup
  else
    full_backup
  fi
}

function restore() {
  # Some info output
  info "----------------------------"
  info
  info "$0: MySQL backup script"
  info "Restore started: $(date)"
  info
  
  XOPTIONS=$XOPTIONS" --use-memory=$MEMORY"
  
  debug "parent_dir : $PARENT_DIR"
  debug "fullrestts : $FULLRESTTS"
  debug "incrrestts : $INCRRESTTS"
  debug "xoptions : $XOPTIONS"
  debug "restore : $RESTORE"

  function is_compressed() {
      if [ "$(find $1 -iname '*.qp' | wc -l )" -gt "0" ]; then
        return 0
      else
        return 1
      fi
  }

  if [ "$RESTORE" == "true" ]; then
     if [ $MYSQLVER ]; then
       debug "MYSQLVER: ${MYSQLVER}"
       MYSQLVER=${MYSQLVER%%.}
       debug "MYSQLVER: ${MYSQLVER}"
       XOPTIONS=$XOPTIONS" --ibbackup xtrabackup_$MYSQLVER"
       debug "XOPTIONS: ${XOPTIONS}"
     else
       info "Copy back function was used."
       info "MySQL datadir must be empty for this to work and mysql version (-V) is required"
       exit 1
     fi
  fi
  if [ "$INCRRESTTS" == "" ]; then

    info "Restore full backup from $(basename $BACKUP_SRC_DIR)"

    cp -R $BACKUP_SRC_DIR/* $RESTORE_DIR

    if is_compressed $RESTORE_DIR; then
      debug "full backup is compressed"
      info "Decompressing full backup"
      find $RESTORE_DIR -iname '*.qp' -execdir qpress -d {} ./  \;
      find $RESTORE_DIR -iname '*.qp' -execdir rm -f {} \;
    fi

  else
    if [ -n $INCRRESTTS ]; then
      FULLBACKUP=$PARENT_DIR/$FULLRESTTS
      FULLBACKUP=${FULLBACKUP/\/incr\//\/full\/}

      debug "FULLBACKUP : $FULLBACKUP"

      if [ ! -d $FULLBACKUP ]; then
        error "Full backup: $FULLBACKUP does not exist."
        exit 1
      fi

      if [ ! -d $BACKUP_SRC_DIR ]; then
        error "Full backup: $BACKUP_SRC_DIR does not exist."
        exit 1
      fi

      info "Restore full backup from $FULLRESTTS, up to incremental from  $INCRRESTTS"
      cp -R $FULLBACKUP/* $RESTORE_DIR
      if is_compressed $RESTORE_DIR; then
          debug "full backup is compressed"
          info "Decompressing full backup"
          find $RESTORE_DIR -iname '*.qp' -execdir qpress -d {} ./  \;
          find $RESTORE_DIR -iname '*.qp' -execdir rm -f {} \;
      fi

      info "Replay committed transactions on full backup"
      debug "running :  $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --apply-log --redo-only $RESTORE_DIR > $ERRFILE 2>&1"
      $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS $USEROPTIONS --apply-log --redo-only $RESTORE_DIR > $ERRFILE 2>&1
      check_innobackupex_error

      # Apply incrementals to base backup
      debug "find $PARENT_DIR/$FULLRESTTS -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -n"
      for i in $(find $PARENT_DIR/$FULLRESTTS -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -n); do
        info "Applying $i to full ..."
        mkdir $RESTORE_DIR/$i
        cp -R $PARENT_DIR/$FULLRESTTS/$i/* $RESTORE_DIR/$i
        if is_compressed $RESTORE_DIR/$i; then
          debug "incremental backup $i is compressed"
          info "Decompressing incremental backup: $i"
          find $RESTORE_DIR/$i -iname '*.qp' -execdir qpress -d {} ./  \;
          find $RESTORE_DIR/$i -iname '*.qp' -execdir rm -f {} \;
        fi

        debug "running : $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --apply-log --redo-only $RESTORE_DIR --incremental-dir=$RESTORE_DIR/$i > $ERRFILE 2>&1"
        $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS $USEROPTIONS --apply-log --redo-only $RESTORE_DIR --incremental-dir=$RESTORE_DIR/$i > $ERRFILE 2>&1
        check_innobackupex_error
        rm -rf $RESTORE_DIR/$i

        if [ $INCRRESTTS = $i ]; then
          break # break. we are restoring up to this incremental.
        fi

      done
    else
      error "Unknown backup type"
      exit 1
    fi
  fi
  prepare_restore

  if [ "$RESTORE" == "true" ]; then
    do_restore
  fi

  rm -f $ERRFILE
  info "Backup restored successfully."
  info "If copy-back was used, you should be able to start mysql now."
  info "Otherwise files are located in $RESTORE_DIR"
  info "Verify files ownership in mysql data dir."
  info "Run 'chown -R mysql:mysql /path/to/data/dir' if necessary."
  info "If SELinux is enabled, you may need the following :"
  info
  info "semanage fcontext -a -t mysqld_db_t \"/path/to/data(/.*)?\""
  info "restorecon -Rv /path/to/data/"
  info
  info "Completed: $(date)"
  exit 0
}

prepare_restore() {
  info "Preparing restore ..."
  debug "running : $INNOBACKUPEX_PATH $XOPTIONS --defaults-file=$MYCNF --apply-log $RESTORE_DIR > $ERRFILE 2>&1"
  $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --apply-log $RESTORE_DIR > $ERRFILE 2>&1
  check_innobackupex_error
}

do_restore() {
  info
  info "Restoring ..."
  $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --copy-back $RESTORE_DIR > $ERRFILE 2>&1
  check_innobackupex_error
}

# Main routine
# default variables
COMPRESSION=false
XBSTREAM=false
XOPTIONS=''
INVALID=false
RESTORE=false
MYSQL_PROC_COUNT=2
# Parse commandline arguments
while getopts 'a:b:c:hHk:l:m:w:p:s:rt:T:u:v:V:zO:' OPTION
 do
  case $OPTION in
          a)    ACTION=$OPTARG
                ;;
          b)    BIN_DIR=$OPTARG
                ;;
          c)    CONFIG_FILE=$OPTARG
                ;;
          k)    KEEP=$OPTARG
                ;;
          l)    LIFE=$OPTARG
                ;;
          m)    MEMORY=$OPTARG
                ;;
          w)    WORK_DIR=$OPTARG
                WORK_DIR=${WORK_DIR%%/}
                ;;
          p)    PASSWORD=$OPTARG
                ;;
          s)    BACKUP_SRC_DIR=$OPTARG
                ;;
          r)    RESTORE=true
                ;;
          t)    TMP_DIR=$OPTARG
                ;;
          T)    TYPE=$OPTARG
                ;;
          u)    USER=$OPTARG
                ;;
          v)    VERSION=$OPTARG
                ;;
          V)    MYSQLVER=$OPTARG
                ;;
          z)    COMPRESSION=true
                ;;
          O)    XOPTIONS=$OPTARG
                ;;
          H)    echo 'Usage example :'
                echo '1. Full manual backup with compression :'
                printf "      %s -a backup -T full -u user -p password -w /path/to/backup/dir/ -z\n" $(basename $0) >&2
                echo '2. Incremental manual backup without compression :'
                printf "      %s -a backup -T incr -u user -p password -w /path/to/backup/dir\n" $(basename $0) >&2
                echo '3. Backup with full backup taken every 24 hours, incremental backups in between and 5 days retention :'
                echo '   Note: This is intended to be run from cron every hour or every few hours'
                printf "      %s -a backup -u user -p password -w /path/to/backup/dir -k 5 -l 86400\n" $(basename $0) >&2
                echo '4. Restore backup to a directory'
                printf "      %s -a restore -u user -p password -w /path/to/restore/dir -s /backups/incr/2017-08-14_10-48-33/2017-08-14_11-10-01/\n" $(basename $0) >&2
                exit 0
                ;;
          h|?)  printf "Usage: %s -a ACTION [-u USER] [-p PASSWORD] [-b BIN_DIR] [-c CONFIG_FILE] [-k KEEP] [-l LIFE] [-m MEMORY] [-w WORK_DIR] [-s BACKUP_SRC_DIR] [-t TMP_DIR] [-T TYPE] [-v VERSION] [-h] [-H]\n" $(basename $0) >&2
                echo 'Options:'
                echo '     -h                   display help'
                echo '     -a ACTION            specify operation to perform (backup|restore)'
                echo '     -u USER              set MySQL user (if backing up)'
                echo '     -p PASSWORD          set MySQL user password (if backing up)'
                echo '     -b BIN_DIR           specify the directory where binaries reside (/usr/bin by default)'
                echo '     -c CONFIG_FILE       specify the MySQL config file path (/etc/mysql/my.cnf by default)'
                echo '     -k KEEP              specify the number of full backups (and its incrementals) to keep (1 by default)'
                echo '     -l LIFE              specify the lifetime of the latest full backup in seconds (86400 by default)'
                echo '     -m MEMORY            specify the amount of memory to use when preparing the backup (1024M by default)'
                echo '     -s BACKUP_SRC_DIR    specify the backups directory to restore'
                echo '     -r                   specify if backup should be restored to datadir location using copy-back method (disabled by default)'
                echo '     -t TMP_DIR           specify the temporary directory path (/tmp by default)'
                echo '     -w WORK_DIR          specify the working directory (/mnt/backup | /mnt/restore by default, depending on action)'
                echo '     -T TYPE              specify the backup type (full|incremental|auto)'
                echo '     -v VERSION           specify innobackupex version to use (1.5.1 by default)'
                echo '     -V MYSQLVER          specify MySQL version to use (5.5 by default)'
                echo '     -w WORK_DIR          specify the working directory (/mnt/backup | /mnt/restore by default, depending on action)'
                echo '     -z                   compress backups'
                echo '     -O OPTIONS           set extra options for innobackupex --galera-info --parallel=4 --compress --compress-threads=4'
                echo '     -H                   usage examples'
                exit 0
                ;;
  esac
done
shift $(($OPTIND - 1))

BIN_DIR=${BIN_DIR:-/usr/bin}
TMP_DIR=${TMP_DIR:-/tmp}
DATA_DIR=${DATA_DIR:-/var/lib/mysql}
VERSION=${VERSION:-1.5.1}
MYSQLVER=${MYSQLVER:-5.5}
if [ -f $BIN_DIR/innobackupex ]; then
  INNOBACKUPEX_BIN=innobackupex
  INNOBACKUPEX_PATH=${BIN_DIR}/${INNOBACKUPEX_BIN}
elif [ -f $BIN_DIR/innobackupex-${VERSION} ]; then
  INNOBACKUPEX_BIN=innobackupex-${VERSION}
  INNOBACKUPEX_PATH=${BIN_DIR}/${INNOBACKUPEX_BIN}
else
  INNOBACKUPEX_BIN=missing
fi
USEROPTIONS="--user=${USER} --password=${PASSWORD}"
ERRFILE="$TMP_DIR/innobackupex-${ACTION}.$$.tmp"
TYPE="${TYPE:-auto}"
MYCNF=${CONFIG_FILE:-/etc/my.cnf}
MYSQL_PATH=${BIN_DIR}/mysql
MYSQLADMIN_PATH=${BIN_DIR}/mysqladmin
if [ $ACTION == 'backup' ]; then
  BACKUP_DIR=${WORK_DIR:-/mnt/backups} # Backups base directory
  BACKUP_DIR=${BACKUP_DIR%%/}
  FULLBACKUP_DIR=$BACKUP_DIR/full # Full backups directory
  INCRBACKUP_DIR=$BACKUP_DIR/incr # Incremental backups directory
elif [ $ACTION == 'restore' ]; then
  if [[ $BACKUP_SRC_DIR =~ "full" ]]; then
    PARENT_DIR=$(dirname $BACKUP_SRC_DIR)
    FULLRESTTS=${BACKUP_SRC_DIR#$PARENT_DIR}
    INCRRESTTS=""
  elif [[ $BACKUP_SRC_DIR =~ "incr" ]]; then
    TMP_PARENT_DIR=$(dirname $BACKUP_SRC_DIR)
    PARENT_DIR=$(dirname $TMP_PARENT_DIR)
    FULLRESTTS=${BACKUP_SRC_DIR#$PARENT_DIR}
    INCRRESTTS=${BACKUP_SRC_DIR#$TMP_PARENT_DIR}
    FULLRESTTS=${FULLRESTTS%$INCRRESTTS}
  fi
  FULLRESTTS=${FULLRESTTS//\/}
  INCRRESTTS=${INCRRESTTS//\/}
  RESTORE_DIR=${WORK_DIR:-/mnt/restore} # Backups restore directory
  FULLBACKUP_DIR=$BACKUP_SRC_DIR/full # Full backups directory
  INCRBACKUP_DIR=$BACKUP_SRC_DIR/incr # Incremental backups directory
fi
FULLBACKUPLIFE=${LIFE:-86400} # Lifetime of the latest full backup in seconds
KEEP=${KEEP:-1} # Number of full backups (and its incrementals) to keep
MEMORY=${MEMORY:-1024M} # Amount of memory to use when preparing the backup

debug "ACTION: ${ACTION}"
debug "INNOBACKUPEX_BIN: ${INNOBACKUPEX_BIN}"
debug "USEROPTIONS: ${USEROPTIONS}"
debug "TYPE: ${TYPE}"
debug "ERRFILE: ${ERRFILE}"
debug "MYCNF: ${MYCNF}"
debug "INNOBACKUPEX_PATH: ${INNOBACKUPEX_PATH}"
debug "MYSQL_PATH: ${MYSQL_PATH}"
debug "MYSQLADMIN_PATH: ${MYSQLADMIN_PATH}"
debug "WORK_DIR: ${WORK_DIR}"
if [ $ACTION == 'backup' ]; then
  debug "BACKUP_DIR: ${BACKUP_DIR}"
  debug "FULLBACKUP_DIR: ${FULLBACKUP_DIR}"
  debug "INCRBACKUP_DIR: ${INCRBACKUP_DIR}"
fi
if [ $ACTION == 'restore' ]; then
  debug "RESTORE_DIR: ${RESTORE_DIR}"
  debug "FULLRESTTS: $FULLRESTTS"
  debug "INCRRESTTS: $INCRRESTTS"
fi
debug "FULLBACKUPLIFE: ${FULLBACKUPLIFE}"
debug "KEEP: ${KEEP}"
debug "MEMORY: ${MEMORY}"
debug "COMPRESSION: ${COMPRESSION}"
debug "XBSTREAM: ${XBSTREAM}"
debug "XOPTIONS: ${XOPTIONS}"
debug "DRYRUN: ${DRYRUN}"
debug "RESTORE: ${RESTORE}"

# Validate input
validate

# Execute the requested action
eval $ACTION

xtrabackup_runner_24.sh

#!/bin/bash

# Script to create full and incremental backups (for all databases on server) using innobackupex from Percona.
# http://www.percona.com/doc/percona-xtrabackup/innobackupex/innobackupex_script.html
#
# (C)2017 Piotr Gbyliczek (p.gbyliczek at node4.co.uk)
# - changed to use build in compression mechanism
# - corrected restore process (restore from mixed compressed and uncompressed incrementals is supported as well)
# - tidied up script a bit and added new checks
# - removed unneeded xbstream usage (useful only for network transfers, not compatible with incremental backups)
# - removed dry-run
# 2012 Brad Svee modified to try to use xbstream
# (C)2012 Atha Kouroussis @ Vurbia Technologies International Inc.
# (C)2010 Owen Carter @ Mirabeau BV
# This script is provided as-is; no liability can be accepted for use.
# You are free to modify and reproduce so long as this attribution is preserved.

LOG_OUTPUT=${LOG_OUTPUT:-/dev/stdout}
LOG_DEFAULT_FMT='[$TS][$_LOG_LEVEL_STR][${FUNCNAME[1]}:${BASH_LINENO[0]}]'
LOG_DEBUG_LEVEL=10
LOG_INFO_LEVEL=20
LOG_WARNING_LEVEL=30
LOG_ERROR_LEVEL=40
LOG_CRITICAL_LEVEL=50
LOG_LEVEL=${LOG_LEVEL:-$LOG_INFO_LEVEL}
# LOG_LEVELS structure:
# Level, Level Name, Level Format, Before Log Entry, After Log Entry
LOG_LEVELS=(
  $LOG_DEBUG_LEVEL    'DEBUG   ' "$LOG_DEFAULT_FMT" "\e[1;34m"    "\e[0m"
  $LOG_INFO_LEVEL     'INFO    ' "$LOG_DEFAULT_FMT" "\e[1;32m"    "\e[0m"
  $LOG_WARNING_LEVEL  'WARNING ' "$LOG_DEFAULT_FMT" "\e[1;33m"    "\e[0m"
  $LOG_ERROR_LEVEL    'ERROR   ' "$LOG_DEFAULT_FMT" "\e[1;31m"    "\e[0m"
  $LOG_CRITICAL_LEVEL 'CRITICAL' "$LOG_DEFAULT_FMT" "\e[1;37;41m" "\e[0m"
)

# Some support functions
find_log_level () {
  local LEVEL=$1
  local i
  _LOG_LEVEL_STR="$LEVEL"
  for ((i=0; i<${#LOG_LEVELS[@]}; i+=5)); do
    if [[ "$LEVEL" == "${LOG_LEVELS[i]}" ]]; then
      _LOG_LEVEL_STR="${LOG_LEVELS[i+1]}"
      _LOG_LEVEL_FMT="${LOG_LEVELS[i+2]}"
      _LOG_LEVEL_BEGIN="${LOG_LEVELS[i+3]}"
      _LOG_LEVEL_END="${LOG_LEVELS[i+4]}"
      return 0
    fi
  done
  _LOG_LEVEL_FMT="$LOG_DEFAULT_FMT"
  _LOG_LEVEL_BEGIN=""
  _LOG_LEVEL_END=""
  return 1
}

# General logging function
# $1: Level
log () {
  local LEVEL=$1
  shift
  (( LEVEL < LOG_LEVEL )) && return 1
  local TS=$(date +'%Y-%m-%d %H:%M:%S.%N')
  # Keep digits only up to milliseconds
  TS=${TS%??????}
  find_log_level $LEVEL
  local OUTPUT
  eval "OUTPUT=\"$_LOG_LEVEL_FMT\""
  echo -ne "$_LOG_LEVEL_BEGIN$OUTPUT " > "$LOG_OUTPUT"
  echo -n $@ > "$LOG_OUTPUT"
  echo -e "$_LOG_LEVEL_END" > "$LOG_OUTPUT"
}

shopt -s expand_aliases
alias debug='log 10'
alias info='log 20'
alias warn='log 30'
alias error='log 40'
alias critical='log 50'
alias call_stack='debug Traceback ; log_call_stack'

# Log Call Stack
log_call_stack () {
  local i=0
  local FRAMES=${#BASH_LINENO[@]}
  # FRAMES-2 skips main, the last one in arrays
  for ((i=FRAMES-2; i>=0; i--)); do
    echo '  File' \"${BASH_SOURCE[i+1]}\", line ${BASH_LINENO[i]}, in ${FUNCNAME[i+1]}
    # Grab the source code of the line
    sed -n "${BASH_LINENO[i]}{s/^/    /;p}" "${BASH_SOURCE[i+1]}"
    # TODO extract arugments from "${BASH_ARGC[@]}" and "${BASH_ARGV[@]}"
    # It requires 'shopt -s extdebug'
  done
}

function check_mysql {
  # Check for mysql binary
  if [ ! -f $MYSQL_PATH ]; then
    error "mysql not found under ${MYSQL_PATH}. Please specify the correct binary path (currently ${BIN_DIR}) with -b and/or correct version with -v (currently ${VERSION})"
    return 0
  else
    if [ ! -x $MYSQL_PATH ]; then
      error "mysql found under ${MYSQL_PATH} but we don't seem to have permission to execute"
      return 0
    fi
  fi

  # Check for mysql binary
  if [ ! -f $MYSQLADMIN_PATH ]; then
    error "mysqladmin not found under ${MYSQLADMIN_PATH}. Please specify the correct binary path (currently ${BIN_DIR}) with -b and/or correct version with -v (currently ${VERSION})"
    return true
  else
    if [ ! -x $MYSQLADMIN_PATH ]; then
      error "mysqladmin found under ${MYSQLADMIN_PATH} but we don't seem to have permission to execute"
      return 0
    fi
  fi
  # Check that MySQL is running
  if [ "$(pgrep mysql | wc -l)" -lt "$MYSQL_PROC_COUNT" ]; then
    error "MySQL processes not present, check if server is running"
    return 0
  fi
  # Check we can access MySQL
  if ! $(echo 'exit' | $MYSQL_PATH -s $USEROPTIONS 2> /dev/null) ; then
   error "Supplied mysql username or password appears to be incorrect. Please supply correct credentials with -u and -p"
   return 0
  fi
  return 1
}


function validate {
  # Validate that we were supplied with a valid action to perform
  if [ -z $ACTION ]
  then
    error "You must specify and action to perform (backup or restore)"
    INVALID=true
  else
    case $ACTION in
      backup)
        INVALID=check_mysql

        # Validate the presence of the backup directory
        if [ ! -d $BACKUP_DIR ]
        then
          error "Backup directory ${BACKUP_DIR} does not exist or its not accesible. Please specify the correct path with -w"
          INVALID=true
        else
          if [ ! -d $FULLBACKUP_DIR ]
          then
            warn "Full backup directory $FULLBACKUP_DIR does not exist. Creating..."
            mkdir $FULLBACKUP_DIR
          fi

          if [ ! -d $INCRBACKUP_DIR ]
          then
            warn "Incremental backup directory $INCRBACKUP_DIR does not exist. Creating..."
            mkdir $INCRBACKUP_DIR
          fi
        fi

        # Check and normalize backup type
        case $TYPE in
          full|Full)
            TYPE="full"
            ;;
          incr|incremental|Incremental|diff|differential|Differential)
            TYPE="incr"
            ;;
          auto)
            TYPE="auto"
            ;;
          *)
            error "Unknown backup type ${TYPE}. Valid backup types are full|incremental|auto. Default is auto."
            INVALID=true
            ;;
        esac

        if [ $RESTORE == 'true' ]; then
          warn "restore flag set in backup mode - ignoring"
        fi
        ;;
      restore)
        if [ "$(pgrep mysql | wc -l)" -lt "$MYSQL_PROC_COUNT" ]; then
          warn "MySQL not running, ensure that MySQL version is set to right value using -V"
        fi
        # Validate the presence of the backup directory
        if [ ! -d $RESTORE_DIR ]
        then
          error "Backup directory ${RESTORE_DIR} does not exist or its not accesible. Please specify the correct path with -w"
          INVALID=true
        fi
        if [ "$(ls -1A $RESTORE_DIR)" ]
        then
          error "Restore directory ${RESTORE_DIR} not empty. Please specify empty directory"
          INVALID=true
        fi
        if [ $RESTORE == 'true' ]; then
          if [ "$(ls -1A $DATA_DIR)" ]
          then
            error "Data directory $DATA_DIR not empty. Please specify empty directory"
            INVALID=true
          fi
        fi
        # Validate that we were supplied with a valid directory to restore from
        if [ ! -e $BACKUP_SRC_DIR ]
        then
          error "You must specify a valid directory to restore from"
          INVALID=true
        elif [ ! -d $BACKUP_SRC_DIR ]
        then
          error "Restore directory ${BACKUP_SRC_DIR} does not exist or its not accesible. Please specify the correct path with -w"
          INVALID=true
        fi
        ;;
      *)
        error "Invalid action ${ACTION}. Valid actions are backup|restore"
        INVALID=true
        ;;
    esac
  fi

  # Validate the innobackupex binary is present and executable
  if [ ! -f $INNOBACKUPEX_PATH ]
  then
    error "innobackupex not found under ${BIN_DIR}. Please specify the correct binary path (currently ${BIN_DIR}) with -b and/or correct version with -v (currently ${VERSION})"
    INVALID=true
  else
    if [ ! -x $INNOBACKUPEX_PATH ]
    then
      error "innobackupex found under ${BIN_DIR} but we don't seem to have permission to execute"
      INVALID=true
    fi
  fi

  # Validate the presence of the MySQL config file
  if [ ! -f $MYCNF ]
  then
    error "MySQL config file not found under ${MYCNF}. Please specify the correct path with -c"
    INVALID=true
  fi

  # Exit if we found problems
  if $INVALID
  then
    critical "Validation errors found. Exiting."
    exit 2
  fi
}

# Check for errors in innobackupex output
check_innobackupex_error() {
        if [ -z "$(tail -1 $ERRFILE | grep 'completed OK!')" ] ; then
    critical "$INNOBACKUPEX_BIN failed:"; echo
    critical "---------- ERROR OUTPUT from $INNOBACKUPEX_BIN ----------"
    cat $ERRFILE
    rm -f $ERRFILE
    exit 1
  fi
}

function backup {
  # Grab start time
  STARTED_AT=$(date +%s)
  info "tmp file location: $ERRFILE"
  # Some info output
  info "----------------------------"
  info
  info "$0: MySQL backup script"
  info "Backup started: $STARTED_AT"
  info

  case $TYPE in
    full)
      full_backup
      ;;
    incr)
      incremental_backup
      ;;
    auto)
      auto_backup
      ;;
  esac

  check_innobackupex_error

  THISBACKUP_DIR=$(awk -- "/Backup created in directory/ { print p[2] }" $ERRFILE)
  rm -f $ERRFILE

  info "Databases backed up successfully to: $THISBACKUP_DIR"
  info

  # Cleanup
  info "Cleanup. Keeping only $KEEP full backups and its incrementals."
  AGE=$(($FULLBACKUPLIFE * $KEEP / 60))
  find $FULLBACKUP_DIR -maxdepth 1 -type d -mmin +$AGE -execdir echo "removing: "$FULLBACKUP_DIR/{} \; -execdir rm -rf $FULLBACKUP_DIR/{} \; -execdir echo "removing: "$INCRBACKUP_DIR/{} \; -execdir rm -rf $INCRBACKUP_DIR/{} \;

  info
  info "completed: $(date)"
  exit 0
}

function find_latest_full_backup {
  # Find latest full backup
  LATEST_FULL=$(find $FULLBACKUP_DIR -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -nr | head -1)
  debug "running : find $FULLBACKUP_DIR -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -nr | head -1"
  debug "LATEST_FULL: $LATEST_FULL"
  # Get latest backup last modification time
  LATEST_FULL_CREATED_AT=$(stat -c %Y $FULLBACKUP_DIR/$LATEST_FULL)
  info "Latest full: $LATEST_FULL_CREATED_AT"
}

function find_latest_incremental_backup {
  # Find latest incremental backup.
  if [ ! -d $INCRBACKUP_DIR/$LATEST_FULL ]; then
     mkdir -p $INCRBACKUP_DIR/$LATEST_FULL
  else
    LATEST_INCR=$(find $INCRBACKUP_DIR/$LATEST_FULL -mindepth 1 -maxdepth 1 -type d | sort -nr | head -1)
    debug "running : find $INCRBACKUP_DIR/$LATEST_FULL -mindepth 1 -maxdepth 1 -type d | sort -nr | head -1"
    debug "LATEST_INCR: $LATEST_INCR"
  fi
  # If this is the first incremental, use the full as base. Otherwise, use the latest incremental as base.
  if [ ! $LATEST_INCR ] ; then
    INCRBASE_DIR=$FULLBACKUP_DIR/$LATEST_FULL
  else
    INCRBASE_DIR=$LATEST_INCR
  fi
  debug "INCRBASE_DIR: $INCRBASE_DIR"
}

function full_backup {
  TSTAMP=$(date +%Y-%m-%d_%H-%M-%S)
  if [ $COMPRESSION == true ] ; then
    compressed_full_backup
  else
    uncompressed_full_backup
  fi
}

function uncompressed_full_backup {
  info "Running new full backup."
     $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS $USEROPTIONS --backup --target-dir=$FULLBACKUP_DIR/$TSTAMP > $ERRFILE 2>&1
}

function compressed_full_backup {
    info "Running new compressed full backup."
    XOPTIONS=$XOPTIONS" --compress"
    $INNOBACKUPEX_PATH --defaults-file=$MYCNF $USEROPTIONS $XOPTIONS --backup --target-dir=$FULLBACKUP_DIR/$TSTAMP 2> $ERRFILE
}

function incremental_backup {
  TSTAMP=$(date +%Y-%m-%d_%H-%M-%S)
  find_latest_incremental_backup
  # Create incremental backups dir if not exists.
  NEW_INCR_DIR=$INCRBACKUP_DIR/$LATEST_FULL
  if [ ! -z  $NEW_INCR_DIR ]; then
    mkdir -p $NEW_INCR_DIR
  fi

  if [ $COMPRESSION == true ] ; then
    compressed_incremental_backup
  else
    uncompressed_incremental_backup
  fi
}

function uncompressed_incremental_backup {
  info "Running new incremental backup using $INCRBASE_DIR as base."
     $INNOBACKUPEX_PATH --defaults-file=$MYCNF $USEROPTIONS $XOPTIONS --backup --target-dir=$NEW_INCR_DIR/$TSTAMP --incremental-basedir=$INCRBASE_DIR > $ERRFILE 2>&1
}

function compressed_incremental_backup {
        info "Running new compressed incremental backup using $INCRBASE_DIR as base."
        XOPTIONS=$XOPTIONS" --compress"
        debug "temp incremental dir :  $NEW_INCR_DIR"
        debug "timestamp :  $TSTAMP"
        debug "Incremental base dir :  $INCRBASE_DIR"
       $INNOBACKUPEX_PATH --defaults-file=$MYCNF $USEROPTIONS $XOPTIONS --backup --incremental-basedir=$INCRBASE_DIR --target-dir=$NEW_INCR_DIR/$TSTAMP 2> $ERRFILE
}

function auto_backup {
  find_latest_full_backup

  # Run an incremental backup if latest full is still valid. Otherwise, run a new full one.
  if [ "$LATEST_FULL" -a $(expr $LATEST_FULL_CREATED_AT + $FULLBACKUPLIFE + 5) -ge $STARTED_AT ] ; then
    incremental_backup
  else
    full_backup
  fi
}

function restore() {
  # Some info output
  info "----------------------------"
  info
  info "$0: MySQL backup script"
  info "Restore started: $(date)"
  info
  
  XOPTIONS=$XOPTIONS" --use-memory=$MEMORY"
  
  debug "parent_dir : $PARENT_DIR"
  debug "fullrestts : $FULLRESTTS"
  debug "incrrestts : $INCRRESTTS"
  debug "xoptions : $XOPTIONS"
  debug "restore : $RESTORE"

  function is_compressed() {
      if [ "$(find $1 -iname '*.qp' | wc -l )" -gt "0" ]; then
        return 0
      else
        return 1
      fi
  }

  if [ "$RESTORE" == "true" ]; then
     if [ $MYSQLVER ]; then
       debug "MYSQLVER: ${MYSQLVER}"
       MYSQLVER=${MYSQLVER%%.}
       debug "MYSQLVER: ${MYSQLVER}"
       XOPTIONS=$XOPTIONS" --ibbackup xtrabackup_$MYSQLVER"
       debug "XOPTIONS: ${XOPTIONS}"
     else
       info "Copy back function was used."
       info "MySQL datadir must be empty for this to work and mysql version (-V) is required"
       exit 1
     fi
  fi
  if [ "$INCRRESTTS" == "" ]; then

    info "Restore full backup from $(basename $BACKUP_SRC_DIR)"

    cp -R $BACKUP_SRC_DIR/* $RESTORE_DIR

    if is_compressed $RESTORE_DIR; then
      debug "full backup is compressed"
      info "Decompressing full backup"
      find $RESTORE_DIR -iname '*.qp' -execdir qpress -d {} ./  \;
      find $RESTORE_DIR -iname '*.qp' -execdir rm -f {} \;
    fi

  else
    if [ -n $INCRRESTTS ]; then
      FULLBACKUP=$PARENT_DIR/$FULLRESTTS
      FULLBACKUP=${FULLBACKUP/\/incr\//\/full\/}

      debug "FULLBACKUP : $FULLBACKUP"

      if [ ! -d $FULLBACKUP ]; then
        error "Full backup: $FULLBACKUP does not exist."
        exit 1
      fi

      if [ ! -d $BACKUP_SRC_DIR ]; then
        error "Full backup: $BACKUP_SRC_DIR does not exist."
        exit 1
      fi

      info "Restore full backup from $FULLRESTTS, up to incremental from  $INCRRESTTS"
      cp -R $FULLBACKUP/* $RESTORE_DIR
      if is_compressed $RESTORE_DIR; then
          debug "full backup is compressed"
          info "Decompressing full backup"
          find $RESTORE_DIR -iname '*.qp' -execdir qpress -d {} ./  \;
          find $RESTORE_DIR -iname '*.qp' -execdir rm -f {} \;
      fi

      info "Replay committed transactions on full backup"
      debug "running :  $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --apply-log --redo-only $RESTORE_DIR > $ERRFILE 2>&1"
      $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS $USEROPTIONS --prepare --apply-log-only --target-dir=$RESTORE_DIR > $ERRFILE 2>&1
      check_innobackupex_error

      # Apply incrementals to base backup
      debug "find $PARENT_DIR/$FULLRESTTS -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -n"
      for i in $(find $PARENT_DIR/$FULLRESTTS -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -n); do
        info "Applying $i to full ..."
        mkdir $RESTORE_DIR/$i
        cp -R $PARENT_DIR/$FULLRESTTS/$i/* $RESTORE_DIR/$i
        if is_compressed $RESTORE_DIR/$i; then
          debug "incremental backup $i is compressed"
          info "Decompressing incremental backup: $i"
          find $RESTORE_DIR/$i -iname '*.qp' -execdir qpress -d {} ./  \;
          find $RESTORE_DIR/$i -iname '*.qp' -execdir rm -f {} \;
        fi

        debug "running : $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --apply-log --redo-only $RESTORE_DIR --incremental-dir=$RESTORE_DIR/$i > $ERRFILE 2>&1"
        $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS $USEROPTIONS --prepare --apply-log-only --target-dir=$RESTORE_DIR --incremental-dir=$RESTORE_DIR/$i > $ERRFILE 2>&1
        check_innobackupex_error
        rm -rf $RESTORE_DIR/$i

        if [ $INCRRESTTS = $i ]; then
          break # break. we are restoring up to this incremental.
        fi

      done
    else
      error "Unknown backup type"
      exit 1
    fi
  fi
  prepare_restore

  if [ "$RESTORE" == "true" ]; then
    do_restore
  fi

  rm -f $ERRFILE
  info "Backup restored successfully."
  info "If copy-back was used, you should be able to start mysql now."
  info "Otherwise files are located in $RESTORE_DIR"
  info "Verify files ownership in mysql data dir."
  info "Run 'chown -R mysql:mysql /path/to/data/dir' if necessary."
  info "If SELinux is enabled, you may need the following :"
  info
  info "semanage fcontext -a -t mysqld_db_t \"/path/to/data(/.*)?\""
  info "restorecon -Rv /path/to/data/"
  info
  info "Completed: $(date)"
  exit 0
}

prepare_restore() {
  info "Preparing restore ..."
  debug "running : $INNOBACKUPEX_PATH $XOPTIONS --defaults-file=$MYCNF --apply-log $RESTORE_DIR > $ERRFILE 2>&1"
  $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --prepare --target-dir=$RESTORE_DIR > $ERRFILE 2>&1
  check_innobackupex_error
}

do_restore() {
  info
  info "Restoring ..."
  $INNOBACKUPEX_PATH --defaults-file=$MYCNF $XOPTIONS --copy-back --target-dir=$RESTORE_DIR > $ERRFILE 2>&1
  check_innobackupex_error
}

# Main routine
# default variables
COMPRESSION=false
XBSTREAM=false
XOPTIONS=''
INVALID=false
RESTORE=false
MYSQL_PROC_COUNT=1
# Parse commandline arguments
while getopts 'a:b:c:hHk:l:m:w:p:s:rt:T:u:v:V:zO:' OPTION
 do
  case $OPTION in
          a)    ACTION=$OPTARG
                ;;
          b)    BIN_DIR=$OPTARG
                ;;
          c)    CONFIG_FILE=$OPTARG
                ;;
          k)    KEEP=$OPTARG
                ;;
          l)    LIFE=$OPTARG
                ;;
          m)    MEMORY=$OPTARG
                ;;
          w)    WORK_DIR=$OPTARG
                WORK_DIR=${WORK_DIR%%/}
                ;;
          p)    PASSWORD=$OPTARG
                ;;
          s)    BACKUP_SRC_DIR=$OPTARG
                ;;
          r)    RESTORE=true
                ;;
          t)    TMP_DIR=$OPTARG
                ;;
          T)    TYPE=$OPTARG
                ;;
          u)    USER=$OPTARG
                ;;
          v)    VERSION=$OPTARG
                ;;
          V)    MYSQLVER=$OPTARG
                ;;
          z)    COMPRESSION=true
                ;;
          O)    XOPTIONS=$OPTARG
                ;;
          H)    echo 'Usage example :'
                echo '1. Full manual backup with compression :'
                printf "      %s -a backup -T full -u user -p password -w /path/to/backup/dir/ -z\n" $(basename $0) >&2
                echo '2. Incremental manual backup without compression :'
                printf "      %s -a backup -T incr -u user -p password -w /path/to/backup/dir\n" $(basename $0) >&2
                echo '3. Backup with full backup taken every 24 hours, incremental backups in between and 5 days retention :'
                echo '   Note: This is intended to be run from cron every hour or every few hours'
                printf "      %s -a backup -u user -p password -w /path/to/backup/dir -k 5 -l 86400\n" $(basename $0) >&2
                echo '4. Restore backup to a directory'
                printf "      %s -a restore -u user -p password -w /path/to/restore/dir -s /backups/incr/2017-08-14_10-48-33/2017-08-14_11-10-01/\n" $(basename $0) >&2
                exit 0
                ;;
          h|?)  printf "Usage: %s -a ACTION [-u USER] [-p PASSWORD] [-b BIN_DIR] [-c CONFIG_FILE] [-k KEEP] [-l LIFE] [-m MEMORY] [-w WORK_DIR] [-s BACKUP_SRC_DIR] [-t TMP_DIR] [-T TYPE] [-v VERSION] [-h] [-H]\n" $(basename $0) >&2
                echo 'Options:'
                echo '     -h                   display help'
                echo '     -a ACTION            specify operation to perform (backup|restore)'
                echo '     -u USER              set MySQL user (if backing up)'
                echo '     -p PASSWORD          set MySQL user password (if backing up)'
                echo '     -b BIN_DIR           specify the directory where binaries reside (/usr/bin by default)'
                echo '     -c CONFIG_FILE       specify the MySQL config file path (/etc/mysql/my.cnf by default)'
                echo '     -k KEEP              specify the number of full backups (and its incrementals) to keep (1 by default)'
                echo '     -l LIFE              specify the lifetime of the latest full backup in seconds (86400 by default)'
                echo '     -m MEMORY            specify the amount of memory to use when preparing the backup (1024M by default)'
                echo '     -s BACKUP_SRC_DIR    specify the backups directory to restore'
                echo '     -r                   specify if backup should be restored to datadir location using copy-back method (disabled by default)'
                echo '     -t TMP_DIR           specify the temporary directory path (/tmp by default)'
                echo '     -w WORK_DIR          specify the working directory (/mnt/backup | /mnt/restore by default, depending on action)'
                echo '     -T TYPE              specify the backup type (full|incremental|auto)'
                echo '     -v VERSION           specify innobackupex version to use (1.5.1 by default)'
                echo '     -V MYSQLVER          specify MySQL version to use (5.5 by default)'
                echo '     -w WORK_DIR          specify the working directory (/mnt/backup | /mnt/restore by default, depending on action)'
                echo '     -z                   compress backups'
                echo '     -O OPTIONS           set extra options for innobackupex --galera-info --parallel=4 --compress --compress-threads=4'
                echo '     -H                   usage examples'
                exit 0
                ;;
  esac
done
shift $(($OPTIND - 1))

BIN_DIR=${BIN_DIR:-/usr/bin}
TMP_DIR=${TMP_DIR:-/tmp}
DATA_DIR=${DATA_DIR:-/var/lib/mysql}
VERSION=${VERSION:-1.5.1}
MYSQLVER=${MYSQLVER:-5.5}
if [ -f $BIN_DIR/xtrabackup ]; then
  INNOBACKUPEX_BIN=xtrabackup
  INNOBACKUPEX_PATH=${BIN_DIR}/${INNOBACKUPEX_BIN}
elif [ -f $BIN_DIR/innobackupex ]; then
  INNOBACKUPEX_BIN=innobackupex
  INNOBACKUPEX_PATH=${BIN_DIR}/${INNOBACKUPEX_BIN}
elif [ -f $BIN_DIR/innobackupex-${VERSION} ]; then
  INNOBACKUPEX_BIN=innobackupex-${VERSION}
  INNOBACKUPEX_PATH=${BIN_DIR}/${INNOBACKUPEX_BIN}
else
  INNOBACKUPEX_BIN=missing
fi
USEROPTIONS="--user=${USER} --password=${PASSWORD}"
ERRFILE="$TMP_DIR/innobackupex-${ACTION}.$$.tmp"
TYPE="${TYPE:-auto}"
MYCNF=${CONFIG_FILE:-/etc/my.cnf}
MYSQL_PATH=${BIN_DIR}/mysql
MYSQLADMIN_PATH=${BIN_DIR}/mysqladmin
if [ $ACTION == 'backup' ]; then
  BACKUP_DIR=${WORK_DIR:-/mnt/backups} # Backups base directory
  BACKUP_DIR=${BACKUP_DIR%%/}
  FULLBACKUP_DIR=$BACKUP_DIR/full # Full backups directory
  INCRBACKUP_DIR=$BACKUP_DIR/incr # Incremental backups directory
elif [ $ACTION == 'restore' ]; then
  if [[ $BACKUP_SRC_DIR =~ "full" ]]; then
    PARENT_DIR=$(dirname $BACKUP_SRC_DIR)
    FULLRESTTS=${BACKUP_SRC_DIR#$PARENT_DIR}
    INCRRESTTS=""
  elif [[ $BACKUP_SRC_DIR =~ "incr" ]]; then
    TMP_PARENT_DIR=$(dirname $BACKUP_SRC_DIR)
    PARENT_DIR=$(dirname $TMP_PARENT_DIR)
    FULLRESTTS=${BACKUP_SRC_DIR#$PARENT_DIR}
    INCRRESTTS=${BACKUP_SRC_DIR#$TMP_PARENT_DIR}
    FULLRESTTS=${FULLRESTTS%$INCRRESTTS}
  fi
  FULLRESTTS=${FULLRESTTS//\/}
  INCRRESTTS=${INCRRESTTS//\/}
  RESTORE_DIR=${WORK_DIR:-/mnt/restore} # Backups restore directory
  FULLBACKUP_DIR=$BACKUP_SRC_DIR/full # Full backups directory
  INCRBACKUP_DIR=$BACKUP_SRC_DIR/incr # Incremental backups directory
fi
FULLBACKUPLIFE=${LIFE:-86400} # Lifetime of the latest full backup in seconds
KEEP=${KEEP:-1} # Number of full backups (and its incrementals) to keep
MEMORY=${MEMORY:-1024M} # Amount of memory to use when preparing the backup

debug "ACTION: ${ACTION}"
debug "INNOBACKUPEX_BIN: ${INNOBACKUPEX_BIN}"
debug "USEROPTIONS: ${USEROPTIONS}"
debug "TYPE: ${TYPE}"
debug "ERRFILE: ${ERRFILE}"
debug "MYCNF: ${MYCNF}"
debug "INNOBACKUPEX_PATH: ${INNOBACKUPEX_PATH}"
debug "MYSQL_PATH: ${MYSQL_PATH}"
debug "MYSQLADMIN_PATH: ${MYSQLADMIN_PATH}"
debug "WORK_DIR: ${WORK_DIR}"
if [ $ACTION == 'backup' ]; then
  debug "BACKUP_DIR: ${BACKUP_DIR}"
  debug "FULLBACKUP_DIR: ${FULLBACKUP_DIR}"
  debug "INCRBACKUP_DIR: ${INCRBACKUP_DIR}"
fi
if [ $ACTION == 'restore' ]; then
  debug "RESTORE_DIR: ${RESTORE_DIR}"
  debug "FULLRESTTS: $FULLRESTTS"
  debug "INCRRESTTS: $INCRRESTTS"
fi
debug "FULLBACKUPLIFE: ${FULLBACKUPLIFE}"
debug "KEEP: ${KEEP}"
debug "MEMORY: ${MEMORY}"
debug "COMPRESSION: ${COMPRESSION}"
debug "XBSTREAM: ${XBSTREAM}"
debug "XOPTIONS: ${XOPTIONS}"
debug "DRYRUN: ${DRYRUN}"
debug "RESTORE: ${RESTORE}"

# Validate input
validate

# Execute the requested action
eval $ACTION