Skip to content

Instantly share code, notes, and snippets.

@deajan

deajan/backup_tool_script

Last active February 15, 2018 07:47
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save deajan/d2cae9399fdd71e97a66fcfd08433051 to your computer and use it in GitHub Desktop.
Save deajan/d2cae9399fdd71e97a66fcfd08433051 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
backup_tool_script
#!/usr/bin/env bash
PROGRAM="backup_tool_script"
PROGRAM_VERSION=0.4.0
PROGRAM_BUILD=2018021403
AUTHOR="(C) 2017-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
IS_STABLE=yes
#TODO
# postponed arrays / files grow a lot. Consider having them "rolling"
# SendMail and SendEmail convert functions inverted, check on osync and obackup
# command line arguments don't take -AaqV for example
# ExecTasks still needs some better call argument list
# ExecTasks sub function relocate
# ofunctions spinner is updated code too
# ofunctions checkRFC addition
# no_maxtime is not used
## backup_tool_script - A script to check burp backup sanity
## backup_tool_script can verify a given number of backups for each client. It can run verifiy operations in parallel.
## Verify operations are timed in order to stop them after a given amount of time, leaving the system performance ready for backup operations.
## The script can also list clients that have outdated backups. It uses two different methods to list clients in order to detect rogue clients.
## It can also ensure that the burp server service is running properly, relaunch it if needed, on a scheduled basis.
## The script can send a warning / error when problems are found, even while operating.
## The script can send a warning / error when disk quotas exceed, even while operating.
## backup_tool_script can also launch vss_strip for each file found in a given directory.
## The script can also send mails directly to the client, if an email address is given in client config file as 'label = email_address : some@example.com'
## When exiting, backup_tool_script ensures that no forked burp processes remain, without touching other burp processes that didn't belong to backup_tool_script.
## Set an unique identifier for the script which will be used for logs and alert mails
INSTANCE_ID="base"
## Backup verifications timers
## After how much time (in seconds) for a single verification a warning should be logged (defaults to 3 hours)
SOFT_MAX_EXEC_TIME_PER_VERIFY=10800
## After how much time (in seconds) for a single verification the process should be stopped (defaults to 5 hours)
HARD_MAX_EXEC_TIME_PER_VERIFY=18000
## After how much seconds of execution of all steps a warning should be logged (defaults to 10 hours)
SOFT_MAX_EXEC_TIME=36000
## After how much seconds of execution of all steps a verification process should be stopped (defaults to 12 hours)
HARD_MAX_EXEC_TIME=43200
# Verify operations checks
## When a client isn't idle, should we postpone verification process
POSTPONE=yes
## When postponed, how much time (in seconds) before next try (defauls to 1 hour)
POSTPONE_TIME=3600
## How many times should we retry the verification command
POSTPONE_RETRY=2
## Backup executable (can be set to /usr/sbin/burp, /usr/local/sbin/burp, or autodetect via $(type -p burp))
BACKUP_EXECUTABLE=/usr/sbin/burp
## Burp service type (can be "initv" or "systemd")
SERVICE_TYPE=systemd
## How many simultaneous verify operations should be launched (please check I/O and CPU usage before increasing this)
PARELLEL_VERIFY_CONCURRENCY=2
# ------------ Mail alert settings -------------
## General alert mail subject
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
## Optional change of mail body encoding (using iconv)
## By default, all mails are sent in UTF-8 format without header (because of maximum compatibility of all platforms)
## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle)
MAIL_BODY_CHARSET="ISO-8859-1"
# ------------ Client specific alert email template --------------
## Email subject
CLIENT_ALERT_SUBJECT="burp - Attention Sauvegarde de votre poste"
## Valid message body placeholders are
## [NUMBERDAYS] is the number of days we check.
## [QUOTAEXCEED] is the size of the actual backup versus the quota.
## [CLIENT] is the client name
## Message sent directly to client email address when no recent backups are found.
CLIENT_ALERT_BODY_OUTDATED="Bonjour,
La sauvegarde burp de votre poste n'a pas été effectuée depuis [NUMBERDAYS] jour(s) pour le client [CLIENT].
Merci de procéder à une sauverde en utilisant l'outil burp sur votre bureau.
L'équipe technique
"
# Message sent directly to client email address when quota exceeded.
CLIENT_ALERT_BODY_QUOTA="Bonjour,
Votre quota de disque a été dépassé ([QUOTAEXCEED]) sur la sauvegarde burp pour le client [CLIENT].
L'équipe technique
"
# ------------ Do not modify under this line unless you have great cow powers --------------
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
fi
export LC_ALL=C
_LOGGER_SILENT=false
_LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date"
if [ "$KEEP_LOGGING" == "" ]; then
KEEP_LOGGING=721
fi
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=false
WARN_ALERT=false
LOCAL_USER=$(whoami)
LOCAL_HOST=$(hostname)
SCRIPT_PID=$$
TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"
## Default log file until config file is loaded
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.log"
elif [ -w . ]; then
LOG_FILE="./$PROGRAM.log"
else
LOG_FILE="/tmp/$PROGRAM.log"
fi
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
RUN_DIR=/var/tmp
else
RUN_DIR=.
fi
#### DEBUG SUBSET ####
## allow function call checks #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG
_DEBUG=yes #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no
_LOGGER_VERBOSE=false
else
trap 'TrapError ${LINENO} $?' ERR
_LOGGER_VERBOSE=true
fi
if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
#### DEBUG SUBSET END ####
#### Logger SUBSET ####
#### RemoteLogger SUBSET ####
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
local IFS="$1"; shift; echo "$*";
}
# Sub function of Logger
function _Logger {
local logValue="${1}" # Log to file
local stdValue="${2}" # Log to screeen
local toStdErr="${3:-false}" # Log to stderr instead of stdout
if [ "$logValue" != "" ]; then
echo -e "$logValue" >> "$LOG_FILE"
# Current log file
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP"
fi
if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
if [ $toStdErr == true ]; then
# Force stderr color in subshell
(>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi
}
# General log function with log levels:
# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
function Logger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date) - "
else
prefix=""
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "$prefix$value" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
else
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}
#### Logger SUBSET END ####
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
if kill -0 "$pid" > /dev/null 2>&1; then
# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
}
function KillAllChilds {
local pids="${1}" # List of parent pids to kill separated by semi-colon
local self="${2:-false}" # Should parent be killed too ?
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
local errorcount=0
IFS=';' read -a pidsArray <<< "$pids"
for pid in "${pidsArray[@]}"; do
KillChilds $pid $self
if [ $? != 0 ]; then
errorcount=$((errorcount+1))
fi
done
return $errorcount
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
__CheckArguments 0-1 $# "$@" #__WITH_PARANOIA_DEBUG
local attachment
local attachmentFile
local subject
local body
if [ "$DESTINATION_MAILS" == "" ]; then
return 0
fi
if [ "$_DEBUG" == "yes" ]; then
Logger "Debug mode, no warning mail will be sent." "NOTICE"
return 0
fi
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
attachment=false
else
attachment=true
fi
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)"
if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID"
elif [ $WARN_ALERT == true ]; then
subject="Warning alert for $INSTANCE_ID"
else
subject="Alert for $INSTANCE_ID"
fi
if [ $runAlert == true ]; then
subject="Currently runing - $subject"
else
subject="Finished run - $subject"
fi
if [ "$attachment" == true ]; then
attachmentFile="$ALERT_LOG_FILE"
fi
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
# Delete tmp log file
if [ "$attachment" == true ]; then
if [ -f "$ALERT_LOG_FILE" ]; then
rm -f "$ALERT_LOG_FILE"
fi
fi
}
# Generic email sending function.
# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or ""
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file"
# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend)
# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or ""
# smtp_server.domain.tld is mandatory, as is smtpPort (should be 25, 465 or 587)
# encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"
function SendEmail {
local subject="${1}"
local message="${2}"
local destinationMails="${3}"
local attachment="${4}"
local senderMail="${5}"
local smtpServer="${6}"
local smtpPort="${7}"
local encryption="${8}"
local smtpUser="${9}"
local smtpPassword="${10}"
__CheckArguments 3-10 $# "$@" #__WITH_PARANOIA_DEBUG
local mail_no_attachment=
local attachment_command=
local encryption_string=
local auth_string=
if [ ! -f "$attachment" ]; then
attachment_command="-a $attachment"
mail_no_attachment=1
else
mail_no_attachment=0
fi
# Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then
if type iconv > /dev/null 2>&1; then
echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP"
message="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP)"
else
Logger "iconv utility not installed. Will not convert email charset." "NOTICE"
fi
fi
if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then
if [ "$smtpPort" == "" ]; then
Logger "Missing smtp port, assuming 25." "WARN"
smtpPort=25
fi
if type sendmail > /dev/null 2>&1; then
if [ "$encryption" == "tls" ]; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
elif [ "$encryption" == "ssl" ]; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
else
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
fi
if [ $? != 0 ]; then
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
# Don't bother try other mail systems with busybox
return 1
else
return 0
fi
else
Logger "Sendmail not present. Won't send any mail" "WARN"
return 1
fi
fi
if type mutt > /dev/null 2>&1 ; then
# We need to replace spaces with comma in order for mutt to be able to process multiple destinations
echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mutt) !!!" "WARN"
else
Logger "Sent mail using mutt." "NOTICE"
return 0
fi
fi
if type mail > /dev/null 2>&1 ; then
# We need to detect which version of mail is installed
if ! $(type -p mail) -V > /dev/null 2>&1; then
# This may be MacOS mail program
attachment_command=""
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $attachment"
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a$attachment"
else
attachment_command=""
fi
echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
echo "$message" | $(type -p mail) -s "$subject" "$destinationMails"
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN"
else
Logger "Sent mail using mail command without attachment." "NOTICE"
return 0
fi
else
Logger "Sent mail using mail command." "NOTICE"
return 0
fi
fi
if type sendmail > /dev/null 2>&1 ; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destinationMails"
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN"
else
Logger "Sent mail using sendmail command without attachment." "NOTICE"
return 0
fi
fi
# Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$senderMail" == "" ]; then
Logger "Missing sender email." "ERROR"
return 1
fi
if [ "$smtpServer" == "" ]; then
Logger "Missing smtp port." "ERROR"
return 1
fi
if [ "$smtpPort" == "" ]; then
Logger "Missing smtp port, assuming 25." "WARN"
smtpPort=25
fi
if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then
Logger "Bogus smtp encryption, assuming none." "WARN"
encryption_string=
elif [ "$encryption" == "tls" ]; then
encryption_string=-starttls
elif [ "$encryption" == "ssl" ]:; then
encryption_string=-ssl
fi
if [ "$smtpUser" != "" ] && [ "$smtpPassword" != "" ]; then
auth_string="-auth -user \"$smtpUser\" -pass \"$smtpPassword\""
fi
$(type mailsend.exe) -f "$senderMail" -t "$destinationMails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtpServer" -port "$smtpPort" $encryption_string $auth_string
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0
fi
fi
# pfSense specific
if [ -f /usr/local/bin/mail.php ]; then
echo "$message" | /usr/local/bin/mail.php -s="$subject"
if [ $? != 0 ]; then
Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
else
Logger "Sent mail using pfSense mail.php." "NOTICE"
return 0
fi
fi
# If function has not returned 0 yet, assume it is critical that no alert can be sent
Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
}
#### TrapError SUBSET ####
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_LOGGER_SILENT == false ]; then
(>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m")
fi
}
#### TrapError SUBSET END ####
# Quick and dirty performance logger only used for debugging
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
local i #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG
done #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
}
# Checks email address validity
function checkRFC822 {
local mail="${1}"
local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$"
if [[ $mail =~ $rfc822 ]]; then
echo 1
else
echo 0
fi
}
function IsInteger {
local value="${1}"
if [[ $value =~ ^[0-9]+$ ]]; then
echo 1
else
echo 0
fi
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
}
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0
else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0
fi
}
## Main asynchronous execution function
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# Example of improved wait $! emulation
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
# sleep 15 &
# pid=$!
# sleep 20 &
# pid2=$!
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
# Bear in mind that given commands and conditions need to be quoted
## ofunctions.sh subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds
# Wrapper function for ExecTasks
function WaitForPids {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogsAtAll="${11:-false}" # Do not log any errors at all (useful for recursive condition calls)
ExecTasks "$id" "WaitForPids" "$mainInput" "" "" "" "" "" "" "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noTimeErrorLog" "$noErrorLogsAtAll"
return $?
}
# Wrapper function for ExecTasks
function ParallelExec {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
local maxPostponeRetries="${16:-3}" # If command gets postponed because of its condition, how many times do we retry on failed condition
local minTimeBetweenRetries="${17:-300}" # Time between postponed commands retries
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}"
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogsAtAll="${11:-false}" # Do not log any errors at all (useful for recursive condition calls)
ExecTasks "$id" "ParallelExec" "$mainInput" "$auxInput" "$numberOfProcesses" "$maxPostponeRetries" "$minTimeBetweenRetries" "$softPerProcessTIme" "$hardPerProcessTime" "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noTimeErrorLog" "$noErrorLogsAtAll"
return $?
}
function ExecTasks {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}"
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogsAtAll="${11:-false}" # Do not log any errors at all (useful for recursive condition calls)
local execTasksMode="${12:-1}" # In which mode the function should work, see above
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
local maxPostponeRetries="${16:-3}" # If command gets postponed because of its condition, how many times do we retry on failed condition
local minTimeBetweenRetries="${17:-300}" # Time between postponed commands retries
local i
Logger "${FUNCNAME[0]} called in mode [$execTasksMode] by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 13-17 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogsAtAll)
for i in "${booleans[@]}"; do
test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses maxPostponeRetries minTimeBetweenRetries)
for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
fi
# Change '-' to '_' in task id
id="${id/-/_}"
# ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing pids of commands currently run
local postponeIfConditionFails # Boolean to check if command needs to be postponed if condition command fails
local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed
local temp
# Common variables
local pid # Current pid working on
local pidState # State of the process
local mainItemCount=0 # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true)
local counter=0
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
local functionMode
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
case $execTasksMode in
1)
IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount="${#pidsArray[@]}"
readFromFile=false
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 1 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
2)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 2 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
3)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount="${#commandsArray[@]}"
readFromFile=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 3 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
4)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount="${#commandsArray[@]}"
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount="${#commandsConditionArray[@]}"
readFromFile=false
postponeIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 4 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
5)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount="${#commandsArray[@]}"
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount="${#commandsConditionArray[@]}"
readFromFile=false
postponeIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 5 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
6)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 6 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
7)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxItemCount=$(wc -l < "$auxInput")
else
Logger "Cannot read file [$auxInput]." "WARN"
fi
postponeIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 7 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
8)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxItemCount=$(wc -l < "$auxInput")
else
Logger "Cannot read file [$auxInput]." "WARN"
fi
postponeIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 8 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
*)
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
exit 1
esac
function _ExecTasksTimeCheck {
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$((SECONDS - seconds_begin))
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
softAlert=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
if [ $noErrorLogsAtAll != true ]; then
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
done
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount
else
return 129
fi
#elif [ $functionMode == "ParallelExec" ]; then
# # Be sure to return an error
# if [ $((mainItemCount - counter + ${#pidsArray[@]})) -ne 0 ]; then
# return $((mainItemCount - counter + ${#pidsArray[@]}))
# else
# return 1
# fi
#fi
fi
}
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do
_ExecTasksTimeCheck
retval=$?
if [ $retval -ne 0 ]; then
return $retval;
fi
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do
_ExecTasksTimeCheck
retval=$?
if [ $retval -ne 0 ]; then
return $retval;
fi
executeCommand=false
isPostponedCommand=false
currentCommand=""
currentCommandCondition=""
needsPostponing=false
if [ $readFromFile == true ]; then
# awk identifies first line as 1 instead of 0 so we need to increase counter
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
fi
# Check if we need to fetch postponed commands
if [ "$currentCommand" == "" ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP")
isPostponedCommand=true
fi
else
currentCommand="${commandsArray[$counter]}"
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition="${commandsConditionArray[$counter]}"
fi
if [ "$currentCommand" == "" ]; then
currentCommand="${postponedCommandsArray[$postponedCounter]}"
currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}"
isPostponedCommand=true
fi
fi
# Check if we execute postponed commands, or if we delay them
if [ $isPostponedCommand == true ]; then
# Get first value before '@'
postponedExecTime="${currentCommand%%@*}"
postponedExecTime=$((SECONDS-postponedExecTime))
# Get everything after first '@'
#WIP temp is not beautiful
temp="${currentCommand#*@}"
# Get first value before '@'
postponedRetryCount="${temp%%@*}"
# Replace currentCommand with actual filtered currentCommand
currentCommand="${temp#*@}"
# Since we read a postponed command, we may decrase postponedItemCounter
postponedItemCount=$((postponedItemCount-1))
#Since we read one line, we need to increase the counter
postponedCounter=$((postponedCounter+1))
else
postponedRetryCount=0
postponedExecTime=0
fi
if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $((minTimeBetweenRetries)) ]) || [ $isPostponedCommand == false ]; then
if [ "$currentCommandCondition" != "" ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
subRetval=$?
if [ $subRetval -ne 0 ]; then
if [ $postponeIfConditionFails == true ]; then
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE"
postponedRetryCount=$((postponedRetryCount+1))
if [ $postponedRetryCount -ge $maxPostponeRetries ]; then
Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE"
else
needsPostponing=true
fi
postponedExecTime=0
else
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE"
fi
else
executeCommand=true
fi
else
executeCommand=true
fi
else
needsPostponing=true
fi
if [ $needsPostponing == true ]; then
postponedItemCount=$((postponedItemCount+1))
if [ $readFromFile == true ]; then
echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP"
echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP"
else
postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand")
postponedCommandsConditionArray+=("$currentCommandCondition")
fi
fi
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$currentCommand"
# Initialize pid execution time array
pidsTimeArray[$pid]=0
else
Logger "Skipping command [$currentCommand]." "DEBUG"
fi
if [ $isPostponedCommand == false ]; then
counter=$((counter+1))
fi
_ExecTasksPidsCheck
done
fi
function _ExecTasksPidsCheck {
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
# Check if pid hasn't run more than soft/hard perProcessTime
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then
if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for pid [$pid] (optional command [${commandsArrayPid[$pid]}])." "WARN"
softAlert=true
SendAlert true
fi
fi
if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then
if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then
Logger "Max hard execution time exceeded for pid [$pid] (optional command [${commandsArrayPid[$pid]}]). Stopping command execution." "ERROR"
fi
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Command with pid [$pid] stopped successfully." "NOTICE"
else
if [ $noErrorLogsAtAll != true ]; then
Logger "Could not stop command with pid [$pid]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
fi
newPidsArray+=($pid)
fi
else
# pid is dead, get its exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
else
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
fi
fi
hasPids=true ##__WITH_PARANOIA_DEBUG
fi
done
# hasPids can be false on last iteration in ParallelExec mode
if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
}
_ExecTasksPidsCheck
done
Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $mainItemCount -eq 1 ]; then
return $retval
else
return $errorcount
fi
}
function CleanUp {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
############################################ END OF OFUNCTIONS CODE
function TrapQuit {
local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
fi
if [ $ERROR_ALERT == true ]; then
Logger "$PROGRAM finished with errors." "ERROR"
if [ "$_DEBUG" != "yes" ]
then
SendAlert
else
Logger "Debug mode, no alert mail will be sent." "NOTICE"
fi
exitcode=1
elif [ $WARN_ALERT == true ]; then
Logger "$PROGRAM finished with warnings." "WARN"
if [ "$_DEBUG" != "yes" ]
then
SendAlert
else
Logger "Debug mode, no alert mail will be sent." "NOTICE"
fi
exitcode=2 # Warning exit code must not force daemon mode to quit
else
Logger "$PROGRAM finished." "ALWAYS"
exitcode=0
fi
CleanUp
KillChilds $SCRIPT_PID > /dev/null 2>&1
exit $exitcode
}
# Takes as many file arguments as needed
function InterleaveFiles {
local counter=0
local hasLine=true
local i
while [ $hasLine == true ]; do
hasLine=false
for i in "$@"; do
line=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$i")
if [ -n "$line" ]; then
echo "$line"
hasLine=true
fi
done
counter=$((counter+1))
done
}
function ListClients {
local backupDir="${1}"
local configFile="${2}"
local clientConfDir="${3}"
__CheckArguments 1-2 $# "$@" #__WITH_PARANOIA_DEBUG
local clientIsIncluded
local clientIsExcluded
local excludeArray
local client
local clientEmail
local configString
local i
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
if [ "$clientConfDir" == "" ]; then
clientConfDir="/etc/burp/clientconfdir"
fi
# File 'backup_stats' is there only when a backup is finished
find "$backupDir" -mindepth 3 -maxdepth 3 -type f -name "backup_stats" | grep -e '.*' > /dev/null
if [ $? != 0 ]; then
Logger "The directory [$backupDir] does not seem to be a burp folder. Please check the path. Additionnaly, protocol 2 directores need to specify the dedup group directory and the client subfolder." "ERROR"
fi
# Using both burp -a S list and find method in order to find maximum backup clients
cmd="$BACKUP_EXECUTABLE $configString -a S | grep \"last backup\" | awk '{print \$1}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP\""
Logger "Running cmd [$cmd]." "DEBUG"
eval "$cmd" &
ExecTasks "${FUNCNAME[0]}_lastbackup" 0 0 1800 3600 1 $KEEP_LOGGING true true false false 1 $!
if [ $? != 0 ]; then
Logger "Enumerating burp clients via [$BACKUP_EXECUTABLE $configString -a S] failed." "ERROR"
else
Logger "Burp method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP)" "DEBUG"
fi
#TODO: sed expressions are GNU and won't probably work on BSD nor Mac
# First exp removes everything before last '/'
find "$backupDir" -mindepth 1 -maxdepth 1 -type d | sed -e "s/\(.*\)\/\(.*\)/\2/g" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP"
while IFS=$'\n' read -r client; do
find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | grep -e '.*' > /dev/null
if [ $? == 0 ]; then
echo "$client" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
fi
done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP"
if [ ! -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" ]; then
touch "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
fi
Logger "Detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG"
# Merge all clients found by burp executable and manual check
sort "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" | uniq > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
while IFS=$'\n' read -r client; do
clientIsIncluded=false
clientIsExcluded=false
IFS=',' read -a includeArray <<< "$INCLUDE_CLIENTS"
for i in "${includeArray[@]}"; do
echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1
if [ $? == 0 ]; then
clientIsIncluded=true
fi
done
IFS=',' read -a excludeArray <<< "$EXCLUDE_CLIENTS"
for i in "${excludeArray[@]}"; do
echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1
if [ $? == 0 ]; then
clientIsExcluded=true
fi
done
if ([ $clientIsIncluded == false ] && [ $clientIsExcluded == true ]); then
Logger "Ommiting client [$client]." "NOTICE"
else
if [ -f "$backupDir$client/current/timestamp" ]; then
Logger "Found client [$client]." "NOTICE"
CLIENT_LIST+=("$client")
# Client email is a label in client config file
# Check whether we can fetch the default value
if [ -f "$clientConfDir/$client" ]; then
clientEmail=$(egrep "^label( )?=( )?email_address( )?:( )?" "$clientConfDir/$client")
if [ "$clientEmail" != "" ]; then
clientEmail="${clientEmail#*:}"
# Remove eventual spaces
clientEmail="${clientEmail/ /}"
if [ $(checkRFC822 "$clientEmail") -eq 1 ]; then
CLIENT_EMAIL["$client"]="$clientEmail"
else
Logger "Client [$client] has a bogus mail address [$clientEmail]." "WARN"
fi
else
Logger "Client [$client] has no mail address set." "NOTICE"
fi
elif [ "$CLIENT_ALERTS" == true ]; then
Logger "Cannot find client config file [$clientConfDir/$client] to fetch email address." "ERROR"
fi
else
Logger "Client [$client] does not have any backups." "WARN"
fi
fi
done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
}
function IsClientIdle {
local client="${1}"
local configFile="${2}"
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local exitCode
local configString
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
Logger "Checking if client [$client] is currently idle." "DEBUG"
cmd="$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1"
ExecTasks "${FUNCNAME[0]}_idle" 0 0 120 300 1 $KEEP_LOGGING true true false false 1 $!
exitCode=$?
if [ $exitCode -ne 0 ]; then
Logger "Client [$client] is currently backing up." "NOTICE"
return $exitCode
else
return $exitCode
fi
}
function VerifyBackups {
local backupDir="${1}"
local numberToVerify="${2}"
local configFile="${3}"
__CheckArguments 2-3 $# "$@" #__WITH_PARANOIA_DEBUG
local backupNumber
local exitCode
local client
local configString
local interleaveFileArgs=()
local interleaveConditionsFileArgs=()
Logger "Running backup verification, concurrency set to $PARELLEL_VERIFY_CONCURRENCY." "NOTICE"
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
for client in "${CLIENT_LIST[@]}"; do
# Only backups containing file backup_stats are valid
find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | sort -nr | head -n $numberToVerify | sed -e 's/.*\([0-9]\{7\}\).*/\1/' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP"
Logger "Can check $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP | wc -l) backups for [$client]." "NOTICE"
while IFS=$'\n' read -r backupNumber; do
#TODO: sed expressions won't probably work on BSD nor Mac
# sed here removes all lines containing only block logs (64 chars + number)
Logger "Preparing verification of backup [$backupNumber] for client [$client]." "NOTICE"
echo "$BACKUP_EXECUTABLE $configString -C $client -a v -b $backupNumber | sed '/^[BbfFvud]\{64\} [0-9]\+$/d' >> \"$LOG_FILE\" 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP"
echo "$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP"
done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP" ]; then
interleaveFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP")
fi
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP" ]; then
interleaveConditionsFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP")
fi
done
InterleaveFiles "${interleaveFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
InterleaveFiles "${interleaveConditionsFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP"
Logger "Executing parallel commands\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG"
if [ "$POSTPONE" != "no" ]; then
ExecTasks "${FUNCNAME[0]}" $SOFT_MAX_EXEC_TIME_PER_VERIFY $HARD_MAX_EXEC_TIME_PER_VERIFY $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1 $KEEP_LOGGING true true false false 8 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP" $PARELLEL_VERIFY_CONCURRENCY $POSTPONE_RETRY $POSTPONE_TIME
else
ExecTasks "${FUNCNAME[0]}" $SOFT_MAX_EXEC_TIME_PER_VERIFY $HARD_MAX_EXEC_TIME_PER_VERIFY $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1 $KEEP_LOGGING true true false false 6 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" "" $PARELLEL_VERIFY_CONCURRENCY
fi
exitCode=$?
if [ $exitCode -ne 0 ]; then
Logger "Client backup verification produced errors [$exitCode]." "ERROR"
else
Logger "Client backup verification succeed." "NOTICE"
fi
Logger "Backup verification done." "NOTICE"
}
function ListOutdatedClients {
local backupDir="${1}"
local oldDays="${2}"
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local found=false
local clientAlertBody
Logger "Checking for outdated clients." "NOTICE"
for client in "${CLIENT_LIST[@]}"; do
recentBackups=$(find "$backupDir$client" -maxdepth 2 -name "backup_stats" -and ! -path "*working*" -ctime -$oldDays | wc -l)
if [ $recentBackups -le 0 ]; then
Logger "Client [$client] has no backups newer than [$oldDays] days." "ERROR"
if [ $CLIENTS_ALERT_OUTDATED == true ]; then
clientAlertBody="${CLIENT_ALERT_BODY_OUTDATED/"[CLIENT]"/$client}"
clientAlertBody="${clientAlertBody/"[NUMBERDAYS]"/$oldDays}"
if [ "${CLIENT_EMAIL[$client]}" != "" ]; then
SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}"
Logger "Sent outdated client mail to [${CLIENT_EMAIL[$client]}]." "NOTICE"
else
Logger "Client [$client] does not have a mail address. Cannot send notification." "ERROR"
fi
fi
found=true
fi
done
if [ $found == false ]; then
Logger "No outdated clients found." "NOTICE"
else
Logger "Outdated client checks done." "NOTICE"
fi
}
function ListQuotaExceedClients {
local backupDir="${1}"
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
local found=false
local lastBackupDir
local clientAlertBody
local bytesEstimated
local quotaExceed
local quotaDiff
Logger "Checking for clients with exceeded quota." "NOTICE"
for client in "${CLIENT_LIST[@]}"; do
lastBackupDir=$(find "$backupDir$client" -maxdepth 2 -name "*current")
if zcat "$lastBackupDir/log.gz" | grep quota > /dev/null 2>&1; then
bytesEstimated=$(zcat "$lastBackupDir/log.gz" | grep "Bytes estimated")
bytesEstimated="${bytesEstimated##*:}"
# Remove leading spaces
bytesEstimated="${bytesEstimated# *}"
quotaExceed=$(zcat "$lastBackupDir/log.gz" | grep "quota")
quotaExceed="${quotaExceed##*:}"
quotaDiff="$bytesEstimated /$quotaExceed)"
Logger "Client [$client] quota exceed ($quotaDiff)." "WARN"
if [ $CLIENTS_ALERT_QUOTAS == true ]; then
clientAlertBody="${CLIENT_ALERT_BODY_QUOTA/"[CLIENT]"/$client}"
clientAlertBody="${clientAlertBody/"[QUOTAEXCEED]"/$quotaDiff}"
if [ "${CLIENT_EMAIL[$client]}" != "" ]; then
SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}"
Logger "Sent quota exceeded mail to [${CLIENT_EMAIL[$client]}]." "NOTICE"
else
Logger "Client [$client] does not have a mail address. Cannot send notification" "ERROR"
fi
fi
found=true
fi
done
if [ $found == false ]; then
Logger "No clients with exceeded quota found." "NOTICE"
else
Logger "Quota checks done." "NOTICE"
fi
}
function VerifyLastWarnings {
local backupDir="${1}"
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
local found=false
Logger "Checking for warnings in last backups." "NOTICE"
for client in "${CLIENT_LIST[@]}"; do
if [ -f "$backupDir$client/current/log.gz" ]; then
if zcat "$backupDir$client/current/log.gz" | grep "WARNING" > /dev/null 2>&1; then
Logger "Client [$client] has the following warnings:" "WARN"
Logger "$(zcat $backupDir$client/current/log.gz | grep WARNING)" "WARN"
found=true
fi
elif [ -f "$backupDir$client/current/log" ]; then
if cat "$backupDir$client/current/log" | grep "WARNING" > /dev/null 2>&1; then
Logger "Client [$client] has the following warnings:" "WARN"
Logger "$(grep WARNING $backupDir$client/current/log)" "WARN"
found=true
fi
else
Logger "No log file found for analysis in [$backupDir$client/current]." "WARN"
fi
done
if [ $found == false ]; then
Logger "No warnings found in last backups." "NOTICE"
fi
}
function UnstripVSS {
local path="${1}"
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
# We need to have a modular temp extension so we will not overwrite potential existing files
local tempExtension="$SCRIPT_PID.$TSTAMP.old"
if ! type vss_strip > /dev/null 2>&1; then
Logger "Could not find vss_strip binary. Please check your path variable." "CRITICAL"
exit 1
fi
find "$path" -type f -print0 | while IFS= read -r -d $'\0' file; do
Logger "Unstripping file [$file]." "NOTICE"
mv -f "$file" "$file.$tempExtension"
if [ $? -ne 0 ]; then
Logger "Could not move [$file] to [$file.$tempExtension] for processing." "WARN"
continue
else
vss_strip -i "$file.$tempExtension" -o "$file"
if [ $? -ne 0 ]; then
Logger "Could not vss_strip [$file.$tempExtension] to [$file]." "WARN"
mv -f "$file.$tempExtension" "$file"
if [ $? -ne 0 ]; then
Logger "Coult not move back [$file.$tempExtension] to [$file]." "WARN"
fi
else
rm -f "$file.$tempExtension"
if [ $? -ne 0 ]; then
Logger "Could not delete temporary file [$file.$tempExtension]." "WARN"
continue
fi
fi
fi
done
# Cannot get exitcode since find uses a subshell. Getting exit code from Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
return 2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
return 1
fi
return 0
}
function VerifyService {
local serviceName="${1}"
local serviceType="${2}"
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local serviceNameArray
local serviceStatusCommand
local serviceStartCommand
local i
IFS=',' read -a serviceNameArray <<< "$serviceName"
for i in "${serviceNameArray[@]}"; do
if [ "$serviceType" == "initv" ]; then
serviceStatusCommand="service $i status"
serviceStartCommand="service $i start"
elif [ "$serviceType" == "systemd" ]; then
serviceStatusCommand="systemctl status $i"
serviceStartCommand="systemctl start $i"
else
serviceStatusCommand="service $i status"
serviceStartCommand="systemctl start $i"
Logger "No valid service type given [$serviceType]. Trying default initV style." "ERROR"
fi
eval "$serviceStatusCommand" > /dev/null 2>&1 &
ExecTasks "${FUNCNAME[0]}_statuscmd" 0 0 120 300 1 $KEEP_LOGGING true true false false 1 $!
if [ $? -ne 0 ]; then
Logger "Service [$i] is not started. Trying to start it." "WARN"
eval "$serviceStartCommand" > /dev/null 2>&1 &
ExecTasks "${FUNCNAME[0]}_startcmd" 0 0 120 300 1 $KEEP_LOGGING true true false false 1 $!
if [ $? -ne 0 ]; then
Logger "Cannot start service [$i]." "CRITICAL"
SendAlert
else
Logger "Service [$i] was successfuly started." "WARN"
SendAlert
fi
else
Logger "Service [$i] is running." "NOTICE"
fi
done
}
function Init {
# Set error exit code if a piped command fails
set -o pipefail
set -o errtrace
trap TrapQuit TERM EXIT HUP QUIT
}
function Usage {
if [ "$IS_STABLE" != "yes" ]; then
echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m"
fi
echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD"
echo "$AUTHOR"
echo "$CONTACT"
echo ""
echo "Usage:"
echo "$0 [OPTIONS]"
echo ""
echo "[OPTIONS]"
echo "-d, --backup-dir=\"\" The directory where the client backup directories are"
echo "-o, --check-outdated-clients=n Check for clients that don't have backups newer than n days"
echo "-v, --verify-last-backups=n Verify the last n backups of all clients"
echo "-q, --verify-quotas Check for client quotas"
echo "-i, --include-clients=\"\" Comma separated list of clients to include. This list takes grep -e compatible regular expressions, includes prevail excludes"
echo "-e, --exclude-clients=\"\" Comma separated list of clients to exclude. This list takes grep -e compatible regular expressions"
echo "-c, --config-file=\"\" Path to optional burp client configuration file (defaults to /etc/burp/burp.conf)"
echo "-s, --vss-strip-path=\"\" Run vss_strip for all files in given path"
echo "-j, --verify-service=\"\" Comma separated list of burp services to check and restart if they aren't running"
echo "-w, --verify-warnings Check for warnings in last backup logs"
echo "-A, --clients-alert-quotas Use email defined in client config to alert users about exceeded disk quotas (see email template in header)"
echo "-a, --clients-alert-outdated Use email defined in client config to alert users about outdated backups (see email template in header)"
echo "-z, --clientconfdir=\"\" Path of clientconfdir in order to fetch email addresses when client alerts are used (defaults to /etc/burp/clientconfdir)"
echo ""
echo "Examples:"
echo "$0 -d /path/to/burp/protocol1 -v 3 -c /etc/burp/burp.conf"
echo "$0 -d /path/to/burp/protocol2/global/clients --check-outdated-clients7 --exclude-clients=restoreclient,burp-ui.local"
echo "$0 --vss-strip-path=/path/to/restored/files"
echo "$0 -j burp.service"
echo "Exclude via regex all clients beginning with 'cli' and otherclient1/2:"
echo "$0 --backup-dir=/path/to/burp/protocol1 --exclude-clients=cli.*,otherclient1,otherclient2"
echo ""
echo "Additionnal options"
echo "--no-maxtime Don't stop checks after the configured maximal time in script"
echo "-s, --silent Don't output to stdout, log file only"
echo "--errors-only Don't output anything but errors."
echo ""
echo "--destination-mails=\"\" Space separated list of email adresses where to send warning and error mails"
echo "--instance-id=\"\" Arbitrary identifier for log files and alert mails"
exit 128
}
#### SCRIPT ENTRY POINT
DESTINATION_MAILS=""
no_maxtime=false
ERROR_ALERT=false
WARN_ALERT=false
CONFIG_FILE=""
BACKUP_DIR=""
VERIFY_BACKUPS=""
INCLUDE_CLIENTS=""
EXCLUDE_CLIENTS=""
OUTDATED_DAYS=""
CLIENT_LIST=()
declare -A CLIENT_EMAIL
VSS_STRIP_DIR=""
VERIFY_SERVICE=""
VERIFY_WARNINGS=false
VERIFY_QUOTAS=false
CLIENTS_ALERT_QUOTAS=false
CLIENTS_ALERT_OUTDATED=false
CLIENT_CONF_DIR=""
function GetCommandlineArguments {
local isFirstArgument=true
if [ $# -eq 0 ]
then
Usage
fi
while [ $# -gt 0 ]; do
## Options name is $1, argument is $2 unless there is a separator other than space
case $1 in
--instance-id=*)
INSTANCE_ID="${1##*=}"
;;
--silent)
_LOGGER_SILENT=true
;;
--verbose)
_LOGGER_VERBOSE=true
;;
--no-maxtime)
no_maxtime=true
;;
--help|-h|--version)
Usage
;;
--backup-dir=*)
BACKUP_DIR="${1##*=}"
;;
-d)
BACKUP_DIR="${2}"
shift
;;
--check-outdated-clients=*)
OUTDATED_DAYS="${1##*=}"
;;
-o)
OUTDATED_DAYS="${2}"
shift
;;
--verify-last-backups=*)
VERIFY_BACKUPS="${1##*=}"
;;
-v)
VERIFY_BACKUPS="${2}"
shift
;;
-q|--verify-quotas)
VERIFY_QUOTAS=true
;;
--include-clients=*)
INCLUDE_CLIENTS="${1##*=}"
;;
-i)
INCLUDE_CLIENTS="${2}"
shift
;;
--exclude-clients=*)
EXCLUDE_CLIENTS="${1##*=}"
;;
-e)
EXCLUDE_CLIENTS="${2}"
shift
;;
--config-file=*)
CONFIG_FILE="${1##*=}"
;;
-c)
CONFIG_FILE="${2}"
shift
;;
--vss-strip-path=*)
VSS_STRIP_DIR="${1##*=}"
;;
-s)
VSS_STRIP_DIR="${2}"
shift
;;
-j)
VERIFY_SERVICE="${2}"
shift
;;
-A|--client-alert-quotas)
CLIENTS_ALERT_QUOTAS=true
;;
-a|--client-alert-outdated)
CLIENTS_ALERT_OUTDATED=true
;;
-z)
CLIENT_CONF_DIR="${2}"
shift
;;
--clientconfdir=*)
CLIENT_CONF_DIR="${1##*=}"
;;
--verify-service=*)
VERIFY_SERVICE="${1##*=}"
;;
-w|--verify-warnings)
VERIFY_WARNINGS=true
;;
--errors-only)
_LOGGER_ERR_ONLY=true
;;
--destination-mails=*)
DESTINATION_MAILS="${1##*=}"
;;
--no-maxtime)
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
;;
*)
if [ $isFirstArgument == false ]; then
Logger "Unknown option '${1}'" "CRITICAL"
Usage
fi
;;
esac
shift
isFirstArgument=false
done
}
GetCommandlineArguments "$@"
Init
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
DATE=$(date)
Logger "---------------------------------------------------------------------" "NOTICE"
Logger "$DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS"
Logger "---------------------------------------------------------------------" "NOTICE"
Logger "Instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
if ! type -p "$BACKUP_EXECUTABLE" > /dev/null 2>&1; then
Logger "Cannot find [$BACKUP_EXECUTABLE]. Please modify binary path in $0 script header." "CRITICAL"
exit 126
fi
if [ "$VSS_STRIP_DIR" != "" ]; then
if [ -d "$VSS_STRIP_DIR" ]; then
UnstripVSS "$VSS_STRIP_DIR"
exit $?
else
Logger "Bogus path given to unstrip [$VSS_STRIP_DIR]." "CRITICAL"
exit 1
fi
fi
if [ "$VERIFY_SERVICE" != "" ]; then
VerifyService "$VERIFY_SERVICE" "$SERVICE_TYPE"
fi
if [ "$BACKUP_DIR" != "" ]; then
if [ ! -d "$BACKUP_DIR" ]; then
Logger "Backup dir [$BACKUP_DIR] doesn't exist." "CRITICAL"
exit 1
else
# Make sure there is only one trailing slash on path
BACKUP_DIR="${BACKUP_DIR%/}/"
fi
if [ "$CONFIG_FILE" != "" ]; then
if [ ! -f "$CONFIG_FILE" ]; then
Logger "Bogus configuration file [$CONFIG_FILE] given." "CRITICAL"
exit 1
fi
fi
if [ "$CLIENT_CONF_DIR" != "" ]; then
if [ ! -d "$CLIENT_CONF_DIR" ]; then
Logger "Bogus clientconfdir [$CLIENT_CONF_DIR] given." "CRITICAL"
exit 1
fi
fi
ListClients "$BACKUP_DIR" "$CONFIG_FILE" "$CLIENT_CONF_DIR"
if [ $VERIFY_WARNINGS == true ]; then
VerifyLastWarnings "$BACKUP_DIR"
fi
if [ $VERIFY_QUOTAS == true ]; then
ListQuotaExceedClients "$BACKUP_DIR"
fi
if [ "$OUTDATED_DAYS" != "" ]; then
if [ $(IsInteger "$OUTDATED_DAYS") -ne 0 ]; then
ListOutdatedClients "$BACKUP_DIR" $OUTDATED_DAYS
else
Logger "Bogus --check-outdated-clients value [$OUTDATED_DAYS]." "CRITICAL"
exit 1
fi
fi
if [ "$VERIFY_BACKUPS" != "" ]; then
if [ $(IsInteger "$VERIFY_BACKUPS") -ne 0 ]; then
VerifyBackups "$BACKUP_DIR" $VERIFY_BACKUPS "$CONFIG_FILE"
else
Logger "Bogus --verify-last-backups value [$VERIFY_BACKUPS]." "CRITICAL"
exit 1
fi
fi
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment