Last active
February 9, 2018 04:16
-
-
Save hdoverobinson/5567eaa776445972967cd03a4dfbc9b4 to your computer and use it in GitHub Desktop.
Start, stop, and back up Open Satellite Project goesdump and xritdemod
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
###AUTHOR### | |
#Harry Dove-Robinson 2018-02-08 | |
#harry@doverobinson.me | |
#https://gist.github.com/hdoverobinson | |
#https://github.com/hdoverobinson | |
###NOTES### | |
#Used by wx-star.com for 24/7 operation of Open Satellite Project software for HRIT/EMWIN direct readout | |
#Manages OSP processes inside GNU Screen sessions, performs daily backups and rotation of OSP output data | |
#Packages installed: cpulimit, diffutils, pigz, screen | |
#It is recommended to set the system timezone to UTC | |
###USAGE### | |
#When this script is run at least once per 24 hours: | |
#-OSP processes will be restarted | |
#-Up to 48 hours of OSP output data will be stored ephemerally in $OUTPUT_DIR | |
#-Exactly 24 hours of OSP output data created between midnight today and midnight yesterday will be backed up as tar.gz in $ARCHIVE_DIR/$YESTERDAY_YEAR | |
#Accepts arguments "stop", "rotate", and "start". When no arguments are provided the script will run all three functions in that order. Rotation will be performed in background to minimize downtime and after | |
#a waiting period so OSP can finish writing its output from the previous day | |
#Accepts options "hard" and "verbose" to send SIGKILL to processes instead of SIGINT, or to send all command output to stdout via "set -x", respectively | |
#Examples: | |
#"./osp-handler" to stop all OSP processes, attempt to backup and rotate OSP output data, and start OSP processes again | |
#"./osp-handler stop hard verbose" to only send SIGKILL to all OSP processes and show output verbosely | |
#"./osp-handler rotate" to only attempt to backup and rotate OSP output data with minimal output to stdout | |
#"0 */4 * * * root /bin/bash -c '/root/osp-handler.sh > /var/log/osp-handler.log 2>&1'" in a cron.d file to run all functions every 4 hours and store output to logfile | |
#where daily tar.gz archives will be stored | |
ARCHIVE_DIR="/media/data/archive" | |
#where xritdemod binaries are located | |
XRITDEMOD_DIR="/root/osp-build/bin/xritdemod" | |
#where goesdump binaries are located | |
GOESDUMP_DIR="/root/osp-build/bin/goesdump" | |
#goesdump output directory | |
OUTPUT_DIR="$GOESDUMP_DIR/output" | |
#number of days to retain tar.gz archives | |
ARCHIVE_HOLD="365" | |
#total CPU percentage to limit backup processes | |
CPU_LIMIT="100" | |
#time to wait for OSP to finish writing yesterday's files before attempting a backup | |
WAIT_TIME="30m" | |
#time between process stop and start attempts | |
SLEEP_TIME="1s" | |
cpulimit_wait () | |
{ | |
#force a wait for cpulimit processes to close before proceeding | |
PID=$(ps -eo pid,comm,args | grep -w "cpulimit" | grep -v "grep" | awk '$2 == "cpulimit"' | grep "$1" | awk '{print $1}') && | |
while kill -0 $PID > /dev/null 2>&1 | |
do | |
sleep "$SLEEP_TIME" | |
done && | |
wait | |
} | |
stop () | |
{ | |
echo "$(tput setaf 2 2> /dev/null)Stopping OSP...$(tput sgr0 2> /dev/null)" && | |
GOESDUMP=$(ps -eo pid,comm,args | grep -w "./goesdump" | grep -v "grep" | awk '$2 == "goesdump"' | awk '{print $1}') && | |
XRITDECODER=$(ps -eo pid,comm,args | grep -w "./xritDecoder" | grep -v "grep" | awk '$2 == "xritDecoder"' | awk '{print $1}') && | |
XRITDEMODULATOR=$(ps -eo pid,comm,args | grep -w "./xritDemodulator" | grep -v "grep" | awk '$2 == "xritDemodulator"' | awk '{print $1}') && | |
while kill -0 $GOESDUMP > /dev/null 2>&1 | |
do | |
kill -$SIG $GOESDUMP > /dev/null 2>&1 && sleep "$SLEEP_TIME" | |
done && | |
while kill -0 $XRITDECODER > /dev/null 2>&1 | |
do | |
kill -$SIG $XRITDECODER > /dev/null 2>&1 && sleep "$SLEEP_TIME" | |
done && | |
while kill -0 $XRITDEMODULATOR > /dev/null 2>&1 | |
do | |
kill -$SIG $XRITDEMODULATOR > /dev/null 2>&1 && sleep "$SLEEP_TIME" | |
done && | |
sync && | |
echo 3 > /proc/sys/vm/drop_caches && | |
if [[ $(screen -ls | wc -l) -gt 2 ]] | |
then | |
screen -wipe > /dev/null 2>&1 | |
fi && | |
wait | |
} | |
rotate () | |
{ | |
TODAY="$(date -u '+%F' --date="today")" && | |
YESTERDAY="$(date -u '+%F' --date="yesterday")" && | |
YESTERDAY_YEAR="$(date -u '+%Y' --date="yesterday")" && | |
YESTERDAY_MONTH="${YESTERDAY_YEAR}-$(date -u '+%m' --date="yesterday")" && | |
MAX_ARCHIVE_DAY="$(date -u '+%F' --date="today -$ARCHIVE_HOLD days")" && | |
if [[ -d "$OUTPUT_DIR" ]] && ! [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" ]] | |
then | |
echo "$(tput setaf 2 2> /dev/null)Waiting for $WAIT_TIME before starting backup...$(tput sgr0 2> /dev/null)" && | |
sleep "$WAIT_TIME" && | |
echo "$(tput setaf 2 2> /dev/null)Archiving files in $OUTPUT_DIR from $YESTERDAY to $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz...$(tput sgr0 2> /dev/null)" && | |
mkdir -p "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY" && | |
FIND_CMD="find "$OUTPUT_DIR" \( -newermt "$YESTERDAY" ! -newermt "$TODAY" \) -type f ! -name '*dup*' ! -name '*.zip' ! \( -path '*/EMWIN/*' -name '*.lrit' \)" && | |
#multithreaded tar.gz backup of files from $FIND_CMD, print $FIND_CMD output to list | |
tar -cf - -C "$(realpath "$OUTPUT_DIR/../")" --files-from=<(eval "$FIND_CMD 2> /dev/null" | sed "s@$(realpath "$OUTPUT_DIR/../")/@@g" | tee "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list") | cpulimit -q -l "$CPU_LIMIT" -- pigz > "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" && | |
cpulimit_wait pigz && | |
#compare tar.gz contents to files in $OUTPUT_DIR, print matches to list | |
tar -dzvf "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" -C "$(realpath "$OUTPUT_DIR/../")" > "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" && | |
#make sha256 checksum of tar.gz backup | |
cpulimit -q -l "$CPU_LIMIT" -- sha256sum "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" > "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.sha256" && | |
cpulimit_wait sha256sum && | |
#check if all backup files are present, compare list of source files to list of files inside the archive | |
if [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" ]] && [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" ]] && \ | |
[[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" ]] && [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.sha256" ]] && \ | |
SRC_FILES="$(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" | wc -l)" && \ | |
BACK_FILES="$(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" | wc -l)" && \ | |
#filename comparison excludes EMWIN because of possible bug in tar where underscores in long EMWIN filenames were sometimes printed as \022 but were printed normally in source | |
cmp -s <(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" | grep -v '/EMWIN/' | sort -n) <(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" | grep -v '/EMWIN/' | sort -n) && \ | |
[[ "$SRC_FILES" -gt 0 ]] && [[ "$BACK_FILES" -gt 0 ]] && [[ "$SRC_FILES" -eq "$BACK_FILES" ]] | |
then | |
#delete source file list and old data if archive valid | |
rm "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" && | |
echo "$(tput setaf 2 2> /dev/null)Archive $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz complete!$(tput sgr0 2> /dev/null)" && | |
echo "$(tput setaf 2 2> /dev/null)Deleting files from $OUTPUT_DIR older than $YESTERDAY...$(tput sgr0 2> /dev/null)" && | |
find "$OUTPUT_DIR" ! -newermt "$YESTERDAY" -type f -delete && | |
echo "$(tput setaf 2 2> /dev/null)Deleting archives from $ARCHIVE_DIR/$YESTERDAY_YEAR older than $MAX_ARCHIVE_DAY...$(tput sgr0 2> /dev/null)" && | |
find "$ARCHIVE_DIR" ! -newermt "$MAX_ARCHIVE_DAY" -type f -delete && | |
echo "$(tput setaf 2 2> /dev/null)Archive and data rotation complete!$(tput sgr0 2> /dev/null)" | |
else | |
#delete attempted archive files if failed | |
echo "$(tput setaf 1 2> /dev/null)Archive failed: $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.$(tput sgr0 2> /dev/null)" && | |
if [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" ]] | |
then | |
echo "$(tput setaf 1 2> /dev/null)Deleting failed archive: $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.$(tput sgr0 2> /dev/null)" && | |
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" && | |
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" && | |
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" && | |
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.sha256" | |
fi && | |
wait && | |
exit 1 | |
fi | |
else | |
echo "$(tput setaf 3 2> /dev/null)Archive for $YESTERDAY already exists at $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.$(tput sgr0 2> /dev/null)" | |
fi | |
} | |
start () | |
{ | |
echo "$(tput setaf 2 2> /dev/null)Starting OSP...$(tput sgr0 2> /dev/null)" && | |
screen -dmS xritDemodulator bash -c "cd $XRITDEMOD_DIR/ && ./xritDemodulator" && | |
sleep "$SLEEP_TIME" && | |
screen -dmS xritDecoder bash -c "cd $XRITDEMOD_DIR/ && ./xritDecoder" && | |
sleep "$SLEEP_TIME" && | |
screen -dmS goesdump bash -c "cd $GOESDUMP_DIR/ && ./goesdump" | |
} | |
echo "$(tput setaf 2 2> /dev/null)$(date -u '+%F %R:%S UTC') Starting $(basename "$0")$(tput sgr0 2> /dev/null)" && | |
if ! command -v cmp > /dev/null 2>&1 || ! command -v cpulimit > /dev/null 2>&1 || ! command -v pigz > /dev/null 2>&1 || ! command -v screen > /dev/null 2>&1 | |
then | |
echo "$(tput setaf 2 2> /dev/null)Installing requisite packages...$(tput sgr0 2> /dev/null)" && | |
apt-get -qq update && | |
apt-get -qq install cpulimit diffutils pigz screen | |
fi && | |
if [[ "$@" =~ .*verbose.* ]] | |
then | |
set -x | |
else | |
set +x | |
fi && | |
if [[ "$@" =~ .*hard.* ]] | |
then | |
SIG=9 | |
else | |
SIG=2 | |
fi && | |
if [[ "$@" =~ .*stop.* ]] | |
then | |
stop | |
elif [[ "$@" =~ .*rotate.* ]] | |
then | |
rotate | |
elif [[ "$@" =~ .*start.* ]] | |
then | |
start | |
else | |
stop && | |
(rotate &) && | |
start | |
fi && | |
wait && | |
exit |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment