Skip to content

Instantly share code, notes, and snippets.

@johann8384
Created May 18, 2020 18:39
Show Gist options
  • Save johann8384/f76eab7de9e6ba75b9de1d1c452fbeed to your computer and use it in GitHub Desktop.
Save johann8384/f76eab7de9e6ba75b9de1d1c452fbeed to your computer and use it in GitHub Desktop.
#! /bin/bash
logger Recorder Check Running
SERVER=localhost
PORT=2003
TOTAL_SENT_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Sending ' | wc -l)
NO_STREAM_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'No Stream for ' | wc -l)
DECODE_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Control Channel Message Decode Rate: 0/sec' | wc -l)
DURATION_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Deleting this call as it has a duration less than minimum duration of' | wc -l)
ENCRYPTED_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Not Recording: ENCRYPTED' | wc -l)
OLDER1=$(find /tmp/streamthis-*-lastrun -not -newermt '-1 minutes' | wc -l)
OLDER5=$(find /tmp/streamthis-*-lastrun -not -newermt '-5 minutes' | wc -l)
OLDER10=$(find /tmp/streamthis-*-lastrun -not -newermt '-10 minutes' | wc -l)
AGE_COUNT=$(find /tmp/streamthis-scclaw-lastrun -not -newermt '-5 minutes' | wc -l)
find /tmp/streamthis-*-lastrun | while read file; do
DATE=`date +%s`;
UTIME=`stat -L --format %Y $file`;
AGE=`expr $DATE - $UTIME`;
STREAM=`echo $file | awk -F '-' '{print $2}'`;
METRIC="recorder.recorder3.stream.$STREAM.age $AGE"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
SENT_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Sending ' | grep $STREAM | wc -l)
METRIC="recorder.recorder3.streams.$STREAM.sent.count $SENT_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
done
HISTORY=$1
LIMIT=$2
METRIC="recorder.recorder3.streams.older.1 $OLDER1"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
METRIC="recorder.recorder3.streams.older.5 $OLDER5"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
METRIC="recorder.recorder3.streams.older.10 $OLDER10"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
METRIC="recorder.recorder3.streams.sent.count $TOTAL_SENT_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
METRIC="recorder.recorder3.streams.none.count $NO_STREAM_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
METRIC="recorder.recorder3.streams.encrypted.count $ENCRYPTED_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
METRIC="recorder.recorder3.recorder.errors.decoder.count $DECODE_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
if [ "$DECODE_COUNT" -gt "$LIMIT" ]; then
logger Too many control channel decode failures, count: $DECODE_COUNT is higher than limit: $LIMIT restarting recorder;
METRIC="recorder.recorder3.recorder.restart.decoder.count 1"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
systemctl restart recorder;
RETVAL=1;
else
METRIC="recorder.recorder3.recorder.restart.decoder.count 0"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
RETVAL=0;
fi
METRIC="recorder.recorder3.recorder.errors.duration.count $DURATION_COUNT"
if [ "$DURATION_COUNT" -gt "$LIMIT" ]; then
logger Too many 0 duration calls, count: $DURATION_COUNT is higher than limit: $LIMIT restarting recorder;
METRIC="recorder.recorder3.recorder.restart.duration.count 1"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
systemctl restart recorder;
RETVAL=1;
else
METRIC="recorder.recorder3.recorder.restart.duration.count 0"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
RETVAL=0;
fi
if [ "$AGE_COUNT" -gt "0" ]; then
logger The SCCLAW Stream is too old, restarting recorder;
METRIC="recorder.recorder3.recorder.restart.age.count 1"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
systemctl restart recorder;
RETVAL=1;
else
METRIC="recorder.recorder3.recorder.restart.age.count 0"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
RETVAL=0;
fi
exit $RETVAL;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment