Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
skyline analyzer watcher
#!/bin/bash
#### snype_skyline_analyzer.sh ####
#
####
# DESCRIPTION
# This mitigates against analyzer running but it connection timing out
#
# THIS IS JUST A GIST
# THIS IS JUST A GIST
# Deal with logging and variables such as SERVER, statsd_node and if you have
# monit in the mix, etc yourself and your own skyline log paths etc these are
# are just bits for reference for a pattern to run via cron every minute
# Example script log is just flushed here on every run
LOGFILE=/var/log/skyline/snype_skyline_analyzer.log
> $LOGFILE
function log () {
local log_string="$1"
local output_type="$2"
echo "$(date +%Y%m%d%H%M%S) - $output_type: $log_string" >> $LOGFILE
}
# Handle Connection timed out
ANALYZER_RESTART=0
HORIZON_RESTART=0
LOGFILE_EPOCH=$(stat --format=%Y /var/log/skyline/analyzer.log)
SECONDS_DIFFERENCE=$(( $TIMESTAMP - $LOGFILE_EPOCH ))
if [ $SECONDS_DIFFERENCE -gt 180 ]; then
log "The analyzer log was last modified $SECONDS_DIFFERENCE seconds ago" warn
log "analyzer set to restart" notice
ANALYZER_RESTART=1
fi
CONNECTION_TIMEOUT=$(tail -n 100 /var/log/skyline/analyzer.log | grep -c "Connection timed out")
if [ $CONNECTION_TIMEOUT -gt 0 ]; then
log "The analyzer log reports connection timeout" warn
log "horizon and analyzer set to restart" notice
ANALYZER_RESTART=1
HORIZON_RESTART=1
fi
if [ $ANALYZER_RESTART -eq 0 ]; then
LOG_ENTRY_DATE=$(tail -n 1 /var/log/skyline/analyzer.log | cut -d' ' -f1-2)
LOG_EPOCH=$(date -d "$LOG_ENTRY_DATE" +%s)
SECONDS_DIFF=$(( $TIMESTAMP - $LOG_EPOCH ))
if [ $SECONDS_DIFF -gt 180 ]; then
log "The analyzer log was last modified $SECONDS_DIFF seconds ago" warn
log "analyzer set to restart" notice
ANALYZER_RESTART=1
fi
fi
if [ $HORIZON_RESTART -eq 1 ]; then
log "Restarting horizon" notice
/sbin/service horizon stop >> $LOGFILE
/sbin/service horizon start >> $LOGFILE
echo "$HOSTNAME.skyline.horizon.restarted:$TIMESTAMP|g"| nc -w 5 -u $statsd_node 8125
log "Submitted $HOSTNAME.skyline.horizon.restarted:$TIMESTAMP|g to $statsd_node" notice
fi
if [ $ANALYZER_RESTART -eq 1 ]; then
log "Restarting analyzer" notice
/sbin/service analyzer stop >> $LOGFILE
/sbin/service analyzer start >> $LOGFILE
echo "$HOSTNAME.skyline.analyzer.stalled:$TIMESTAMP|g"| nc -w 5 -u $statsd_node 8125
log "Submitted $HOSTNAME.skyline.analyzer.stalled:$TIMESTAMP|g to $statsd_node" notice
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.