Script to be run from cron every X minutes to e-mail the stack trace of any tomcat exceptions discovered in the previous X minutes.

log_check_es

#!/bin/bash
#
# log_check_es
#
# Check ElasticSearch for recent exceptions from the live tomcat servers
 
##Debug
#set -x
 
# Exit on Error
set -e
 
##########
# CONFIG #
##########
 
# Minutes Ago
query_minutes=${MINUTES:-5};
query_message='\"Servlet.service() for servlet billbuster threw exception\"';
 
host='http://logstash:9200';
 
mail_admin='true';
admin_email="admin@example.com"
 
##########
# SET-UP #
##########
 
# Now is a Point-in-Time
now_nano=$(date "+%s%N");
now_epoch=$(echo ${now_nano} | cut -b1-10);
 
# Set query timestamps for ElasticSearch
query_offset=$((${query_minutes} * 60000));
query_to=$(echo $now_nano | cut -b1-13);
query_from=$((${query_to} - ${query_offset}));
 
# Get human-friendly dates
query_from_epoch=$(echo ${query_from} | cut -b1-10);
query_to_pretty=$(date -d @${now_epoch});
query_from_pretty=$(date -d @${query_from_epoch});
 
# Set ElasticSearch query
query='{
  "query": {
    "bool": {
      "must": [
        {
          "field": {
            "message": "'${query_message}'"
          }
        },
        {
          "field": {
            "type": "tomcat"
          }
        },
        {
          "range": {
            "@timestamp": {
              "from": '"${query_from}"',
              "to": '"${query_to}"'
            }
          }
        }
      ]
    }
  }
}'
 
#############
# FUNCTIONS #
#############
 
mail_admin () {
  subject=$1;
  message=$2;
 
  echo -e "${message}" | mail -s "${subject}" "${admin_email}";
}
 
#############
# EXECUTION #
#############
 
result_count=$(curl -XPOST "${host}/_search?search_type=count" -s -d "${query}" | cut -d"\"" -f17 | sed 's/[\:,]//g');
 
echo "Number of live exceptions in the last ${query_minutes} minutes ($query_from_pretty to $query_to_pretty): ${result_count}";
 
if [[ ${result_count} -gt 0 ]]; then
  echo -e "Here they are:\n";
  results=$(curl -XPOST "${host}/_search?pretty" -s -d "${query}" | sed 's/,/,\n/g');
  echo -e "${results}";
 
  if $mail_admin; then
    subject="Found ${result_count} exceptions in ${query_minutes} mins on live";
    message="log_check_es on $(hostname) has found ${result_count} exceptions within the last ${query_minutes} minutes on the live app servers.\nReview them all here: http://logstash/#/dashboard/elasticsearch/Live%20Exceptions:\n\n ${results}";
    mail_admin "${subject}" "${message}";
  fi;
 
fi;
 
exit 0;