Capture Apache debug

watch_apache_log.sh

#!/bin/bash
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License <gnu.org/licenses/gpl.html> for more details.


# Usage example:
# $ sudo -b bash -c './watch_scoreboard.sh 2>&1 | tee -a watch_scoreboard.log > /dev/null'

# Customize variables as desired
logFile=/var/log/httpd/error_log
regEx='scoreboard is full'
triggerFile=$(mktemp)
outputDir=/root/pstack-$(hostname -s)-$(date +%F.%R)

if [[ $(id -u) != 0 ]]; then
  echo "ERR: Must run as root"
  exit 1
elif ! command -v inotifywait >/dev/null; then
  echo "ERR: Missing 'inotifywait' command; install inotify-tools from EPEL"
  exit 1
elif [[ ! -r ${logFile} ]]; then
  echo "ERR: Unable to read logfile '${logFile}'"
  exit 1
elif ! mkdir -p "${outputDir}"; then
  echo "ERR: Unable to create '${outputDir}'"
  exit 1
fi >&2

# Kick off tail in background subshell
( tail -Fn0 "${logFile}" | grep --line-buffered "${regEx}"  >> "${triggerFile}" ) &

# Ensure tail & everything else is killed in case of abnormal exit
trap "pkill --pgroup 0" INT

# Start waiting for modifications to trigger file
echo "Watching file '${logFile}' for regex '${regEx}' ..."
inotifywait -e modify "${triggerFile}"

loopCount=20
sleepInterval=20s
echo "Trigger met; Starting pstack captures of httpd processes"
for y in $(seq 1 ${loopCount}); do
  curl -so "${outputDir}/server-status-$(printf '%02d' ${y}).html" http://localhost/server-status
  echo Begin loop ${y} of ${loopCount} >&2
  echo -e "\nDate: $(date)"
  for pid in $(ps -C httpd,httpd.worker --noheader -o pid); do
    echo "    PID: ${pid}"
    pstack ${pid}
  done
  if [[ ${y} -lt ${loopCount} ]]; then
    echo Sleeping ${sleepInterval} ... >&2
    sleep ${sleepInterval}
  fi
done >> "${outputDir}/pstack.cap"

tar -czf "${outputDir}.tar.gz" "${outputDir}"
echo DONE
echo "Send '${outputDir}.tar.gz'"
echo
echo "Killing background tail-follow process"
kill ${!}
rm "${triggerFile}" 2>/dev/null
echo FINISHED