Observes OSRM instance for busy threads

osrm-thread-watchdog.sh

#!/usr/bin/env bash

# Observes OSRM instance for busy threads.
# Remark: an instance must have any activity, otherwise the script will assume
#         all forks are long-running.
#
# Arguments:
#  $1 - timeout to consider a fork as "dirty" (in seconds)
#
# Example:
#  osrm-fork-watchdog.sh 60
#
# Author: Kanstantsin Kamkou <2ka.by>

set -e

TIMEOUT=$1
MOTHER_PATTERN="osrm-routed"

[[ -z ${TIMEOUT} ]] && (echo "Missing argument" && exit 1)

declare -a JOBS

function thread_verify()
{
  SCRIPT=$(cat <<EOF
RESULT=\$(timeout --preserve-status $1 strace -e trace=desc -qcp $2 2>&1)
[[ -z "\${RESULT}" ]] && exit 1 || exit 0
EOF
)

  (eval "${SCRIPT}") & JOBS[$!]="$2"
}

for instance_pid in $(ps -ef | grep -v docker | grep -v grep | grep "${MOTHER_PATTERN}" | awk '{print $2}'); do
  echo "OSRM=${instance_pid}"

  BUSY_COUNT=0
  THREADS_COUNT=0

  for thread_pid in $(ps -Tf -p ${instance_pid} --no-headers | awk '{print $3}' | grep -v ${instance_pid}); do
    echo " Thread PID=${thread_pid}"
    THREADS_COUNT=$((THREADS_COUNT + 1))
    thread_verify "${TIMEOUT}" "${thread_pid}"
  done

  for bg_pid in ${!JOBS[@]}; do
    wait "${bg_pid}" && BUSY_COUNT=$((BUSY_COUNT + 1)) && kill -s SIGHUP "${JOBS[$bg_pid]}"
  done

  LOAD_PERCENT=`echo "${BUSY_COUNT} / ${THREADS_COUNT} * 100" | bc -l`

  echo "Instance: ${instance_pid}"
  echo "Busy: ${BUSY_COUNT} of ${THREADS_COUNT}"
  echo "Load: $(printf "%d" "${LOAD_PERCENT}" 2>/dev/null) %"
done

exit 0