Filter journalctl alerts and forward via mail

journal-alert.sh

#!/bin/bash
set -euf -o pipefail

recipients=(
  'root'
)

journal_filter=(
  --priority 4
  --since -1d
)

json_whitelist=(
  'false'
)

json_blacklist=(
  # only forward warnings or higher levels
  #'.PRIORITY|tonumber? >= 5'
  # ignore kernel notices
  #'.SYSLOG_IDENTIFIER == "kernel" and (.PRIORITY|tonumber) >= 4'
  # ignore systemd
  '.SYSLOG_IDENTIFIER == "systemd-udevd" and (.MESSAGE|startswith("Could not generate persistent MAC address"))'
)

query_log() {
  jq_whitelist="$(printf ' or (%s)' "${json_whitelist[@]}")"
  jq_blacklist="$(printf ' or (%s)' "${json_blacklist[@]}")"

  journalctl --no-pager --no-tail --output-fields=__REALTIME_TIMESTAMP,PRIORITY,SYSLOG_IDENTIFIER,MESSAGE -o json "${journal_filter[@]}" \
    | jq -Sc "select (${jq_whitelist:4} or ((${jq_blacklist:4})|not))"
}

format_log() {
  jq -r '(.__REALTIME_TIMESTAMP|tonumber|(./1e6)|strflocaltime("%b %d %H:%M:%S")) + " " + .PRIORITY + " " + .SYSLOG_IDENTIFIER + ": " + .MESSAGE'
}

if [[ "${1:-}" != "--send-mail" ]]; then
  query_log | format_log
  exit 0
fi

name="$(basename "${BASH_SOURCE[0]}")"
hash_file="/var/run/${name}.sha256"
sendmail="/usr/sbin/sendmail"

# check latest entry changed
current_hash="$(sha256sum <(query_log | tail -n 1) | cut -d ' ' -f 1)"

if [[ "${current_hash}" == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' ]]; then
  # empty report
  exit 0
elif [[ "${current_hash}" == "$(head -n 1 "${hash_file}" 2> /dev/null)" ]]; then
  # no changes to report
  exit 0
fi

# forward system log to recipients
echo -n "${current_hash}" > "${hash_file}"
cat \
  <(echo "Subject: Journal Alert") \
  <(echo "Content-Type: text/plain; charset=\"utf-8\"" ) \
  <(echo "" ) \
  <(query_log | tail -n 15 | format_log ) \
  | "${sendmail}" "${recipients[@]}"