monitor failed systemd units with cron and send mail

monitor-systemd.sh

#!/bin/bash

# crontab entry:
# */10 * *  *    *     /usr/local/sbin/monitor-systemd.sh --cron

tmpdir=$(mktemp --tmpdir -d systemd-monitor-XXXXXXX)
trap 'rm -rf "${tmpdir}"' EXIT

sendmail=0
if [ "$1" == "--cron" ]; then
	sendmail=1

	touch "${tmpdir}/body_prefix"
	# dup STD{OUT,ERR}
	exec 5>&1
	exec 6>&2
	# close STD{IN,OUT,ERR}, pipe into body
	exec <&-
	exec >"${tmpdir}/body"
	exec 2>&1
fi

printf_body_prefix() {
	printf -- "$@" >> "${tmpdir}/body_prefix"
}

send_mail() {
	if [ "$sendmail" -ne 1 ]; then return 0; fi
	# restore fds, don't send mail twice
	sendmail=0
	exec 1>&5
	exec 2>&6
	exec 5>&-
	exec 6>&-

	# don't send empty mail
	if [ ! -s "${tmpdir}/body_prefix" -a ! -s "${tmpdir}/body" ]; then
		return 0
	fi

	(
		printf 'Subject: [%s] failed systemd units\n' "$(hostname -f)"
		printf 'Precedence: bulk\n'
		printf 'Content-Type: text/plain; charset="UTF-8"\n'
		printf 'Content-Transfer-Encoding: quoted-printable\n'
		printf '\n'
		cat "${tmpdir}/body_prefix" "${tmpdir}/body" | perl -MMIME::QuotedPrint -pe '$_=MIME::QuotedPrint::encode($_);'
	) | \
		/usr/sbin/sendmail "root"
}

checked() {
	"$@"
	rc=$?
	if [ $rc -ne 0 ]; then
		printf_body_prefix "Failed to run: %s\nExit code: %s\n\n---\n" "$*" "$rc"
		send_mail
		exit 0
	fi
}

checked systemctl list-units --state=failed --full --plain --no-legend > "${tmpdir}/failed_units"

while read unit rem; do
	systemctl status --no-pager "${unit}"
done < "${tmpdir}/failed_units"

send_mail