My restic backup solution using Backblaze B2 storage, systemd timers and email notifications on failure

restic-backup.service

[Unit]
Description=Backup with restic to Backblaze B2
OnFailure=status-email-user@%n.service

[Service]
Type=simple
Nice=10
ExecStart=/usr/local/sbin/restic_backup.sh
# $HOME or $XDG_CACHE_HOME must be set for restic to find /root/.cache/restic/
Environment="HOME=/root"

restic-backup.timer

[Unit]
Description=Backup with restic on schedule

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target

restic-check.service

[Unit]
Description=Check restic backup Backblaze B2 for errors
OnFailure=status-email-user@%n.service
Conflicts=restic.service

[Service]
Type=simple
Nice=10
ExecStart=/usr/local/sbin/restic_check.sh

restic-check.timer

[Unit]
Description=Check restic backup Backblaze B2 for errors on a schedule

[Timer]
OnCalendar=monthly
Persistent=true

[Install]
WantedBy=timers.target

restic_backup.sh

#!/usr/bin/env bash
# Make backup my system with restic to Backblaze B2.
# This script is typically run by: /etc/systemd/system/restic-backup.{service,timer}

# Exit on failure, pipe failure
set -e -o pipefail

# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
me=$(basename "$0")
now=$(date +%F_%R)
log_dir=/var/local/log/restic
log_file="${log_dir}/${now}_${me}.$$.log"
test -d $log_dir || mkdir -p $log_dir
exec > >(tee -i $log_file)
exec 2>&1

# Clean up lock if we are killed.
# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
# However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
exit_hook() {
	echo "In exit_hook(), being killed" >&2
	jobs -p | xargs kill
	restic unlock
}
trap exit_hook INT TERM

RETENTION_DAYS=7
RETENTION_WEEKS=12
RETENTION_MONTHS=18
RETENTION_YEARS=4

BACKUP_PATHS="/ /boot /home /mnt/media"
BACKUP_EXCLUDES="--exclude-file /.rsync_exclude --exclude-file /mnt/media/.rsync_exclude --exclude-file /home/erikw/.rsync_exclude"
BACKUP_TAG=systemd.timer

# Set all environment variables like
# B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc.
source /etc/restic/b2_env.sh


# NOTE start all commands in background and wait for them to finish.
# Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered.
# However if put in subprocesses, wait(1) waits until the process finishes OR signal is received.
# Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash

# Remove locks from other stale processes to keep the automated backup running.
restic unlock &
wait $!

# See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html
#restic backup --tag $BACKUP_TAG --one-file-system $BACKUP_EXCLUDES $BACKUP_PATHS &
#wait $!

# Until 
# https://github.com/restic/restic/issues/1557
# is fixed with the PR
# https://github.com/restic/restic/pull/1494
# we have to use a work-around and skip the --one-file-system and explicitly black-list the paths we don't want, as described here
# https://forum.restic.net/t/full-system-restore/126/8?u=fd0
restic backup \
	--tag $BACKUP_TAG \
	--exclude-file /.restic-excludes \
	$BACKUP_EXCLUDES \
	/ &
wait $!

# See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
restic forget \
	--tag $BACKUP_TAG \
	--keep-daily $RETENTION_DAYS \
	--keep-weekly $RETENTION_WEEKS \
	--keep-monthly $RETENTION_MONTHS \
	--keep-yearly $RETENTION_YEARS &
wait $!

# Remove old data not linked anymore.
# See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html
restic prune &
wait $!


# Check repository for errors.
# NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often.
#restic check &
#wait $!

restic_check.sh

#!/usr/bin/env bash
# Check my backup with  restic to Backblaze B2 for errors.
# This script is typically run by: /etc/systemd/system/restic-check.{service,timer}

# Exit on failure, pipe failure
set -e -o pipefail

# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
me=$(basename "$0")
now=$(date +%F_%R)
log_dir=/var/local/log/restic
log_file="${log_dir}/${now}_${me}.$$.log"
test -d $log_dir || mkdir -p $log_dir
exec > >(tee -i $log_file)
exec 2>&1

# Clean up lock if we are killed.
# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
# However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
exit_hook() {
	echo "In exit_hook(), being killed" >&2
	jobs -p | xargs kill
	restic unlock
}
trap exit_hook INT TERM



source /etc/restic/b2_env.sh

# Remove locks from other stale processes to keep the automated backup running.
# NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script.
#restic unlock &
#wait $!

# Check repository for errors.
restic check &
wait $!

status-email-user@.service

# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO

[Unit]
Description=Send status email for %i to user

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/systemd-email abc@gmail.com %i
User=root
Group=systemd-journal

systemd-email

#!/usr/bin/env sh
# Send email notification from systemd.
# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO
# Usage: systemd-email <recipinent-email> <failed-systemd-unit-name>


# According to 
# http://www.flashissue.com/blog/gmail-sending-limits/
# Gmail blocks your account if you send more than 500 emails per day, which is one email every
# (24 * 60 * 60) / 500 = 172.8 second => choose a min wait time which is significantly longer than this to be on the safe time to not exceed 500 emails per day.
# However this source
# https://group-mail.com/sending-email/email-send-limits-and-options/
# says the limit when not using the Gmail webinterface but going directly to the SMTP server is 100-150 per day, which yelds maximum one email every
# (24 * 60 * 60) / 100 = 864 second 
# One option that I used with my old Axis cameras it to use my gmx.com accunt for sending emails instead, as there are (no?) higher limits there.
MIN_WAIT_TIME_S=900
SCRIPT_NAME=$(basename $0)
LAST_RUN_FILE="/tmp/${SCRIPT_NAME}_last_run.txt"

last_touch() {
	stat -c %Y $1
}

waited_long_enough() {
	retval=1
	if [ -e $LAST_RUN_FILE ]; then
		now=$(date +%s)
		last=$(last_touch $LAST_RUN_FILE)
		wait_s=$(expr $now - $last)
		if [ "$wait_s" -gt "$MIN_WAIT_TIME_S" ]; then
			retval=0
		fi
	else
		retval=0
	fi

	[ $retval -eq 0 ] && touch $LAST_RUN_FILE
	return $retval
}


# Make sure that my Gmail account dont' get shut down because of sending too many emails!
if ! waited_long_enough; then
	echo "Systemd email was not sent, as it's less than ${MIN_WAIT_TIME_S} seconds since the last one was sent."
	exit 1
fi


recipinent=$1
system_unit=$2

sendmail -t <<ERRMAIL
To: $recipinent
From: systemd <root@$HOSTNAME>
Subject: [systemd-email] ${system_unit}
Content-Transfer-Encoding: 8bit
Content-Type: text/plain; charset=UTF-8

$(systemctl status --full "$system_unit")
ERRMAIL