Secure External Backup with ZFS Native Encryption

build-encryption-key.sh

#!/bin/bash
#
# This script usually is called on the first login and asks for a password
# to build the LUKS + ZFS encryption keys which then are stored only in volatile
# memory /mnt/ramfs (ramfs).
# This script is added to your /root/.profile in order you won't forget to
# build the encryption key each time you reboot the server.
#
# We are using ramfs instead of tmpfs as there is no swapping support in 
# ramfs which is good in a security perspective.
# see: http://www.thegeekstuff.com/2008/11/overview-of-ramfs-and-tmpfs-on-linux
#

################### CONFIGURATION #######################
RAMFS_PATH='/mnt/ramfs'
RAMFS_SIZE='20M'
LUKS_KEYFILE=$RAMFS_PATH/luks_pw
ZFS_KEYFILE=$RAMFS_PATH/zfs_enc_key
SALT='**************************************************'
SHA1_CHECK='fb9e740efe20f541349d37eff7aa34efd4ac823d'
#########################################################

printinfo() {
    echo "[INFO] $1"
}

printwarn() {
    echo "[WARNING] $1" | grep --color "WARNING"
}

if [[ -f "$LUKS_KEYFILE" && -f "$ZFS_KEYFILE" ]]; then
    # exit silently, as the key file already exists
    exit
else
    printwarn "The LUKS ($LUKS_KEYFILE) and ZFS ($ZFS_KEYFILE) key files don't yet exist!"
fi

# set up RAM disk if it is not yet mounted
if ! mountpoint -q "$RAMFS_PATH"; then
    echo "Setting up ramfs on $RAMFS_PATH (size=$RAMFS_SIZE) ..."
    mkdir -p $RAMFS_PATH
    mount -t ramfs -o size=$RAMFS_SIZE ramfs $RAMFS_PATH
    if ! grep -q "$RAMFS_PATH" /etc/fstab; then
        echo "Adding line to /etc/fstab to persist mounting of $RAMFS_PATH ..."
        echo "ramfs   $RAMFS_PATH              ramfs   defaults,size=$RAMFS_SIZE        0 0" >> /etc/fstab
    fi
fi

# make this script start on each login
SCRIPT=$(readlink -f $0)
if ! grep -q "$SCRIPT" /root/.profile; then
    echo "$SCRIPT" >> /root/.profile
fi

# get password from interactive user input
while read -s -p 'Unlock encryption keys: ' PASS && [[ $(echo -n "$PASS" | wc --chars) -lt 8 ]] ; do
    echo
    echo "Your password must be at least 8 characters long!"
done
echo

# calculate encryption key (SHA-512 hash of salt.password concatenation)
KEY=$(echo -n "$SALT.$PASS" | sha512sum | cut -d' ' -f1)

# store LUKS key file to ramfs
touch $LUKS_KEYFILE && chmod 600 $LUKS_KEYFILE
echo -n "$KEY" > $LUKS_KEYFILE

# SHA-1 check of the key - assure you have correctly built it by entering the correct password
KEY_SHA1=`cat $LUKS_KEYFILE | sha1sum | cut -d' ' -f1`
if [ "$KEY_SHA1" != "$SHA1_CHECK" ]; then
    printwarn "Your key does not seem to be correct. You might have entered the wrong password. Please run `basename $SCRIPT` again!"
    printinfo "If you are sure you have entered the right password, try to set SHA1_CHECK='$KEY_SHA1' in `basename $SCRIPT`."
    rm -f $LUKS_KEYFILE
    exit 1
fi

printinfo "The LUKS key file was successfully stored in $LUKS_KEYFILE."

# ZFS raw key needs to be 32 characters long
head -c 32 $LUKS_KEYFILE > $ZFS_KEYFILE
printinfo "The ZFS key file was successfully stored in $ZFS_KEYFILE."

############################
# SSH private key encryption
############################
#
# INITIAL SETUP:
#
# Initially, we need to create an encrypted version of id_rsa and destroy the plaintext private key:
#   $ cat ~/.ssh/id_rsa | openssl enc -e -aes-256-cbc -pbkdf2 -a -pass file:/mnt/ramfs/luks_pw > ~/.ssh/id_rsa.encrypted
#
# or you could also copy the password from /mnt/ramfs/luks_pw and enter it interactively:
#   $ cat ~/.ssh/id_rsa | openssl enc -e -aes-256-cbc -pbkdf2 -a > ~/.ssh/id_rsa.encrypted
#   $ enter aes-256-cbc encryption password: (...)
#   $ chmod 600 ~/.ssh/id_rsa.encrypted
#   $ shred -u ~/.ssh/id_rsa
#
# Updating the encrypted version of id_rsa:
#   $ cat /mnt/ramfs/id_rsa.decrypted | openssl enc -e -aes-256-cbc -pbkdf2 -a -pass file:/mnt/ramfs/luks_pw > ~/.ssh/id_rsa.encrypted
# Testing encryption and decryption:
#   $ cat /mnt/ramfs/id_rsa.decrypted | openssl enc -e -aes-256-cbc -pbkdf2 -a -pass file:/mnt/ramfs/luks_pw > ~/.ssh/id_rsa.encrypted-testing
#   $ cat ~/.ssh/id_rsa.encrypted-testing | openssl base64 -d | openssl enc -d -aes-256-cbc -pbkdf2 -pass file:/mnt/ramfs/luks_pw > /mnt/ramfs/id_rsa.testing
#   $ diff /mnt/ramfs/id_rsa.decrypted /mnt/ramfs/id_rsa.testing
#   $ shred -u /mnt/ramfs/id_rsa.testing
#
if [ -f ~/.ssh/id_rsa.encrypted ]; then
    touch $RAMFS_PATH/id_rsa.decrypted && chmod 600 $RAMFS_PATH/id_rsa.decrypted
    cat ~/.ssh/id_rsa.encrypted | openssl base64 -d | openssl enc -d -aes-256-cbc -pbkdf2 -pass file:$LUKS_KEYFILE > $RAMFS_PATH/id_rsa.decrypted
    ln -sf $RAMFS_PATH/id_rsa.decrypted ~/.ssh/id_rsa
    printinfo "SSH private key was successfully decrypted to $RAMFS_PATH/id_rsa.decrypted."
else
    printwarn "Please encrypt your SSH private key to ~/.ssh/id_rsa.encrypted so I can decrypt it to ramfs."
fi

common-functions.sh

#!/bin/bash

####### CONFIGURATION #################
TSFORMAT="%Y-%m-%d %H:%M:%S"
EXECUTING_SCRIPT=$(basename $0)
#######################################

printinfo() {
    echo "[`date +"$TSFORMAT"`] $1"
}

printwarn() {
    echo "[`date +"$TSFORMAT"`] WARNING: $1" | grep --color "WARNING"
}

printerr() {
    >&2 echo "[`date +"$TSFORMAT"`] ERROR: $1"
}

errquit() {
    if [ -n "$1" ]; then
        printerr "$1"
    fi
    if [ -n "$LOCKFILE" ]; then
        rm -f $LOCKFILE
    fi
    exit 1
}

assert_single_instance() {
    # Check if another instance of this script is already running
    # https://stackoverflow.com/a/16807995/5982842
    for pid in $(pidof -x $EXECUTING_SCRIPT); do
        if [ $pid != $$ ]; then
            errquit "$EXECUTING_SCRIPT process is already running with PID $pid"
        fi
    done
}

logall_output_stderr_stdout() {
    logfile=${1:-${EXECUTING_SCRIPT}.log}
    # Append STDOUT and STDERR to logfile and output both at the same time
    exec >  >(tee -ia $logfile)
    exec 2> >(tee -ia $logfile >&2)
}

extbackup-migrate-zfs-snaps.sh

#!/bin/bash
######### CONFIGURATION ############
DISKS=3
DATAPOOL=dpool
OLDLABEL=1a
BKUP_SERVER="backup"
####################################

usage() {
    echo "Usage:"
    echo "$(basename $0) [-l|--label OLDLABEL]"
    echo
    echo "Where:"
    echo "  -l|--label OLDLABEL  The label of the disks where the previous snapshots have been performed (default: 1a)"
    echo "  -s|--dryrun          Dry-run / simulation mode: Don't rename any snapshots"
    echo "  -p|-purge            Purge all snapshots that don't match current disk label (only run this as a final migration!)"
    exit 1
}

# Get options
while :
do
    case $1 in
        -l | --label)
            OLDLABEL="${2:-$OLDLABEL}"
            shift
            shift
            ;;
        -s | --dryrun)
            DRYRUN="true"
            shift
            ;;
        -p | --purge)
            PURGE="true"
            shift
            ;;
        -h | --help)
            usage
            ;;
        *)  # no more options. Stop while loop
            break
            ;;
    esac
done

# include common functions
source /usr/local/sbin/common-functions.sh

# zfs command convienence aliases
zfslistname='zfs list -H -o name'

# abort on missing label
if [ ! -f /$DATAPOOL/LABEL ]; then
    errquit "Disk label (/$DATAPOOL/LABEL) does not exist. Backup script aborted!!!"
fi
label=$(cat /$DATAPOOL/LABEL)
if [[ ! $label =~ ^[1-$DISKS]a$ ]]; then
    errquit "Invalid label in /$DATAPOOL/LABEL: $label"
fi

dspattern="$DATAPOOL/zfsdisks/[[:alnum:]-]+"
for dataset in $($zfslistname -t filesystem -d 2 $DATAPOOL); do
    if [[ $dataset =~ ^$dspattern$ ]]; then
        # local purge run (as a final migration)
        if [ -n "$PURGE" ]; then
            for snapshot in $($zfslistname -t snapshot -s creation $dataset | grep @rep_extbackup_${OLDLABEL}_); do
                cmd="zfs destroy $snapshot"
                if [ -z "$DRYRUN" ]; then
                    printinfo "$cmd"
                    $cmd
                else
                    printinfo "(DRYRUN) $cmd"
                fi
            done
            continue # don't proceed with snapshot renaming below, as that is part of first migration
        fi

        # snapshot renaming (first migration)
        snapold=$($zfslistname -t snapshot -s creation $dataset | grep @rep_extbackup | tail -n1)
        if [[ -z "$snapold" ]]; then
            printwarn "Skipping $dataset: no extbackup snapshot found."
            continue
        fi
        snapnew="${snapold/@rep_extbackup_$OLDLABEL/@rep_extbackup_$label}"
        cmd="zfs rename $snapold $snapnew"
        if [ -z "$DRYRUN" ]; then
            printinfo "$cmd"
            $cmd
            if [ $? -ne 0 ]; then
                errquit "Failed to locally rename snapshot. Migration script aborted!"
            fi
            ssh $BKUP_SERVER $cmd
            if [ $? -ne 0 ]; then
                errquit "Failed to remotely rename snapshot on $BKUP_SERVER. Migration script aborted!"
            fi
        else
            printinfo "(DRYRUN) $cmd"
        fi
    fi
done

extbackup-zfs.sh

#!/bin/bash

######### CONFIGURATION ############
USB_DEVICE=/dev/sdb
DISKS=3
DATAPOOL=dpool
BKUP_SERVER="backup"
BKUP_SRCDIR="/backup"
MAXSNAP=1
KEYFILE="/mnt/ramfs/zfs_enc_key"
LOGFILE="/var/log/extbackup.log"
EXCLUDES=() # datasets that should be excluded from backup
SCRIPTNAME=$(basename $0)
CREATE_RERUN=1 # default to creating rerun on failure
RERUN_FILE="/tmp/$SCRIPTNAME.rerun"
####################################

usage() {
    echo "Usage:"
    echo "$(basename $0) [-m|--maxsnap N] [-r|--rerun]"
    echo
    echo "Where:"
    echo "  -m|--maxsnap N  The number of snapshots to keep until older ones are erased (default: 1)"
    echo "  -r|--rerun      Rerun mode: rerun only if previous run has failed"
    exit 1
}

# Get options
while :
do
    case $1 in
        -m | --maxsnap)
            MAXSNAP="${2:-$MAXSNAP}"
            shift
            shift
            ;;
        -r | --rerun)
            RERUN="true"
            shift
            ;;
        -h | --help)
            usage
            ;;
        *)  # no more options. Stop while loop
            break
            ;;
    esac
done

# include common functions
source /usr/local/sbin/common-functions.sh
# ensure this is the only running instance
assert_single_instance
# append STDOUT and STDERR to logfile and output both at the same time
logall_output_stderr_stdout $LOGFILE

# zfs command convienence aliases
zfsgetval='zfs get -H -o value'

# Rerun if `--rerun` was provided and previous run has failed
if [ -n "$RERUN" ]; then
    if [[ ! -f $RERUN_FILE ]]; then
        # printinfo "did not find rerun file $RERUN_FILE; exiting OK"
        exit
    fi
elif [[ $CREATE_RERUN -eq 1 ]]; then
    touch $RERUN_FILE
fi

# check if key file exists
if [ ! -f $KEYFILE ]; then
    errquit "The key file $KEYFILE does not exist yet. Please run build-encryption-key.sh first!"
fi

# First, make sure /dev/sdb will wake up and actually exists (using parted as a simple workaround)
parted -s $USB_DEVICE print > /dev/null
if [ $? -ne 0 ]; then
    errquit "External USB disk does not seem to show up as $USB_DEVICE. Backup script aborted!"
fi

# Import dpool, load encrytion key and mount all pools
printinfo "Importing ZFS pool $DATAPOOL ..."
zpool import $DATAPOOL
if [ $? -ne 0 ]; then
    errquit "Could not import $DATAPOOL. Backup script aborted!"
fi
printinfo "Loading key for ZFS pool $DATAPOOL ..."
zfs load-key $DATAPOOL
printinfo "Mouting/decrypting all ZFS datasets ..."
zfs mount -l -a

# Check if decryption/mounting of dpool was done
if [ $? -ne 0 ]; then
    errquit "Could not mount/decrypt ZFS filesystems. Backup script aborted!"
fi
if [ "$($zfsgetval keystatus $DATAPOOL)" != "available" ]; then
    errquit "ZFS dataset $DATAPOOL does not seem to be correctly decrypted. Backup script aborted!"
fi

# abort on missing /dpool/LABEL or wrongly formatted label
if [ ! -f /$DATAPOOL/LABEL ]; then
    errquit "Disk label (/$DATAPOOL/LABEL) does not exist. Backup script aborted!!!"
fi
label=$(cat /$DATAPOOL/LABEL)
if [[ ! $label =~ ^[1-$DISKS]a$ ]]; then
    errquit "Invalid label in /$DATAPOOL/LABEL: $label"
fi

printinfo "=== STARTING EXTBACKUP TO DISK WITH LABEL $label ==="

# loop over all datasets and run pve-zsync
# `list-datasets` is a magic command that is resolved to `zfs list -H -o name,mountpoint,usedds | grep /backup` on remote
ssh $BKUP_SERVER list-datasets | while read dataset mountpoint usedds; do
    zsync="pve-zsync sync --source $BKUP_SERVER:$dataset --dest $(dirname $dataset) --maxsnap $MAXSNAP --name extbackup_$label"
    # check if the dataset should be excluded from extbackup
    if [[ ${EXCLUDES[*]} =~ $dataset ]]; then
        printwarn "(SKIPPED) $zsync"
        continue
    fi

    # check if dataset exists
    if zfs list $dataset &>/dev/null; then
        # unmount dataset prior to pve-zsync run to avoid "dataset is busy"
        zfs unmount $dataset
    fi
    printinfo "$zsync # $usedds"
    # use `< /dev/null` as workaround to avoid pve-zsync breaking out of while loop (as it somehow expects input)
    $zsync < /dev/null

    # set mountpoint to the same as on remote backup server
    if [ "$($zfsgetval mountpoint $dataset)" != "$mountpoint" ]; then
        cmd="zfs set mountpoint=$mountpoint $dataset"
        printinfo "$cmd"
        $cmd
        sleep 1
    else
        zfs mount $dataset
    fi
done

printinfo "Unmounting/encrypting ZFS pool $DATAPOOL ..."
zfs unmount $DATAPOOL
printinfo "Unloading key for ZFS pool $DATAPOOL ..."
zfs unload-key $DATAPOOL
printinfo "Exporting ZFS pool $DATAPOOL ..."
zpool export $DATAPOOL
if [ $? -ne 0 ]; then
    errquit "Could not export $DATAPOOL. Please fix!"
fi

# Remove rerun file on successful run
rm -f $RERUN_FILE

sync
printinfo "DONE."