zfs-replication

#!/bin/bash
#
# Author: Ryan Kernan
# Date: September 23, 2011
# Version: 1.0
# Credits: Mike La Spina for the original concept and script http://blog.laspina.ca/
#
# Expanded on by Patrick Domack
#
# Function: Provides snapshot and send process which replicates ZFS file systems from a source to target server.
# Maintains a runing snapshot archive (optional) for X hours.
# 
# Input parms:1
# rep_list - a file name var of which the file contains a list of comma delimed:
#      zfs source dataset, zfs source host, zfs target dataset, zfs target host, make new snapshot (yes,no), keep # snaps, keep snaps # hours, mbuffer rate limit read, mbuffer rate limit write

export PATH=/usr/gnu/bin:$PATH:/usr/bin:/usr/sbin:/sbin

rep_list=$( cat "$1" )
def_keep_snaps=3
def_keep_hours=12
def_make_snaps=yes
def_rate=""
replog=replicate.log
LOCKFILE="$1.lock"

sshopt="-T -o Compression=no -x"
compress="lz4 -2 -c"
decompress="lz4cat"
#compress="lz4 -zc1"
#decompress="lz4 -cd"

keep_snaps=$def_keep_snaps
keep_hours=$def_keep_hours
make_snaps=$def_make_snaps
mbuffer_rater=$def_rate
mbuffer_ratew=$def_rate

if [ "$2" == "MASTER" ]; then
  MASTER=`ifconfig -a | grep 10.1.0.115 | wc -l`
  if [ $MASTER -lt 1 ]; then
    echo "Not Master"
    exit
  fi
fi

#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function parse the comma delimited input file and assign the field to the respective
# variables for file system, pool and host names.
#
# Global output parms
# zfspool
# zfspath
# shost
# dhost
 
parse_rep_list() {
        rep_list_line="$1"

        shopt -s nocasematch
        keep_snaps=$def_keep_snaps
        keep_hours=$def_keep_hours
        make_snaps=$def_make_snaps
        mbuffer_rater=$def_rate
        mbuffer_ratew=$def_rate

        zfspaths="$(echo $rep_list_line | cut -d, -f1)"
        shost="$(echo $rep_list_line | cut -d, -f2)"
        zfspathd="$(echo $rep_list_line | cut -d, -f3)"
        dhost="$(echo $rep_list_line | cut -d, -f4)"
        tmp="$(echo $rep_list_line | cut -d, -f5)"
        [[ "$tmp" == "yes" ]] && make_snaps=yes
        [[ "$tmp" == "true" ]] && make_snaps=yes
        [[ "$tmp" == "no" ]] && make_snaps=no
        [[ "$tmp" == "false" ]] && make_snaps=no
        tmp="$(echo $rep_list_line | cut -d, -f6)"
        if [ -n "$tmp" ] && [ "$tmp" -gt 0 ]; then keep_snaps=$tmp; fi
        tmp="$(echo $rep_list_line | cut -d, -f7)"
        if [ -n "$tmp" ] && [ "$tmp" -gt 0 ]; then keep_hours=$tmp; fi
        tmp="$(echo $rep_list_line | cut -d, -f8)"
        if [ -n "$tmp" ]; then mbuffer_rater="-r $tmp"; fi
        tmp="$(echo $rep_list_line | cut -d, -f9)"
        if [ -n "$tmp" ]; then mbuffer_ratew="-R $tmp"; fi
                        
        if [ -z "$zfspathd" ]; then zfspathd=$zfspaths; fi
        zfspools="$(echo $zfspaths | cut -d/ -f1)"
        zfspoold="$(echo $zfspathd | cut -d/ -f1)"
        shopt -u nocasematch
}

#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function issue zfs list commands and assign the variables the last snapshot names for
# both the source and destination hosts.
#
# Global input parms
# zfspath
# shost
# dhost

get_snapshots() {
        snaps_dhost=""
        snaps_shost=""

        snaps_shost=$( zfs list -H -o name -t snapshot | gawk "{ if(\$1 ~ \"${zfspaths}@\") {split(\$1,a,\"@\"); b[lines++]=a[2];}} END {while(lines>0) {print b[--lines]}}" ) 
        snaps_dhost=$( ssh $sshopt -n $dhost "zfs list -H -o name -t snapshot | gawk \"{ if(\\\$1 ~ \\\"${zfspathd}@\\\") {split(\\\$1,a,\\\"@\\\"); b[lines++]=a[2];}} END {while(lines>0) {print b[--lines]}}\" " )
        true
}

find_common_snap() {
        while read last_snap_dhost; do
                while read last_snap_shost; do
                        if [ "$last_snap_dhost" == "$last_snap_shost" ]; then return; fi
                done <<< "$snaps_shost";
        done <<< "$snaps_dhost";
        last_snap_dhost=""
        last_snap_shost=""
}

find_last_snap_shost() {
        while read last_snap_shost; do
                return
        done <<< "$snaps_shost";
}

check_last_snap() {
        last_snap_dhost=""
        last_snap_shost=""

        get_snapshots
        find_common_snap
        find_last_snap_shost
#       echo "Found start $last_snap_dhost, and end $last_snap_shost"
}


#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function check if the destination zfs path exists and assign the result to the
# variable dhost_fs_name.
#
# Global input parms
# zfspath
# dhost

dhost_fs_exists() {

        dhost_fs_name=""
        dhost_fs_name=$(ssh $sshopt -n $dhost zfs list -o name -H $zfspathd | tail -1)

        if [ "$dhost_fs_name" = "" ]; then
                echo $(date --rfc-3339=seconds) "->" $zfspathd file system does not exist on target host $dhost. >> ${replog}
        fi
}

#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function create a zfs path on the destination to allow the receive command
# funtionallity then issue zfs snap and send to transfer the zfs object to the 
# destination host
#
# Global input parms
# zfspath
# dhost

dhost_create_fs() {

        ssh $sshopt -n $dhost zfs create -p $zfspathd 
        ssh $sshopt -n $dhost zfs set mountpoint=none $zfspathd
        last_snap_shost=$( zfs list -o name -t snapshot -H | gawk "{ if(\$1 ~ \"${zfspaths}@\") {split(\$1,a,\"@\"); print a[2];}}" | tail -1 ) 
        echo $(date --rfc-3339=seconds) "->" $last_snap_shost Initial replication to $dhost start. >> ${replog}
        zfs send -v -e -c -R $zfspaths@$last_snap_shost | mbuffer -v3 -m1G -s1M -R 30M $mbuffer_rater | $compress | ssh $sshopt $dhost "mbuffer -v0 -m1G -s1M $mbuffer_ratew | $decompress | zfs recv -v -F $zfspathd"
        echo $(date --rfc-3339=seconds) "->" $last_snap_shost Initial replication to $dhost end. >> ${replog}
}

#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function Issue a snapshot for the source zfs path
#
# Global input parms
# zfspath

create_fs_snap() {

        snap_date="$(date +%m-%d-%y-%H:%M)"
        echo $(date --rfc-3339=seconds) "->" $zfspaths@$snap_date Snapshot creation start on $shost >> ${replog}
        zfs snapshot -r $zfspaths@$snap_date 
        echo $(date --rfc-3339=seconds) "->" $zfspaths@$snap_date Snapshot creation end on $shost >> ${replog}
}

#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function create a zfs send/recv command set based on a the zfs path source 
# and target hosts for an established replication state. (aka incremental replication)
#
# Global input parms
# zfspath
# dhost

incr_repl_fs() {

        echo $(date --rfc-3339=seconds) "->" $zfspaths\@$last_snap_dhost $zfspaths\@$last_snap_shost Incremental send to $dhost start. >> ${replog}
        zfs send -e -c -R -I $zfspaths\@$last_snap_dhost $zfspaths\@$last_snap_shost | mbuffer -v3 -m1G -s1M -R 30M $mbuffer_rater | $compress | ssh $sshopt $dhost "mbuffer -v0 -m1G -s1M $mbuffer_ratew | $decompress | zfs recv -F $zfspathd" >> ${replog}
        echo $(date --rfc-3339=seconds) "->" $zfspaths\@$last_snap_dhost $zfspaths\@$last_snap_shost Incremental send to $dhost end. >> ${replog}
}

#######################################################################################
####################################Function###########################################
#######################################################################################
#
# Function to clean up snapshots that are older than X days old X being the 
# value set by "keep_snaps" on both the source and destination hosts.

clean_snaps() {

        PreviousSnapDate=""
        PreviousSnap=""
        CurrentYear="$( date +%y )"
        CurrentMonth="$( date +%m )"
        CurrentDay="$( date +%d )"
        keep_hours2=$(($keep_hours+1))

        if [ "$zfspaths" != "" ]; then
                snap_list=$(zfs list -o name -t snapshot | grep $zfspaths)

                while read snaps; do
                        stringpos=0
                        let stringpos=$(expr index "$snaps" @)+1
                        SnapDate=$( expr substr $snaps $stringpos 8 )
                        let stringpos=$(expr index "$snaps" @)+10
                        SnapTime=$( expr substr $snaps $stringpos 5 )
                        SnapDay="$(echo $SnapDate | cut -d- -f2)"
                        SnapMonth="$(echo $SnapDate | cut -d- -f1)"       
                        SnapYear="$(echo $SnapDate | cut -d- -f3)"
                        SnapHour="$(echo $SnapTime | cut -d: -f1)"

                        SnapDate="$SnapMonth-$SnapDay-$SnapYear"
                        SnapDateHour="$SnapMonth-$SnapDay-$SnapYear-$SnapHour"
                                
                        if [ "$(date +%m-%d-%y --date="$keep_snaps days ago")" = "$SnapDate" ]; then
                                echo "Destroying $SnapDate snapshot $snaps on $shost" >> ${replog}
                                /sbin/zfs destroy $snaps
                                #echo "Destroying $SnapDate snapshot $snaps on $dhost" >> ${replog}
                                #ssh -n $dhost zfs destroy $snaps
                        fi
                        if [ "$(date +%m-%d-%y-%H --date="$keep_hours hours ago")" = "$SnapDateHour" ]; then
                                echo "Destroying $SnapDateHour snapshot $snaps on $shost" >> ${replog}
                                /sbin/zfs destroy $snaps
                                #echo "Destroying $SnapDateHour snapshot $snaps on $dhost" >> ${replog}
                                #ssh -n $dhost zfs destroy $snaps
                        fi
                        if [ "$(date +%m-%d-%y-%H --date="$keep_hours2 hours ago")" = "$SnapDateHour" ]; then
                                echo "Destroying $SnapDateHour snapshot $snaps on $shost" >> ${replog}
                                /sbin/zfs destroy $snaps
                                #echo "Destroying $SnapDateHour snapshot $snaps on $dhost" >> ${replog}
                                #ssh -n $dhost zfs destroy $snaps
                        fi
                done <<< "$snap_list"
        fi
}

#######################################################################################
#####################################Main Entery#######################################
#######################################################################################
#
#
# Init Global Parms

zfspaths="" 
zfspathd=""
shost="" 
dhost=""

if [ -e ${LOCKFILE} ] && kill -0 `cat ${LOCKFILE}`; then
    echo "already running"
    exit
fi
        
# make sure the lockfile is removed when we exit and then claim it
trap "rm -f ${LOCKFILE}; exit" INT TERM EXIT
echo $$ > ${LOCKFILE}
        

# Create the snapshots all at the same time

while read line; do
        [[ "$line" =~ ^\#.*$ ]] && continue
        [[ "$line" =~ ^\ *$ ]] && continue

        parse_rep_list $line

        if [ -n "$zfspaths" ]; then
                if [ "$make_snaps" == "yes" ]; then
                        create_fs_snap >> ${replog}             #Create a new snapshot of the path spec.
                fi
        fi
done <<< "$rep_list"

# Send the snapshots to the remote and create the fs if required 

while read line; do
        [[ "$line" =~ ^\#.*$ ]] && continue
        [[ "$line" =~ ^\ *$ ]] && continue

        parse_rep_list $line 

        if [ -n "$zfspathd" ]; then
                dhost_fs_exists                                                                 # Test for the existence of our listed 
                                                                                                                # zfs file system path on the target host.
                if [ -n "$dhost_fs_name" ]; then
                        check_last_snap >> ${replog}                    # Get the start and stop snaps.
#                       echo $(date --rfc-3339=seconds) "-> DEBUG last_snap_shost=$last_snap_shost, last_snap_dhost=$last_snap_dhost"
                        if [ -n "$last_snap_shost" ] && [ -n "$last_snap_dhost" ] && [ "$last_snap_shost" != "$last_snap_dhost" ]; then
                                incr_repl_fs >> ${replog}                               # Initiate a dif replication.
                                clean_snaps                                                                     # Clean up any snapshots that are X days old.
                        fi
                else
                        dhost_create_fs >> ${replog}                    # Create a first time replication.
                fi
          fi
done <<< "$rep_list"

rm -f ${LOCKFILE}
exit 0