ceph sync

#!/bin/bash
##################################################################################
#   Author: Gagandeep Arora
#   Date: 01-Oct-2014
# Purpose: To sync the rbds using incremenatl snapshots from production ceph cluster
# 	   	   to backup ceph cluster
##################################################################################
mirrorPool() { 
    #list all images in the pool
    IMAGES=`$SOURCERBDCMD ls $SOURCEPOOL` 

    for LOCAL_IMAGE in $IMAGES; do 
        #check whether remote host/pool has image 
        if [[ -z $($DESTRBDCMD ls $DESTPOOL | grep $LOCAL_IMAGE) ]];then
            echo "info: image does not exist in remote pool. creating new image"
    	    #todo: check succesful creation
    	    $DESTRBDCMD create $DESTPOOL/$LOCAL_IMAGE -s 1
        fi

        #create today's snapshot
        if [[ -z $($SOURCERBDCMD snap ls $SOURCEPOOL/$LOCAL_IMAGE | grep $TODAY) ]]; then
	        echo "info: creating snapshot $SOURCEPOOL/$LOCAL_IMAGE@$TODAY"
	        $SOURCERBDCMD snap create $SOURCEPOOL/$LOCAL_IMAGE@$TODAY
        else
	        echo "warning: source image $SOURCEPOOL/$LOCAL_IMAGE@$TODAY already exists"
        fi

        # check whether to do a init or a full
        if [[ -z $($DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE) ]]; then
	        echo "info: no snapshots found for $DESTPOOL/$LOCAL_IMAGE doing init"
	        $SOURCERBDCMD export-diff $SOURCEPOOL/$LOCAL_IMAGE@$TODAY - | $DESTRBDCMD import-diff - $DESTPOOL/$LOCAL_IMAGE
        else
	        echo "info: found previous snapshots for $DESTPOOL/$LOCAL_IMAGE doing diff"
    	    #check yesterday's snapshot exists at remote pool
    	    if [[ -z $($DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE | grep $YESTERDAY) ]]; then
	            echo "error: --from-snap $LOCAL_IMAGE@$YESTERDAY does not exist on remote pool"
		        continue
	        fi
    	    #check todays's snapshot already exists at remote pool
	        if [[ -z $($DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE | grep $TODAY) ]]; then
	    
		        #Used tee to send the exported diff to both the import-diff and to md5sum
	            $SOURCERBDCMD export-diff --from-snap $YESTERDAY $SOURCEPOOL/$LOCAL_IMAGE@$TODAY -|tee >($DESTRBDCMD import-diff - $DESTPOOL/$LOCAL_IMAGE)|md5sum|cut -d ' ' -f1 >/var/tmp/ceph_hash
	            SOURCE_HASH="$(cat /var/tmp/ceph_hash"
	            #comparing changed extents between source and destination
    	        DEST_HASH=`$DESTRBDCMD export-diff --from-snap $YESTERDAY $DESTPOOL/$LOCAL_IMAGE@$TODAY - | md5sum | cut -d ' ' -f 1`
    	        if [ $SOURCE_HASH == $DEST_HASH ]; then
	        	    echo "info: changed extents hash check ok"

		            #Delete yesterdays snapshot from source
		            $SOURCERBDCMD snap rm $SOURCEPOOL/$LOCAL_IMAGE@$YESTERDAY
                    # Delete old snapshots from destination
                        deleteOldSnaps
                    
	            else
		            echo "error: changed extents hash on source and destination don't match: $SOURCE_HASH not equals $DEST_HASH"
	            fi
	        else
	            echo "error: snapshot $DESTPOOL/$LOCAL_IMAGE@$TODAY already exists, skipping"
	        
	        fi
        fi

    done
}

#Delete snapshots older than 35 days from destination cluster
deleteOldSnaps() {
    keepDate=$(date +%Y%m%d --date="35 days ago")
    #get all the snapshots of the image
    snaps=$($DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE|awk '{print $2}'|grep "^bkupsnap")
    
    for snap in $snaps; do
        snapDate=$(echo $snap|tr -d [:alpha:]|tr -d '-') #extract snapshot date
        if [[ $keepDate -gt $snapDate ]]; then
            $($DESTRBDCMD snap rm $DESTPOOL/$LOCAL_IMAGE@$snap)
        fi
    done
}

#what is today's date?
TODAY=`date +"bkupsnap%Y-%m-%d"`
YESTERDAY=`date +"bkupsnap%Y-%m-%d" --date="1 days ago"`

SOURCECLUSTER="prod"
DESTCLUSTER="backup"
SOURCERBDCMD="rbd --cluster $SOURCECLUSTER --no-progress"
DESTRBDCMD="rbd --cluster $DESTCLUSTER --no-progress"

#Get the list of all the source pools
SOURCEPOOLS="$(rados --cluster cephprod lspools)"

exec >/var/tmp/ceph_sync.log 2>&1
date

#Mirror each rbd pool in turn
for pool in $SOURCEPOOLS; do
    if [[ -n $($SOURCERBDCMD ls) ]]; then #mirror only the pools that have rbds
	    SOURCEPOOL="$pool"
	    DESTPOOL="$pool"
        echo
	    echo "=========================="
	    echo "Info: Mirroring pool $pool"
	    echo "=========================="
	    mirrorPool
    fi
done

if [[ -z $(grep error /var/tmp/ceph_sync.log) ]]; then
    cat /var/tmp/ceph_sync.log|mailx -s "Ceph sync- Successfull" XX@YY.com.au
else
    cat /var/tmp/ceph_sync.log|mailx -s "Ceph sync- FAILED" XX@YY.com.au
fi
date