backup solr cloud v2 courtesy of elyograg @ #solr

backup_solr_cloud.sh

#!/bin/bash

# Defaults.  Override in /etc/default/solr-backup.

# Unprivileged user/group that we will use to run this script.
B_USR=ubuntu
B_GRP=${B_USR}

# Domain SUFFIX.
SUFFIX=.dnsxSUFFIX.yourhost.com

#create mapping arrays for where to rsync from and to
SOURCE=(host1 host2 host3 host4)
TARGET=(host2 host3 host4 host1)
NUMHOSTS=4

# Source and destination dirs
LINK_SRC=/mnt/solr/data
LINK_DEST=/mnt/solr/backup_data
BACKUP_DEST=/mnt/backup

# the location of the id program
IDPROG=/usr/bin/id

DEFAULTS=/etc/default/solr-backup
if [ -f ${DEFAULTS} ];
then
  . ${DEFAULTS}
fi

B_ID=`${IDPROG} -u ${B_USR}`

if [ ${UID} -eq 0 ]; then
        exec su ${B_USR} -c "$0 $*"
elif [ ${UID} -eq ${B_ID} ]; then
        echo "do nothing" > /dev/null
else
        echo "This needs to be run as root or ${B_USR}: uid = ${UID} "
        exit 1
fi

let "LASTHOST = ${NUMHOSTS} - 1"

for i in {0..${LASTHOST}}
do
  LOCAL=${SOURCE[i]}
  REMOTE=${TARGET[i]}

  #remove previous backup so we don't fill up the disk...
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${REMOTE}${SUFFIX} "sudo rm -fR ${BACKUP_DEST}/${LOCAL}/"

  #prep receiving areas for solr index backups
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${REMOTE}${SUFFIX} "sudo mkdir -p ${BACKUP_DEST}/${LOCAL}/; sudo chown ${B_USR}.${B_GRP} ${BACKUP_DEST}/${LOCAL}/"

  #clean up old hardlinks if they're there
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${LOCAL}${SUFFIX} 'sudo rm -fR ${LINK_DEST}'

  #make backup location for local solr
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${LOCAL}${SUFFIX} 'sudo mkdir -p ${LINK_DEST} && sudo chown ${B_USR}.${B_GRP} ${LINK_DEST}'

  #make hard links to get index snapshot
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${LOCAL}${SUFFIX} 'sudo cp -lr ${LINK_SRC}/* ${LINK_DEST}/.'

  #rsync hard links to remote server, xargs for each directory via find to get better throughput
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${LOCAL}${SUFFIX} "cd ${LINK_DEST} && find . -maxdepth 1 -mindepth 1 -type d | xargs -n 1 -P 7 -I {} rsync -azh -e 'ssh -o StrictHostKeyChecking=no' {} ${B_USR}@${REMOTE}${SUFFIX}:${BACKUP_DEST}/${LOCAL}/"

  #remove hard links
  ssh -o 'StrictHostKeyChecking no' ${B_USR}@${LOCAL}${SUFFIX} "sudo rm -fR ${LINK_DEST}"

done

modify xargs -n 1 -P 7 if that hammers your host too hard. this assumes your solr lives at /mnt/solr, i have multiple cores per instance so i have /mnt/solr /mnt/solr1 /mnt/solr2 etc...