backup_es.sh

#!/bin/bash

TODAY=`date +"%Y%m%d"`
DATADIR="/var/lib/elasticsearch/data"

BACKUPDIR="/mnt/elasticsearch_snapshots/$TODAY"

# now lets tar up our data files. these are huge, so lets be nice
for idx in `find $DATADIR -mindepth 5 -maxdepth 5 -type d -wholename '*/indices/*'`; do
  indexname=`echo $idx | awk 'BEGIN {FS="/"};{print $NF}'`
  mkdir -p $BACKUPDIR/

  # create mapping file with index settings. this metadata is required by ES to use index file data
  echo "Backing up metadata..."
  curl -XGET "http://localhost:9200/$indexname/_settings?pretty=true" 2>/dev/null | sed '1,2d' | sed '$d' | sed '$d' > $BACKUPDIR/settings.$indexname
  curl -XGET "http://localhost:9200/$indexname/_mapping?pretty=true" 2>/dev/null | sed '1,2d' | sed '$d' | sed '$d' > $BACKUPDIR/mappings.$indexname

  echo "Creating restore script"
  # time to create our restore script! oh god scripts creating scripts, this never ends well…
  cat << EOF > $BACKUPDIR/$indexname.restore.sh
#!/bin/bash
# this script requires $indexname.tar and will restore it into elasticsearch
# it is ESSENTIAL that the index you are restoring does NOT exist in ES. delete it
# if it does BEFORE trying to restore data.

# create index and mapping
echo "Creating index and mappings… "
curl -XPUT 'http://localhost:9200/$indexname' -d '{`cat $BACKUPDIR/settings.$indexname`,"mappings":{`cat $BACKUPDIR/mappings.$indexname`}}'
echo

# extract our data files into place
echo "Restoring index (this may take a while)..."
tar xvf $indexname.tar -C $DATADIR --strip-components=4
echo
EOF
  rm $BACKUPDIR/settings.$indexname
  rm $BACKUPDIR/mappings.$indexname

  echo "Flushing index $indexname"
  curl -XPOST "http://localhost:9200/$indexname/_flush"
  echo

  echo "Disabling flush of transaction logs for $indexname"
  curl -XPUT "http://localhost:9200/$indexname/_settings?pretty=true" -d '{"index":{"translog":{"disable_flush":"true"}}}'
  echo

  echo "Backing up data files for $indexname"
  nice -n 19 tar cf $BACKUPDIR/$indexname.tar -C $DATADIR $idx

  echo "Enabling flush of transaction logs for $indexname"
  curl -XPUT "http://localhost:9200/$indexname/_settings?pretty=true" -d '{"index":{"translog":{"disable_flush":"false"}}}'
  echo
done

echo "Don't forget to restart ES after restoring indexes."