ElasticSearch log index backup & restore scripts

backup.sh

#!/bin/bash
# herein we backup our indexes! this script should run at like 3 AM every first day of the month, after logstash
# rotates to a new ES index and theres no new data coming in to the old one. we grab metadatas,
# compress the data files and backs up to whatever long term storage.

. ./config.sh

echo "Checking that index exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
then
  echo "Index $INDEXNAME doesn't exist, nothing to backup"
  exit 0
fi

# create mapping file with index settings. this metadata is required by ES to use index file data
echo -n "Backing up metadata of index $INDEXNAME ... "
curl -XGET -o $TMPDIR/mapping $ESURL"/$INDEXNAME/_mapping?pretty=true" > /dev/null 2>&1
sed -i '1,2d' $TMPDIR/mapping #strip the first two lines of the metadata
echo '{"settings":{"number_of_shards":5,"number_of_replicas":1},"mappings":{' > $MAPPING
# prepend hardcoded settings metadata to index-specific metadata
cat $TMPDIR/mapping >> $MAPPING
echo "DONE!"

# now lets tar up our data files. these are huge, so lets be nice
echo -n "Backing up data files of index $INDEXNAME (this may take some time) ... "
mkdir -p $BACKUPDIR
cd $INDEXDIR
nice -n 19 tar -zcf $BACKUPDIR/$INDEXNAME.tar.gz $INDEXNAME
echo "DONE!"

# push both tar.gz and metadatas to tape
echo -n "Saving to tape (this may take some time) ..."
$BACKUPCMD $BACKUPDIR/$INDEXNAME.tar.gz $BACKUPTARGET.tar.gz
$BACKUPCMD $MAPPING $BACKUPTARGET-mapping.json
echo "DONE!"

# cleanup tmp files
rm $TMPDIR/mapping

config.sh

#!/bin/bash

# We want to archive previous index, here last month index
INDEXNAME="logs-"`date --date="last month" +"%Y-%m"` # this had better match the index name in ES

INDEXDIR="/cygdrive/d/Data/cedric.vidal/apps/elasticsearch-0.20.2-node-1/data/cls-log-test/nodes/0/indices"
CURDIR="/cygdrive/d/Data/cedric.vidal/AMQ/CLS/backup-es"

# Local configuration
BACKUPCMD="cp"
BACKUPTARGET=$CURDIR"/es-tape/$INDEXNAME"

# S3 Configuration
# BACKUPCMD="/usr/local/backupTools/s3cmd --config=/usr/local/backupTools/s3cfg put"
# BACKUPTARGET="s3://backups/elasticsearch/$INDEXNAME"

BACKUPDIR=$CURDIR"/es-backups"

ESURL="http://localhost:9200"
TMPDIR=$CURDIR"/tmp"
MAPPING=$BACKUPDIR/$INDEXNAME-mapping.json
RESTARTCMD="" #/etc/init.d/es restart"

delete-index.sh

#!/bin/bash

. ./config.sh

echo "Checking that index exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
then
  echo "Index $INDEXNAME doesn't exist, nothing to delete"
  exit 0
fi

echo -n "Deleting index $INDEXNAME ... "

curl -XDELETE "$ESURL/$INDEXNAME/"> /dev/null 2>&1
echo "DONE!"

delete.sh

#!/bin/bash

. ./config.sh

TIMESTAMPFAIL=`curl -s $ESURL/_status?pretty=true |grep index |grep log |sort |uniq |awk -F\" '{print $4}' |grep 1970 |wc -l`
if [ -n $TIMESTAMPFAIL ]
then
curl -s $ESURL/_status?pretty=true |grep index |grep log |sort |uniq |awk -F\" '{print $4}' |grep 1970 | while read line
do
echo "Indices with screwed-up timestamps found; removing"
echo -n "Deleting index $line: "
curl -s -XDELETE $ESURL/$line/
echo "DONE!"
done
fi

echo "Checking that index exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
then
  echo "Index $INDEXNAME doesn't exist, nothing to delete"
  exit 0
fi

echo -n "Deleting index $INDEXNAME ... "

curl -XDELETE "$ESURL/$INDEXNAME/"> /dev/null 2>&1
echo "DONE!"

restore.sh

#!/bin/bash
# Performs 'rotation' of ES indices. Maintains only 8 indicies (1 week) of logstash logs; this script
# is to be run at midnight daily and removes the oldest one (as well as any 1970s-era log indices,
# as these are a product of timestamp fail). Please note the insane amount of error-checking
# in this script, as ES would rather delete everything than nothing…
# Before we do anything, let's get rid of any nasty 1970s-era indices we have floating around

. ./config.sh

echo "Checking that index doesn't already exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 1 ]
then
  echo "Index $INDEXNAME already exists, delete it before restoring. exiting"
  exit 0
fi

echo "Restoring index $INDEXNAME"

# create index and mapping
echo -n "Creating index and mappings ..."
curl -XPUT "$ESURL/$INDEXNAME/" -d @$MAPPING > /dev/null 2>&1
echo "DONE!"

# extract our data files into place
echo -n "Restoring index (this may take a while) ..."
cd $INDEXDIR
tar -zxvf $BACKUPDIR/$INDEXNAME.tar.gz
echo "DONE!"

# restart ES to allow it to open the new dir and file data
echo -n "Restarting Elasticsearch ..."
$RESTARTCMD
echo "DONE!"