Backup and restore cassandra table for a fresh start (e.g. when you're RIPped by tombstones)

backup_restore_cassandra.sh

#!/usr/bin/env bash
# RUN THIS ON EACH CASSANDRA NODE!

DEBUG=${DEBUG:-true} # change to false or run as 'DEBUG=false backup_restore_cassandra.sh' in prod

CQLSH=${CQLSH:-cqlsh} # pass required parameters if needed
KEYSPACE_NAME=${KEYSPACE_NAME:-profile}
TABLE_NAME=${TABLE_NAME:-device}
SNAPSHOT_TAG=${SNAPSHOT_TAG:-${TABLE_NAME}_`date +%Y%m%d_%H%M%S`}
KEYSPACE_DIRS="/dcos/volume*/${KEYSPACE_NAME}" # change appropriately!
RESTART_CMD="sudo systemctl restart cassandra.service"

RUN=""
if ${DEBUG}
then
  RUN=echo
fi
confirm() {
    # call with a prompt string or use a default
    read -r -p "${1:-Are you sure? [y/N]} " response
    case "$response" in
        [yY][eE][sS]|[yY]) 
            true
            ;;
        *)
            false
            ;;
    esac
}

confirm "Remove all existing snapshots for ${KEYSPACE_NAME}.${TABLE_NAME} table? [y/N]" && {
  for f in ${KEYSPACE_DIRS}/${TABLE_NAME}-*
    do 
      if [ -d "$f/snapshots" ]
      then
          $RUN sudo rm -rf $f/snapshots
      fi
    done
}
echo "Making snapshots."
$RUN nodetool snapshot -t ${SNAPSHOT_TAG} --table ${TABLE_NAME} ${KEYSPACE_NAME} 
# DANGEROUS ZONE
echo "ATTENTION: next command will destroy all data in your table."
echo "This should only be done once after all other nodes finished doing their snapshots."
echo "Please manually run next command using cqlsh:"
echo -e "\n\t${CQLSH} -e \"TRUNCATE TABLE ${KEYSPACE_NAME}.${TABLE_NAME}\"\n"
confirm "Enter Y after the command has finished:" && {
  for f in ${KEYSPACE_DIRS}/${TABLE_NAME}-*
    do 
      # ls $f/snapshots/${SNAPSHOT_TAG}
      if [ -d "$f/snapshots/${SNAPSHOT_TAG}" ]
      then
          $RUN sudo cp -a $f/snapshots/${SNAPSHOT_TAG}/* $f/
      fi
    done

  echo "Refreshing nodes now."
  $RUN nodetool refresh ${KEYSPACE_NAME} ${TABLE_NAME} 
  echo "Restarting Cassandra node now."
  $RESTART_CMD
}