The script runs as a cron on one of our nodes (NC1) and executes nodetool status every minutes. It reports for the down nodes to datadog.

Cassandra Node health checkup

#!/bin/sh
#Check if the node on which the cron is running is down?
#if down the SELF_NODE value will node be empty.
SELF_NODE="$(nodetool status 2>&1| grep 'Failed')"


NODE="$(nodetool status | grep 'DN' | sed 's/ \{2,\}/ /g' | cut -d' ' -f2 | head -n1)"



#If no node down than exit
if [ "$NODE" == "" ] && [ "$SELF_NODE" == "" ]
then
  echo "No Node down"
  exit
fi

if [ "$NODE" == "192.168.X.X3" ]
then
  TRIGGER_NODE="NC4"
elif [ "$NODE" == "192.168.X.X4" ]
then
  TRIGGER_NODE="NC5"
elif [ "$NODE" == "192.168.X.X5" ]
then
  TRIGGER_NODE="NC2"
elif [ "$NODE" == "192.168.X.X2" ]
then
  TRIGGER_NODE="NC6"
elif [ "$NODE" == "192.168.X.X1" ]
then
  TRIGGER_NODE="NC3"
else
  TRIGGER_NODE="NC1" //
fi

TEXT="CASANDRA NODE "$TRIGGER_NODE" DOWN"
echo $TEXT

curl  -X POST -H "Content-type: application/json" \
-d '{
      "title": "'"$TEXT"'",
      "text": "Cassandra Node Down",
      "priority": "normal",
      "alert_type": "error",
      "tags": "'"$TRIGGER_NODE"'"
  }' \
'https://app.datadoghq.com/api/v1/events?api_key=[API_KEY]'