aerospike alert

aerospike_space_alert.sh

#!/bin/bash
SlackChannelName=abc
SlackHookURL="abc"
MessageBody=
MessageTitle="CRITICAL:/aerospike_space_alert.sh [$(hostname)/$(hostname -i)]"
AerospikeIP=192.168.0.1:3000
OLDIFS=$IFS; IFS=$'\n'
#HWMDisk=60
#HWMMem=75
HWMDisk=55
HWMMem=72
ALERT_INTERVAL=900
for namespace in $(asinfo -h ${AerospikeIP} -v "namespaces" -l); do 
	Alerts=()
	ALERT_FILE="/tmp/aerospike_ns_${namespace}_sg.txt"
	for details in $(asadm -h ${AerospikeIP} -e "asinfo -v 'namespace/\"$namespace' -l" | grep "memory_free_pct\|device_free_pct\|returned:"| awk '{ printf "%s", $0; if (NR % 3 == 0) print ""; else printf " " }'); do
		hostname=$(echo $details | awk '{print $(NF-3)}'|tr -d "(|)")
		memory_free_pct=$(echo $details | awk '{print $(NF-1)}')
		device_free_pct=$(echo $details | awk '{print $(NF)}')
		memory_used_pct=$(expr 100 - ${memory_free_pct##*=})
		disk_used_pct=$(expr 100 - ${device_free_pct##*=})
		if [ "${memory_used_pct}" -gt "${HWMMem}" ];then
			Alerts=("${Alerts[@]}" "\\\n${hostname}:${namespace}=")
			Alerts=("${Alerts[@]}" "\\\tmemory_used_pct(>${HWMMem}%):${memory_used_pct}%")
		fi
		if [ "${disk_used_pct}" -gt "${HWMDisk}" ];then
			Alerts=("${Alerts[@]}" "\\\tdisk_used_pct(>${HWMMem}%):${disk_used_pct}%")
		fi
		unset hostname memory_free_pct device_free_pct memory_used_pct disk_used_pct
	done
	if [ "${#Alerts[@]}" -gt 0 ];then
		if [ ! -f "$ALERT_FILE" ]; then
		     touch ${ALERT_FILE}
		     EXECUTE=true
		else
		    if [ $(stat --format=%Y $ALERT_FILE) -le $(( $(date +%s) - ${ALERT_INTERVAL} )) ]; then
			EXECUTE=true
		    fi
		fi
		if [ "$EXECUTE" = "true" ]; then
			MessageBody="Aerospike:$(printf '\n%s' "${Alerts[@]}")"
			[ -n "$MessageBody" ] && statusCode=$(bash -x /custom-scripts/postToSlack -t "$MessageTitle" -b "$MessageBody" -c "$SlackChannelName" -u "$SlackHookURL" -s "critical")
			touch ${ALERT_FILE}
		fi
	else
        	if [ -f "$ALERT_FILE" ]; then
                	MessageBody=$(echo -e "Aerospike:\nNo worries, it is back to normal!!")
                	[ -n "$MessageBody" ] && bash -x /custom-scripts/postToSlack -t "$MessageTitle" -b "$MessageBody" -c "$SlackChannelName" -u "$SlackHookURL" -s "ok"
                	rm -f ${ALERT_FILE}
        	fi
	fi
	unset Alerts
done

aerospike_xdr1.sh

#!/bin/bash
SlackChannelName=
SlackHookURL=""
MessageBody=
MessageTitle="CRITICAL:/aerospike_xdr.sh [$(hostname)/$(hostname -i)]"
OLDIFS=$IFS; IFS=$'\n'
ALERT_FILE='/tmp/aer_xdr_alert_1.txt'
OUTPUT_FILE='/tmp/aer_xdr_output_1.txt'
ALERT_INTERVAL=900
asadm -h 192.168.0.1:3000 -e "info xdr" > ${OUTPUT_FILE}
output=$(awk '/^aerospike/{
++count
n=split($6,a,":"); 
totaltime+=(a[1]*3600+a[2]*60+a[3])
}END{
	if (count > 0) {
		avgtime = totaltime / count
		printf("\\nTotal Time/%d nodes=%02d:%02d:%02d(HH:MM:SS)\\nAvg Time/per node=%02d:%02d:%02d(HH:MM:SS)", count,totaltime/3600,totaltime%3600/60,totaltime%3600%60,avgtime/3600,avgtime%3600/60,avgtime%3600%60)
	}
}' ${OUTPUT_FILE})
hours=$(echo -e "$output" | awk -F : '/^Avg/{print $1}') 
if [ ${hours##*=} -ge 1 ]; then
	if [ ! -f "$ALERT_FILE" ]; then
             touch ${ALERT_FILE}
             EXECUTE=true
        else
            if [ $(stat --format=%Y $ALERT_FILE) -le $(( $(date +%s) - ${ALERT_INTERVAL} )) ]; then
             	EXECUTE=true
            fi
       	fi
        if [ "$EXECUTE" = "true" ]; then
		MessageBody=$(echo -e "Aerospike Replication Delay:\n${output}")
      		[ -n "$MessageBody" ] && bash -x /custom-scripts/postToSlack -t "$MessageTitle" -b "$MessageBody" -c "$SlackChannelName" -u "$SlackHookURL" -s "critical"
                touch ${ALERT_FILE}
        fi
else
	if [ -f "$ALERT_FILE" ]; then
		MessageBody=$(echo -e "Aerospike Replication Delay:\nNo worries, it is back to normal!!")
		[ -n "$MessageBody" ] && bash -x /custom-scripts/postToSlack -t "$MessageTitle" -b "$MessageBody" -c "$SlackChannelName" -u "$SlackHookURL" -s "ok"
		rm -f ${ALERT_FILE}
	fi
fi

aerospike_xdr2.sh

#!/bin/bash
SlackChannelName=
SlackHookURL=""
MessageBody=
MessageTitle="CRITICAL:/aerospike_xdr.sh [$(hostname)/$(hostname -i)]"
OLDIFS=$IFS; IFS=$'\n'
ALERT_FILE='/tmp/aer_xdr_alert_2.txt'
ALERT_INTERVAL=900
AerospikeIP=192.168.0.1:3000
output=$(asinfo -h ${AerospikeIP} -v 'statistics' -l | grep xdr_timelag)
echo output=${output}
if [ ${output##*=} -ge 3600 ]; then
	if [ ! -f "$ALERT_FILE" ]; then
             touch ${ALERT_FILE}
             EXECUTE=true
        else
            if [ $(stat --format=%Y $ALERT_FILE) -le $(( $(date +%s) - ${ALERT_INTERVAL} )) ]; then
             	EXECUTE=true
            fi
       	fi
        if [ "$EXECUTE" = "true" ]; then
		MessageBody=$(printf 'Aerospike Replication Delay:\t%dd:%dh:%dm:%ds' $((${output##*=}/86400)) $((${output##*=}%86400/3600)) $((${output##*=}%3600/60)) $((${output##*=}%60)))
      		[ -n "$MessageBody" ] && bash -x /custom-scripts/postToSlack -t "$MessageTitle" -b "$MessageBody" -c "$SlackChannelName" -u "$SlackHookURL" -s "critical"
                touch ${ALERT_FILE}
        fi
else
	if [ -f "$ALERT_FILE" ]; then
		MessageBody=$(printf 'Aerospike Replication Delay:\n%dd:%dh:%dm:%ds\t[No worries, it is back to normal!!]' $((${output##*=}/86400)) $((${output##*=}%86400/3600)) $((${output##*=}%3600/60)) $((${output##*=}%60)))
		[ -n "$MessageBody" ] && bash -x /custom-scripts/postToSlack -t "$MessageTitle" -b "$MessageBody" -c "$SlackChannelName" -u "$SlackHookURL" -s "ok"
		rm -f ${ALERT_FILE}
	fi
fi