Last active
June 12, 2023 17:48
-
-
Save eliphaslevy/9b633899b227bd6964daef469755fb6a to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# gist.github.com/eliphaslevy | |
# License: public domain | |
# Purpose: monitor interface for traffic spikes, capture some of it, and alert via email | |
# Run with environment DEBUG=1 set to see its output to terminal, | |
# otherwise will use 'logger' to write to syslog. | |
# I have put it on /etc/cron.d/cron.hourly to ensure it keeps running. | |
MYNAME=$(basename $0) | |
# debug to stdout instead of syslog | |
if [ "${DEBUG:-0}" != "0" ] ; then | |
logger() { echo "$(date) $@" ; } | |
fi | |
trap 'errno=$?; logger "$MYNAME PID $$ stop. errno:$errno" ; exit ${errno}' EXIT INT TERM | |
# exclusive lock | |
lockdir=/var/run/$(basename $0) | |
if [ "${MYLOCK:-}" != "locked" ]; then | |
logger "$MYNAME start. PARENT=$$" | |
if [ ! -d "$lockdir" ]; then | |
rm -f "$lockdir" 2>&1 || true | |
mkdir "$lockdir" || exit 13 | |
fi | |
export MYLOCK=locked | |
export DEBUG=${DEBUG:-0} | |
exec nice ionice flock --nonblock -E 0 -e "$lockdir" /bin/bash $0 "$@" | |
exit $? | |
fi | |
# personalization | |
email=youremail@example.com | |
dumpdir=/root/captures | |
dumpuser=root | |
dumpgroup=root | |
# interface to monitor | |
interface=$(ip route ls|sed -n '/default/{s/.* dev \([^ ]\+\).*/\1/;p;q;}') | |
packet_threshold=5000000 # what is "too much bandwidth?" | |
buffer_size=120 # seconds of high bandwidth sustained, to trigger an email | |
log_packets=10000 # packets to log on tcpdump each time the alarm is triggered | |
log_timeout=15 # timeout for waiting the capture to finish - if the burst stops early | |
mail_wait=300 # one email each five minutes is ok | |
mail_counter=3 # but do not send more than 3 consecutive ones | |
# sent mail counter - is reset on each burst end | |
mails_sent=0 | |
# keep $keep_days, remove older | |
keep_days=3 | |
install -d -m 750 -o $dumpuser -g $dumpgroup $dumpdir | |
logger "$MYNAME start: CHILD=$$ INTERFACE=$interface THRESHOLD=$packet_threshold BUFFER=$buffer_size" | |
echo "$$" >$lockdir/$MYNAME.pid | |
# save my stat - if I am updated, I will re-exec myself | |
MYSTAT=$(stat -c %y $0) | |
cleanup_dumps() { | |
find "$dumpdir" -type f -name "dump-${HOSTNAME}-*.cap" -mtime +$keep_days | while read fn ; do | |
logger "$MYNAME rm old dump: $fn" | |
rm "$fn" | |
done | |
} | |
do_dump() { | |
logger "$MYNAME high traffic: $pkt/s - starting a tcpdump" | |
fname=$dumpdir/dump-$HOSTNAME-$(date +"%Y%m%d_%H%M%S").cap | |
timeout $log_timeout tcpdump -n -i $interface -c $log_packets -w "$fname" | |
chown "$username:$groupname" "$fname" | |
logger "$MYNAME packets dumped to $fname" | |
cleanup_dumps | |
if [ "$mails_sent" -gt "$mail_counter" ] ; then # stop emails. | |
logger "$MYNAME won't send email, $mails_sent consecutive ones" | |
return | |
fi | |
logger "$MYNAME sending email alert" | |
echo "$MYNAME captured file: $fname" | mail -s "PROBLEM: $pkt/s traffic on $HOSTNAME" "$email" | |
mails_sent=$((mails_sent+1)) | |
} | |
declare -a buffer | |
while true; do | |
pkt_old=$(sed -n "/^ *$interface: /s/$interface: \([0-9]\+\) .*/\1/p" /proc/net/dev) | |
sleep 1 | |
pkt_new=$(sed -n "/^ *$interface: /s/$interface: \([0-9]\+\) .*/\1/p" /proc/net/dev) | |
pkt=$(( $pkt_new - $pkt_old )) | |
[ "${DEBUG:-0}" == "1" ] && logger "$MYNAME $pkt/s" | |
if [ $pkt -gt $packet_threshold ]; then | |
logger "$MYNAME #${#buffer[*]} $pkt/s - above threshold." | |
buffer+=($pkt) | |
if [ "${#buffer[*]}" -ge "$buffer_size" ] ; then | |
do_dump | |
logger "$MYNAME sleeping $mail_wait seconds." | |
sleep $mail_wait | |
buffer=() # reset buffer | |
fi | |
else | |
mails_sent=0 # reset mail counter | |
buffer=() # reset buffer | |
fi | |
CURSTAT=$(stat -c %y $0) | |
if [ "$MYSTAT" != "$CURSTAT" ]; then | |
logger "$MYNAME restart - status changed from '$MYSTAT' to '$CURSTAT'." | |
exec $0 $LOCKED_ARG | |
fi | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment