Skip to content

Instantly share code, notes, and snippets.

@eliphaslevy
Last active June 12, 2023 17:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save eliphaslevy/9b633899b227bd6964daef469755fb6a to your computer and use it in GitHub Desktop.
Save eliphaslevy/9b633899b227bd6964daef469755fb6a to your computer and use it in GitHub Desktop.
#!/bin/bash
# gist.github.com/eliphaslevy
# License: public domain
# Purpose: monitor interface for traffic spikes, capture some of it, and alert via email
# Run with environment DEBUG=1 set to see its output to terminal,
# otherwise will use 'logger' to write to syslog.
# I have put it on /etc/cron.d/cron.hourly to ensure it keeps running.
MYNAME=$(basename $0)
# debug to stdout instead of syslog
if [ "${DEBUG:-0}" != "0" ] ; then
logger() { echo "$(date) $@" ; }
fi
trap 'errno=$?; logger "$MYNAME PID $$ stop. errno:$errno" ; exit ${errno}' EXIT INT TERM
# exclusive lock
lockdir=/var/run/$(basename $0)
if [ "${MYLOCK:-}" != "locked" ]; then
logger "$MYNAME start. PARENT=$$"
if [ ! -d "$lockdir" ]; then
rm -f "$lockdir" 2>&1 || true
mkdir "$lockdir" || exit 13
fi
export MYLOCK=locked
export DEBUG=${DEBUG:-0}
exec nice ionice flock --nonblock -E 0 -e "$lockdir" /bin/bash $0 "$@"
exit $?
fi
# personalization
email=youremail@example.com
dumpdir=/root/captures
dumpuser=root
dumpgroup=root
# interface to monitor
interface=$(ip route ls|sed -n '/default/{s/.* dev \([^ ]\+\).*/\1/;p;q;}')
packet_threshold=5000000 # what is "too much bandwidth?"
buffer_size=120 # seconds of high bandwidth sustained, to trigger an email
log_packets=10000 # packets to log on tcpdump each time the alarm is triggered
log_timeout=15 # timeout for waiting the capture to finish - if the burst stops early
mail_wait=300 # one email each five minutes is ok
mail_counter=3 # but do not send more than 3 consecutive ones
# sent mail counter - is reset on each burst end
mails_sent=0
# keep $keep_days, remove older
keep_days=3
install -d -m 750 -o $dumpuser -g $dumpgroup $dumpdir
logger "$MYNAME start: CHILD=$$ INTERFACE=$interface THRESHOLD=$packet_threshold BUFFER=$buffer_size"
echo "$$" >$lockdir/$MYNAME.pid
# save my stat - if I am updated, I will re-exec myself
MYSTAT=$(stat -c %y $0)
cleanup_dumps() {
find "$dumpdir" -type f -name "dump-${HOSTNAME}-*.cap" -mtime +$keep_days | while read fn ; do
logger "$MYNAME rm old dump: $fn"
rm "$fn"
done
}
do_dump() {
logger "$MYNAME high traffic: $pkt/s - starting a tcpdump"
fname=$dumpdir/dump-$HOSTNAME-$(date +"%Y%m%d_%H%M%S").cap
timeout $log_timeout tcpdump -n -i $interface -c $log_packets -w "$fname"
chown "$username:$groupname" "$fname"
logger "$MYNAME packets dumped to $fname"
cleanup_dumps
if [ "$mails_sent" -gt "$mail_counter" ] ; then # stop emails.
logger "$MYNAME won't send email, $mails_sent consecutive ones"
return
fi
logger "$MYNAME sending email alert"
echo "$MYNAME captured file: $fname" | mail -s "PROBLEM: $pkt/s traffic on $HOSTNAME" "$email"
mails_sent=$((mails_sent+1))
}
declare -a buffer
while true; do
pkt_old=$(sed -n "/^ *$interface: /s/$interface: \([0-9]\+\) .*/\1/p" /proc/net/dev)
sleep 1
pkt_new=$(sed -n "/^ *$interface: /s/$interface: \([0-9]\+\) .*/\1/p" /proc/net/dev)
pkt=$(( $pkt_new - $pkt_old ))
[ "${DEBUG:-0}" == "1" ] && logger "$MYNAME $pkt/s"
if [ $pkt -gt $packet_threshold ]; then
logger "$MYNAME #${#buffer[*]} $pkt/s - above threshold."
buffer+=($pkt)
if [ "${#buffer[*]}" -ge "$buffer_size" ] ; then
do_dump
logger "$MYNAME sleeping $mail_wait seconds."
sleep $mail_wait
buffer=() # reset buffer
fi
else
mails_sent=0 # reset mail counter
buffer=() # reset buffer
fi
CURSTAT=$(stat -c %y $0)
if [ "$MYSTAT" != "$CURSTAT" ]; then
logger "$MYNAME restart - status changed from '$MYSTAT' to '$CURSTAT'."
exec $0 $LOCKED_ARG
fi
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment