pbkwee/bulkspamban.sh

## bulkspamban.sh
#!/bin/bash
# Copyright 2018 Peter Bryant
# Licensed as https://www.apache.org/licenses/LICENSE-2.0

[ ! -z "$1" ] && [ "$1" != "--drop" ] && cat << EOJ && exit 0
Create a folders (SPAM_FOLDER=${SPAM_FOLDER:-unset} variable).  Load it up with spam.

This script searches spam folders to find IP subnets (/24s) that have a sent a good amount of spam.

Output commands to move those away and to block those IP ranges.

Add --drop to move those away and block the IPs automatically.

EOJ
[ "$1" = "--drop" ]  && IS_DROP='y'

[ ! -f /etc/bulkspamban.conf ] && cat <<EOJ > /etc/bulkspamban.conf
# directory where known spam is stored.  We will extract spammer IP ranges from here.
ARRAY_SPAM_FOLDERS=( "/var/vmail/user@domain.com/.Junk.sa-filtered/cur/" )

# spam matching the spammer IP ranges will be listed from these folders (and moved out of them if the drop argument is specified)
ARRAY_MAIL_FOLDERS=("/var/vmail/user@domain/cur/" "/var/vmail/user2@domain/cur/")

# the folder where spam from ARRAY_MAIL_FOLDERS will be moved
MOVETO_FOLDER="/var/vmail/user@domain/.Junk/cur/"

COMPLAINT_FROM_EMAIL_ADDRESS=user@domain.com

WHITELIST_ABUSE_REGEX="@google.com|@google.com"

WHITELIST_RECEIVED_REGEX="somehostname|someotheriop"

EOJ

NUMDAYSTOCHECK=${NUMDAYSTOCHECK:-10}
NUMDAYSTOREMOVE=${NUMDAYSTOREMOVE:-$((NUMDAYSTOCHECK*3))}
[ -f /etc/bulkspamban.conf ] && source /etc/bulkspamban.conf

if [ -z "$ARRAY_SPAM_FOLDERS" ] || grep -qai '^ARRAY_SPAM_FOLDERS.*=.*user\@domain' /etc/bulkspamban.conf; then
  echo "Edit the ARRAY_SPAM_FOLDERS setting in /etc/bulkspamban.conf before proceeding." 2>&1
  exit 1
fi


# convert the array into a string that can be used by the find command.
MAIL_FOLDERS_LIST="$(for index in ${!ARRAY_MAIL_FOLDERS[*]};do [  -d ${ARRAY_MAIL_FOLDERS[$index]} ] && echo -n ${ARRAY_MAIL_FOLDERS[$index]} " "; done)"
#MAIL_FOLDERS_LIST="$(for index in ${!ARRAY_MAIL_FOLDERS[*]};do [  -d \"${ARRAY_MAIL_FOLDERS[$index]}\" ] && echo -n \"${ARRAY_MAIL_FOLDERS[$index]}\"; done)"
#MAIL_FOLDERS_LIST="$(for index in ${!ARRAY_MAIL_FOLDERS[*]};do [  -d "${ARRAY_MAIL_FOLDERS[$index]}" ] && echo -n "${ARRAY_MAIL_FOLDERS[$index]} "; done)"
SPAM_FOLDERS_LIST="$(for index in ${!ARRAY_SPAM_FOLDERS[*]};do [  -d ${ARRAY_SPAM_FOLDERS[$index]} ] && echo -n ${ARRAY_SPAM_FOLDERS[$index]} " "; done)"
for SPAM_FOLDER in $SPAM_FOLDERS_LIST; do
  [ ! -d "$SPAM_FOLDER" ] && echo "Set ARRAY_SPAM_FOLDERS= to a directory containing spam.  e.g. in /etc/bulkspamban.conf.  e.g. to /var/vmail/email@address.com/.Junk.sa-filtered/cur/" && exit 1
done

# find /24 ranges sending us 4 or more emails classified as spam in the last few days.

#Received: from mail-yw0-f202.google.com (mail-yw0-f202.google.com [209.85.161.202])
#  by mail.domain (Postfix) with ESMTPS id BD68A6177C
#  for <user@domain>; Tue, 13 Mar 2018 23:35:51 +0000 (UTC)
# in one email can be multiple received froms:
#Received: from mail.example.com (localhost [127.0.0.1])
#Received: from mail1.example.st (mail1.example.st [206.123.115.88])
#Received: from example.com (unknown [162.244.12.88])

#suspect_nets=$(grep 'Received: from' $(find $SPAM_FOLDERS_LIST -mtime -${NUMDAYSTOCHECK} -type f) | tail -n 1 | egrep -v '127.0.0.1|userid|with local' | sed 's/.*from //g' | sed 's/by .*//g' | sed  -e 's/.*\[\([0-9]*\.[0-9]*\.[0-9]*\.\).*/\1/g'  | sort | egrep -v '^10.|^192.168|[a-zA-Z]' | uniq -c | sort -n | awk '{ if ($1 > 4) {print $2} }')
[ -z "${WHITELIST_RECEIVED_REGEX}" ] && WHITELIST_RECEIVED_REGEX="nomatchhere"
suspect_ips=
suspect_nets=
for file in $(find $SPAM_FOLDERS_LIST -mtime -${NUMDAYSTOCHECK} -type f); do
  # e.g.
  # Received: from peelregion.ca (unknown [104.193.255.117])
  #Received: from 10.214.167.174
  #Received: from 209.85.221.43 (EHLO mail-wr1-f43.google.com)
  #received_froms="$(grep 'Received: from' $file  | egrep -v '127.0.0.1|userid|with local')"
  received_ips="$(grep 'Received: from' $file  | egrep -v "${WHITELIST_RECEIVED_REGEX}" | sed 's/.*\[//' | sed 's/\].*//'| egrep '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | sed  -e 's/[^0-9]*\([0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\).*/\1/g' | egrep -v '^10\.|^192.168|^127\.|[a-zA-Z]|^ *$')"
  #received_ips="$(echo "$received_froms" | egrep -v '\($' | sed 's/.*from //g' | sed 's/by .*//g' | sed 's/ (EHLO .*)//' | sed  -e 's/.*\[\([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\).*/\1/g' | egrep -v '^10\.|^192.168|^127\.[a-zA-Z]')"
  suspect_nets="$suspect_nets
$(echo "$received_ips" | sed  -e 's/\([0-9]*\.[0-9]*\.[0-9]*\.\).*/\1/g')
"

suspect_ips="$suspect_ips
$received_ips
"
done
suspect_nets=$(echo "$suspect_nets" | grep -v '^$' | sort | uniq -c | sort -n)
suspect_nets=$(echo "$suspect_nets" | awk '{ if ($1 > 4) {print $2} }')

suspect_ips=$(for i in $(echo "$suspect_ips" | grep -v '^$'); do echo $i | egrep -qai $(echo -n "$suspect_nets" | tr '\n' '|'; echo "|xxxx";)  && continue; echo $i; done | sort | uniq -c | sort -n)
echo "Suspect IPs:"
echo "$suspect_ips" | awk '{ if ($1 > 3 ) {print $1 " x " $2} }'
#echo "$suspect_ips" | awk '{ if ($1 > 3 && $1 <6 ) {print $1 "x " $2} }'
#echo "Bad IPs (would ban):"
#echo "$suspect_ips" | awk '{ if ($1 > 5 ) {print "# " $1 "x " $2; print "iptables -I INPUT -s " $1 " -j DROP -m comment --comment \'bulkspamban: email spam\'"} }'
# show the subjects for those emails

dropped=0
for i in $suspect_nets; do
  # was ${i}0 but that would pick up apnic vs. statically assigned operator
  iptables -L -n | grep -qai "DROP.* $i" && continue
  echo
  echo
  info=$(whois ${i}1 | egrep -i 'netname|OrgName|OrgAbuseEmail|CustName|NetName|Country|Organization-Name|org-name|@' | egrep -iv 'Comment:.*assigned staticall' | tr '*' ' ')
  emailaddy="$(echo "$info" | egrep  -o "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}\b" | sort | uniq | tr '\n' ' ' | tr ' ' ',' | tr '*' ' ' | tr '/' ' ' )"
  iptables -L -n | grep -qai "DROP.* $i" && echo "Already banned $info $emailaddy" && ISBANNED=Y && continue
  echo "Netblock: ${i}0/24"
  # outside quotes to show all on one line
  echo "To: " $emailaddy
  # no quotes to show on one line
  echo $info
  msg="Hi.  We received the following spam emails from your network block. Please reply with what action was taken to prevent further spam from being sent from your network. "
  echo "$msg"
  unset ISBANNED
  unset IS_WHITELISTED
  if [ ! -z "$IS_WHITELISTED" ] || [ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" | egrep -qai "$WHITELIST_ABUSE_REGEX"; then
    # skip MAIL_FOLDERS_LIST since that folder may contain real email from a whitelisted mail
    files="$(grep -l "Received: from .*\[$i" $(find $SPAM_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK}))"
  else
    files="$(grep -l "Received: from .*\[$i" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK}))"
  fi
  [ ! -z "$files" ] && subjects="$(egrep -h  "^Return-Path:|^Subject: |^Received: .*$i"  $files | sed "s/Received: .*$i/Received: $i/g" | sed 's/ by .*//g' | sed 's/])$//g')"
                #confl subjects="$(egrep -h  "^Subject: |^Received: .*$i" $(grep -l "Received: from .*\[$i" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) | sed "s/Received: .*$i/Received: $i/"  | sed 's/ by .*//' | sed 's/])$//')"
                #confl subjects="$(egrep -h  "^Subject: |^Received: .*$i" $(grep -l "Received: from .*\[$i" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) | sed "s/Received: .*$i/Received: $i/"  | sed 's/ by .*//' | sed 's/])$//')"
  echo "$subjects"
  [ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" | egrep -qai "$WHITELIST_ABUSE_REGEX" && echo "Email addresses ($emailaddy) are whitelisted.  Would not block with ip tables." && IS_WHITELISTED=Y && continue


  # manually run this (e.g. grep for DROP in output)
  [  -z "$ISBANNED" ] && echo "iptables -I INPUT -s ${i}0/24 -j DROP -m comment --comment 'bulkspamban: email spam'"
  if [ -z "$IS_DROP" ] ; then continue; fi

  if [ ! -z "$IS_WHITELISTED" ]; then
    true
  else
    [  -z "$ISBANNED" ] && iptables -I INPUT -s ${i}0/24 -j DROP -m comment --comment "bulkspamban: email spam $emailaddy $(echo info | egrep -i 'orgname|netname|org-name' | tr '\n' ' ' )" && echo "Dropped this range of IPs."
    [  -z "$ISBANNED" ] && dropped=$((dropped+1))
  fi
  if [  -z "$ISBANNED" ] && [ -z "$IS_WHITELISTED" ]; then
    grep -qai '^COMPLAINT_FROM_EMAIL_ADDRESS[A-Z].*=.*user\@domain' /etc/bulkspamban.conf && echo "Edit the COMPLAINT_FROM_EMAIL_ADDRESS setting in /etc/bulkspamban.conf before proceeding." 2>&1 && continue

    echo "$msg

$info

$subjects

Sample email with headers:

$(filearray=($files); [ ! -z "$files" ] && [ ! -z "${filearray[1]}" ] && cat "${filearray[1]}" | iconv -f utf-8 -t us-ascii//TRANSLIT | head -n 100 | head -c 6096 | sed 's#http:/#hxxp:/#g' | sed 's#https:/#hxxps:/#g')
" | mail -s "Spam abuse report from ${i}0/24" -a "From: $COMPLAINT_FROM_EMAIL_ADDRESS" -b "$COMPLAINT_FROM_EMAIL_ADDRESS" "$emailaddy"
  fi
  # remove their files from the sa-filtered folder
  grep -qai '^MOVED_TO.*=.*user\@domain' /etc/bulkspamban.conf && echo "Edit the MOVETO_FOLDER setting in /etc/bulkspamban.conf before proceeding." 2>&1 && continue

  [ ! -z "$MOVETO_FOLDER" ] && [ ! -d "$MOVETO_FOLDER" ]  && echo "MOVETO_FOLDER '$MOVETO_FOLDER' does not exist." && continue

  [ -z "$MOVETO_FOLDER" ] && echo "MOVETO_FOLDER not set in /etc/bulkspamban.conf, not moving files" && continue

  for j in $files; do
    [ "$(realpath $j)" == "$(realpath ${MOVETO_FOLDER}/$(basename $j))" ] && continue
    echo mv "\"$j\"" "$MOVETO_FOLDER"
    [ ! -z "$MOVETO_FOLDER" ] && [ -d "$MOVETO_FOLDER" ] && mv "$j" "$MOVETO_FOLDER"
  done
done

function handleip() {
  ip=$1
  unset IS_WHITELISTED
  info=$(whois ${ip} | egrep -i 'netname|OrgName|OrgAbuseEmail|CustName|NetName|Country|Organization-Name|org-name|@' | egrep -iv 'Comment:.*assigned staticall' | tr '*' ' ')
  emailaddy="$(echo "$info" | egrep  -o "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}\b" | sort | uniq | tr '\n' ' ' | tr ' ' ',' | tr '*' ' ' | tr '/' ' ' )"
  if [ ! -z "$IS_WHITELISTED" ] || [ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" | egrep -qai "$WHITELIST_ABUSE_REGEX"; then
    local subjects="$(egrep -h  "^Return-Path:|^Subject: |^Received: .*${ip}" $(grep -l "Received: from .*\[${ip}" $(find $SPAM_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) | sed "s/Received: .*${ip}/Received: ${ip}/"  | sed 's/ by .*//' | sed 's/])$//')"
    files=$(grep -l "Received: from .*\[${ip}" $(find $SPAM_FOLDERS_LIST -type f -mtime -${NUMDAYSTOREMOVE}))
  else
    local subjects="$(egrep -h  "^Return-Path:|^Subject: |^Received: .*${ip}" $(grep -l "Received: from .*\[${ip}" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) | sed "s/Received: .*${ip}/Received: ${ip}/"  | sed 's/ by .*//' | sed 's/])$//')"
    files=$(grep -l "Received: from .*\[${ip}" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOREMOVE}))
  fi
  echo "Bad IP: $ip"
  echo "To: " $emailaddy
  echo $subjects

  [ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" | egrep -qai "$WHITELIST_ABUSE_REGEX" && echo "Email addresses ($emailaddy) are whitelisted.  Would not block with ip tables." && echo "subjects:" && echo "$subjects" && IS_WHITELISTED=Y && return 0
  iptables -L -n | grep -qai "DROP.* $ip" && echo "Already banned $ip $info $emailaddy" && ISBANNED=Y && return
  # manually run this (e.g. grep for DROP in output)
  [  -z "$ISBANNED" ] && echo "iptables -I INPUT -s ${ip} -j DROP -m comment --comment 'bulkspamban: email spam'"
  if [ -z "$IS_DROP" ]; then continue; fi
  [  -z "$ISBANNED" ] && iptables -I INPUT -s ${ip} -j DROP -m comment --comment "bulkspamban: email spam $emailaddy $(echo info | egrep -i 'orgname|netname|org-name' | tr '\n' ' ' )" && echo "Dropped this range of IPs."
  [  -z "$ISBANNED" ] && dropped=$((dropped+1))
  # remove their files from the sa-filtered folder
  grep -qai '^MOVED_TO.*=.*user\@domain' /etc/bulkspamban.conf && echo "Edit the MOVETO_FOLDER setting in /etc/bulkspamban.conf before proceeding." 2>&1 && return

  [ ! -z "$MOVETO_FOLDER" ] && [ ! -d "$MOVETO_FOLDER" ]  && echo "MOVETO_FOLDER '$MOVETO_FOLDER' does not exist." && return

  [ -z "$MOVETO_FOLDER" ] && echo "MOVETO_FOLDER not set in /etc/bulkspamban.conf, not moving files" && return

  for j in $files; do
    [ "$(realpath $j)" == "$(realpath ${MOVETO_FOLDER}/$(basename $j))" ] && continue
    echo mv "\"$j\"" "$MOVETO_FOLDER"
    [ ! -z "$MOVETO_FOLDER" ] && [ -d "$MOVETO_FOLDER" ] && mv "$j" "$MOVETO_FOLDER"
  done
}

for ip in $(echo "$suspect_ips" | awk '{ if ($1 > 5 ) {print $2}}'); do
  handleip $ip
done

[ -z "$IS_DROP" ] && [ $dropped -gt 0 ] &&  echo "Run with $0 drop to ban the IP ranges and move emails out of inboxes to the MOVETO_FOLDER ($MOVETO_FOLDER)"

[ $dropped -gt 0 ] && iptables -L -n | egrep 'pgb|bulkspamban' | sort | uniq > /var/log/iptables.spam.$$.log && echo "Current iptable rules saved to /var/log/iptables.spam.$$.log"
	#!/bin/bash
	# Copyright 2018 Peter Bryant
	# Licensed as https://www.apache.org/licenses/LICENSE-2.0

	[ ! -z "$1" ] && [ "$1" != "--drop" ] && cat << EOJ && exit 0
	Create a folders (SPAM_FOLDER=${SPAM_FOLDER:-unset} variable). Load it up with spam.

	This script searches spam folders to find IP subnets (/24s) that have a sent a good amount of spam.

	Output commands to move those away and to block those IP ranges.

	Add --drop to move those away and block the IPs automatically.

	EOJ
	[ "$1" = "--drop" ] && IS_DROP='y'

	[ ! -f /etc/bulkspamban.conf ] && cat <<EOJ > /etc/bulkspamban.conf
	# directory where known spam is stored. We will extract spammer IP ranges from here.
	ARRAY_SPAM_FOLDERS=( "/var/vmail/user@domain.com/.Junk.sa-filtered/cur/" )

	# spam matching the spammer IP ranges will be listed from these folders (and moved out of them if the drop argument is specified)
	ARRAY_MAIL_FOLDERS=("/var/vmail/user@domain/cur/" "/var/vmail/user2@domain/cur/")

	# the folder where spam from ARRAY_MAIL_FOLDERS will be moved
	MOVETO_FOLDER="/var/vmail/user@domain/.Junk/cur/"

	COMPLAINT_FROM_EMAIL_ADDRESS=user@domain.com

	WHITELIST_ABUSE_REGEX="@google.com\|@google.com"

	WHITELIST_RECEIVED_REGEX="somehostname\|someotheriop"

	EOJ

	NUMDAYSTOCHECK=${NUMDAYSTOCHECK:-10}
	NUMDAYSTOREMOVE=${NUMDAYSTOREMOVE:-$((NUMDAYSTOCHECK*3))}
	[ -f /etc/bulkspamban.conf ] && source /etc/bulkspamban.conf

	if [ -z "$ARRAY_SPAM_FOLDERS" ] \|\| grep -qai '^ARRAY_SPAM_FOLDERS.=.user\@domain' /etc/bulkspamban.conf; then
	echo "Edit the ARRAY_SPAM_FOLDERS setting in /etc/bulkspamban.conf before proceeding." 2>&1
	exit 1
	fi


	# convert the array into a string that can be used by the find command.
	MAIL_FOLDERS_LIST="$(for index in ${!ARRAY_MAIL_FOLDERS[*]};do [ -d ${ARRAY_MAIL_FOLDERS[$index]} ] && echo -n ${ARRAY_MAIL_FOLDERS[$index]} " "; done)"
	#MAIL_FOLDERS_LIST="$(for index in ${!ARRAY_MAIL_FOLDERS[*]};do [ -d \"${ARRAY_MAIL_FOLDERS[$index]}\" ] && echo -n \"${ARRAY_MAIL_FOLDERS[$index]}\"; done)"
	#MAIL_FOLDERS_LIST="$(for index in ${!ARRAY_MAIL_FOLDERS[*]};do [ -d "${ARRAY_MAIL_FOLDERS[$index]}" ] && echo -n "${ARRAY_MAIL_FOLDERS[$index]} "; done)"
	SPAM_FOLDERS_LIST="$(for index in ${!ARRAY_SPAM_FOLDERS[*]};do [ -d ${ARRAY_SPAM_FOLDERS[$index]} ] && echo -n ${ARRAY_SPAM_FOLDERS[$index]} " "; done)"
	for SPAM_FOLDER in $SPAM_FOLDERS_LIST; do
	[ ! -d "$SPAM_FOLDER" ] && echo "Set ARRAY_SPAM_FOLDERS= to a directory containing spam. e.g. in /etc/bulkspamban.conf. e.g. to /var/vmail/email@address.com/.Junk.sa-filtered/cur/" && exit 1
	done

	# find /24 ranges sending us 4 or more emails classified as spam in the last few days.

	#Received: from mail-yw0-f202.google.com (mail-yw0-f202.google.com [209.85.161.202])
	# by mail.domain (Postfix) with ESMTPS id BD68A6177C
	# for <user@domain>; Tue, 13 Mar 2018 23:35:51 +0000 (UTC)
	# in one email can be multiple received froms:
	#Received: from mail.example.com (localhost [127.0.0.1])
	#Received: from mail1.example.st (mail1.example.st [206.123.115.88])
	#Received: from example.com (unknown [162.244.12.88])

	#suspect_nets=$(grep 'Received: from' $(find $SPAM_FOLDERS_LIST -mtime -${NUMDAYSTOCHECK} -type f) \| tail -n 1 \| egrep -v '127.0.0.1\|userid\|with local' \| sed 's/.from //g' \| sed 's/by .//g' \| sed -e 's/.\[\([0-9]\.[0-9]\.[0-9]\.\).*/\1/g' \| sort \| egrep -v '^10.\|^192.168\|[a-zA-Z]' \| uniq -c \| sort -n \| awk '{ if ($1 > 4) {print $2} }')
	[ -z "${WHITELIST_RECEIVED_REGEX}" ] && WHITELIST_RECEIVED_REGEX="nomatchhere"
	suspect_ips=
	suspect_nets=
	for file in $(find $SPAM_FOLDERS_LIST -mtime -${NUMDAYSTOCHECK} -type f); do
	# e.g.
	# Received: from peelregion.ca (unknown [104.193.255.117])
	#Received: from 10.214.167.174
	#Received: from 209.85.221.43 (EHLO mail-wr1-f43.google.com)
	#received_froms="$(grep 'Received: from' $file \| egrep -v '127.0.0.1\|userid\|with local')"
	received_ips="$(grep 'Received: from' $file \| egrep -v "${WHITELIST_RECEIVED_REGEX}" \| sed 's/.\[//' \| sed 's/\].//'\| egrep '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' \| sed -e 's/[^0-9]\([0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\)./\1/g' \| egrep -v '^10\.\|^192.168\|^127\.\|[a-zA-Z]\|^ *$')"
	#received_ips="$(echo "$received_froms" \| egrep -v '\($' \| sed 's/.from //g' \| sed 's/by .//g' \| sed 's/ (EHLO .)//' \| sed -e 's/.\[\([0-9]\.[0-9]\.[0-9]\.[0-9]\).*/\1/g' \| egrep -v '^10\.\|^192.168\|^127\.[a-zA-Z]')"
	suspect_nets="$suspect_nets
	$(echo "$received_ips" \| sed -e 's/\([0-9]\.[0-9]\.[0-9]\.\)./\1/g')
	"

	suspect_ips="$suspect_ips
	$received_ips
	"
	done
	suspect_nets=$(echo "$suspect_nets" \| grep -v '^$' \| sort \| uniq -c \| sort -n)
	suspect_nets=$(echo "$suspect_nets" \| awk '{ if ($1 > 4) {print $2} }')

	suspect_ips=$(for i in $(echo "$suspect_ips" \| grep -v '^$'); do echo $i \| egrep -qai $(echo -n "$suspect_nets" \| tr '\n' '\|'; echo "\|xxxx";) && continue; echo $i; done \| sort \| uniq -c \| sort -n)
	echo "Suspect IPs:"
	echo "$suspect_ips" \| awk '{ if ($1 > 3 ) {print $1 " x " $2} }'
	#echo "$suspect_ips" \| awk '{ if ($1 > 3 && $1 <6 ) {print $1 "x " $2} }'
	#echo "Bad IPs (would ban):"
	#echo "$suspect_ips" \| awk '{ if ($1 > 5 ) {print "# " $1 "x " $2; print "iptables -I INPUT -s " $1 " -j DROP -m comment --comment \'bulkspamban: email spam\'"} }'
	# show the subjects for those emails

	dropped=0
	for i in $suspect_nets; do
	# was ${i}0 but that would pick up apnic vs. statically assigned operator
	iptables -L -n \| grep -qai "DROP.* $i" && continue
	echo
	echo
	info=$(whois ${i}1 \| egrep -i 'netname\|OrgName\|OrgAbuseEmail\|CustName\|NetName\|Country\|Organization-Name\|org-name\|@' \| egrep -iv 'Comment:.assigned staticall' \| tr '' ' ')
	emailaddy="$(echo "$info" \| egrep -o "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}\b" \| sort \| uniq \| tr '\n' ' ' \| tr ' ' ',' \| tr '*' ' ' \| tr '/' ' ' )"
	iptables -L -n \| grep -qai "DROP.* $i" && echo "Already banned $info $emailaddy" && ISBANNED=Y && continue
	echo "Netblock: ${i}0/24"
	# outside quotes to show all on one line
	echo "To: " $emailaddy
	# no quotes to show on one line
	echo $info
	msg="Hi. We received the following spam emails from your network block. Please reply with what action was taken to prevent further spam from being sent from your network. "
	echo "$msg"
	unset ISBANNED
	unset IS_WHITELISTED
	if [ ! -z "$IS_WHITELISTED" ] \|\| [ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" \| egrep -qai "$WHITELIST_ABUSE_REGEX"; then
	# skip MAIL_FOLDERS_LIST since that folder may contain real email from a whitelisted mail
	files="$(grep -l "Received: from .*\[$i" $(find $SPAM_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK}))"
	else
	files="$(grep -l "Received: from .*\[$i" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK}))"
	fi
	[ ! -z "$files" ] && subjects="$(egrep -h "^Return-Path:\|^Subject: \|^Received: .$i" $files \| sed "s/Received: .$i/Received: $i/g" \| sed 's/ by .*//g' \| sed 's/])$//g')"
	#confl subjects="$(egrep -h "^Subject: \|^Received: .$i" $(grep -l "Received: from .\[$i" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) \| sed "s/Received: .$i/Received: $i/" \| sed 's/ by .//' \| sed 's/])$//')"
	#confl subjects="$(egrep -h "^Subject: \|^Received: .$i" $(grep -l "Received: from .\[$i" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) \| sed "s/Received: .$i/Received: $i/" \| sed 's/ by .//' \| sed 's/])$//')"
	echo "$subjects"
	[ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" \| egrep -qai "$WHITELIST_ABUSE_REGEX" && echo "Email addresses ($emailaddy) are whitelisted. Would not block with ip tables." && IS_WHITELISTED=Y && continue


	# manually run this (e.g. grep for DROP in output)
	[ -z "$ISBANNED" ] && echo "iptables -I INPUT -s ${i}0/24 -j DROP -m comment --comment 'bulkspamban: email spam'"
	if [ -z "$IS_DROP" ] ; then continue; fi

	if [ ! -z "$IS_WHITELISTED" ]; then
	true
	else
	[ -z "$ISBANNED" ] && iptables -I INPUT -s ${i}0/24 -j DROP -m comment --comment "bulkspamban: email spam $emailaddy $(echo info \| egrep -i 'orgname\|netname\|org-name' \| tr '\n' ' ' )" && echo "Dropped this range of IPs."
	[ -z "$ISBANNED" ] && dropped=$((dropped+1))
	fi
	if [ -z "$ISBANNED" ] && [ -z "$IS_WHITELISTED" ]; then
	grep -qai '^COMPLAINT_FROM_EMAIL_ADDRESS[A-Z].=.user\@domain' /etc/bulkspamban.conf && echo "Edit the COMPLAINT_FROM_EMAIL_ADDRESS setting in /etc/bulkspamban.conf before proceeding." 2>&1 && continue

	echo "$msg

	$info

	$subjects

	Sample email with headers:

	$(filearray=($files); [ ! -z "$files" ] && [ ! -z "${filearray[1]}" ] && cat "${filearray[1]}" \| iconv -f utf-8 -t us-ascii//TRANSLIT \| head -n 100 \| head -c 6096 \| sed 's#http:/#hxxp:/#g' \| sed 's#https:/#hxxps:/#g')
	" \| mail -s "Spam abuse report from ${i}0/24" -a "From: $COMPLAINT_FROM_EMAIL_ADDRESS" -b "$COMPLAINT_FROM_EMAIL_ADDRESS" "$emailaddy"
	fi
	# remove their files from the sa-filtered folder
	grep -qai '^MOVED_TO.=.user\@domain' /etc/bulkspamban.conf && echo "Edit the MOVETO_FOLDER setting in /etc/bulkspamban.conf before proceeding." 2>&1 && continue

	[ ! -z "$MOVETO_FOLDER" ] && [ ! -d "$MOVETO_FOLDER" ] && echo "MOVETO_FOLDER '$MOVETO_FOLDER' does not exist." && continue

	[ -z "$MOVETO_FOLDER" ] && echo "MOVETO_FOLDER not set in /etc/bulkspamban.conf, not moving files" && continue

	for j in $files; do
	[ "$(realpath $j)" == "$(realpath ${MOVETO_FOLDER}/$(basename $j))" ] && continue
	echo mv "\"$j\"" "$MOVETO_FOLDER"
	[ ! -z "$MOVETO_FOLDER" ] && [ -d "$MOVETO_FOLDER" ] && mv "$j" "$MOVETO_FOLDER"
	done
	done

	function handleip() {
	ip=$1
	unset IS_WHITELISTED
	info=$(whois ${ip} \| egrep -i 'netname\|OrgName\|OrgAbuseEmail\|CustName\|NetName\|Country\|Organization-Name\|org-name\|@' \| egrep -iv 'Comment:.assigned staticall' \| tr '' ' ')
	emailaddy="$(echo "$info" \| egrep -o "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}\b" \| sort \| uniq \| tr '\n' ' ' \| tr ' ' ',' \| tr '*' ' ' \| tr '/' ' ' )"
	if [ ! -z "$IS_WHITELISTED" ] \|\| [ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" \| egrep -qai "$WHITELIST_ABUSE_REGEX"; then
	local subjects="$(egrep -h "^Return-Path:\|^Subject: \|^Received: .${ip}" $(grep -l "Received: from .\[${ip}" $(find $SPAM_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) \| sed "s/Received: .${ip}/Received: ${ip}/" \| sed 's/ by .//' \| sed 's/])$//')"
	files=$(grep -l "Received: from .*\[${ip}" $(find $SPAM_FOLDERS_LIST -type f -mtime -${NUMDAYSTOREMOVE}))
	else
	local subjects="$(egrep -h "^Return-Path:\|^Subject: \|^Received: .${ip}" $(grep -l "Received: from .\[${ip}" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOCHECK})) \| sed "s/Received: .${ip}/Received: ${ip}/" \| sed 's/ by .//' \| sed 's/])$//')"
	files=$(grep -l "Received: from .*\[${ip}" $(find $SPAM_FOLDERS_LIST $MAIL_FOLDERS_LIST -type f -mtime -${NUMDAYSTOREMOVE}))
	fi
	echo "Bad IP: $ip"
	echo "To: " $emailaddy
	echo $subjects

	[ ! -z "$WHITELIST_ABUSE_REGEX" ] && echo "$emailaddy" \| egrep -qai "$WHITELIST_ABUSE_REGEX" && echo "Email addresses ($emailaddy) are whitelisted. Would not block with ip tables." && echo "subjects:" && echo "$subjects" && IS_WHITELISTED=Y && return 0
	iptables -L -n \| grep -qai "DROP.* $ip" && echo "Already banned $ip $info $emailaddy" && ISBANNED=Y && return
	# manually run this (e.g. grep for DROP in output)
	[ -z "$ISBANNED" ] && echo "iptables -I INPUT -s ${ip} -j DROP -m comment --comment 'bulkspamban: email spam'"
	if [ -z "$IS_DROP" ]; then continue; fi
	[ -z "$ISBANNED" ] && iptables -I INPUT -s ${ip} -j DROP -m comment --comment "bulkspamban: email spam $emailaddy $(echo info \| egrep -i 'orgname\|netname\|org-name' \| tr '\n' ' ' )" && echo "Dropped this range of IPs."
	[ -z "$ISBANNED" ] && dropped=$((dropped+1))
	# remove their files from the sa-filtered folder
	grep -qai '^MOVED_TO.=.user\@domain' /etc/bulkspamban.conf && echo "Edit the MOVETO_FOLDER setting in /etc/bulkspamban.conf before proceeding." 2>&1 && return

	[ ! -z "$MOVETO_FOLDER" ] && [ ! -d "$MOVETO_FOLDER" ] && echo "MOVETO_FOLDER '$MOVETO_FOLDER' does not exist." && return

	[ -z "$MOVETO_FOLDER" ] && echo "MOVETO_FOLDER not set in /etc/bulkspamban.conf, not moving files" && return

	for j in $files; do
	[ "$(realpath $j)" == "$(realpath ${MOVETO_FOLDER}/$(basename $j))" ] && continue
	echo mv "\"$j\"" "$MOVETO_FOLDER"
	[ ! -z "$MOVETO_FOLDER" ] && [ -d "$MOVETO_FOLDER" ] && mv "$j" "$MOVETO_FOLDER"
	done
	}

	for ip in $(echo "$suspect_ips" \| awk '{ if ($1 > 5 ) {print $2}}'); do
	handleip $ip
	done

	[ -z "$IS_DROP" ] && [ $dropped -gt 0 ] && echo "Run with $0 drop to ban the IP ranges and move emails out of inboxes to the MOVETO_FOLDER ($MOVETO_FOLDER)"

	[ $dropped -gt 0 ] && iptables -L -n \| egrep 'pgb\|bulkspamban' \| sort \| uniq > /var/log/iptables.spam.$$.log && echo "Current iptable rules saved to /var/log/iptables.spam.$$.log"