update_external_ip_blocking.sh

#!/bin/bash

NETGROUP="5ac902e89c29dc0273fb5490"
TMP_DIR="/tmp"
LIST_FILE_PATH="$TMP_DIR/external_ip_block"
LOGGING_DIR="/var/log"
DRY_RUN=0
NOW=`date +"%Y-%m-%d %T"`
START_TIME=`date +"%T"`
SCRIPT_FILE_NAME=`basename "$0"`
LOG_FILE_NAME="${SCRIPT_FILE_NAME%.*}.log"

# Notify logger that the script starts
logger -s "$SCRIPT_FILE_NAME [$NOW] Updating external IP blocking lists..." 2>> "$LOGGING_DIR/$LOG_FILE_NAME"

#
# Download list of IPs
#

# Emerging Threats
curl -w "\n" -s https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt | grep '^[0-9]' > $TMP_DIR/emergingthreats_block
# Spamhaus
curl -w "\n" -s http://www.spamhaus.org/drop/drop.txt | grep '^[0-9]' | sed -e 's/;.*//' > $TMP_DIR/spamhaus_block
curl -w "\n" -s http://www.spamhaus.org/drop/edrop.txt | grep '^[0-9]' | sed -e 's/;.*//' >> $TMP_DIR/spamhaus_block
# Dshield
curl -w "\n" -s https://www.dshield.org/block.txt > $TMP_DIR/dshield_block
# Blocklist.de
curl -w "\n" -s https://lists.blocklist.de/lists/ssh.txt > $TMP_DIR/blocklistde_block
curl -w "\n" -s https://lists.blocklist.de/lists/apache.txt >> $TMP_DIR/blocklistde_block
curl -w "\n" -s https://lists.blocklist.de/lists/bots.txt >> $TMP_DIR/blocklistde_block
curl -w "\n" -s https://lists.blocklist.de/lists/strongips.txt >> $TMP_DIR/blocklistde_block
curl -w "\n" -s https://lists.blocklist.de/lists/bruteforcelogin.txt >> $TMP_DIR/blocklistde_block


#
# Format Dshield list as other lists
#

# Extract IP
IPADDR=$(cut -f1 $TMP_DIR/dshield_block | grep -o '^[^#]*' | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | sed ':a;N;$!ba;s/\n/ /g')
# Extract netmask
NETMASK=$(cut -f3 $TMP_DIR/dshield_block | grep -o '^[^#]*' | grep -o '[0-9]\{1,2\}' | sed ':a;N;$!ba;s/\n/ /g')

# Create bash lists
A_IPADDR=($IPADDR)
A_NETMASK=($NETMASK)

# Check that IP and netmask counts are identical
if [[ ${#A_IPADDR[@]} != ${#A_NETMASK[@]} ]]; then
  echo "ERROR: Lists length mismatch!"
  echo "IP list:${#A_IPADDR[@]}, Netmask list:${#A_NETMASK[@]}"
  logger -s "$SCRIPT_FILE_NAME [$NOW] Dshield list length mismatch! IP list:${#A_IPADDR[@]}, Netmask list:${#A_NETMASK[@]}" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
fi

# Create final IP list for Dshield
>$TMP_DIR/dshield_block
for (( i=0; i<${#A_IPADDR[@]} ; i++ ));
do
  echo "${A_IPADDR[i]}/${A_NETMASK[i]}" >> $TMP_DIR/dshield_block
done

# Unset variables
IPADDR=
NETMASK=
A_IPADDR=
A_NETMASK=

#
# Create external IP list from other lists
#

# Concatenate IP lists
>$LIST_FILE_PATH
cat $TMP_DIR/emergingthreats_block >> $LIST_FILE_PATH
cat $TMP_DIR/spamhaus_block >> $LIST_FILE_PATH
cat $TMP_DIR/dshield_block >> $LIST_FILE_PATH
cat $TMP_DIR/blocklistde_block >> $LIST_FILE_PATH

rm $TMP_DIR/emergingthreats_block
rm $TMP_DIR/spamhaus_block
rm $TMP_DIR/dshield_block
rm $TMP_DIR/blocklistde_block

# Order and remove duplicates
cp $LIST_FILE_PATH "$TMP_DIR/$NOW'_ip_block'"
sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n -u "$TMP_DIR/$NOW'_ip_block'" | sed '/^$/d' > $LIST_FILE_PATH
rm "$TMP_DIR/$NOW'_ip_block'"

# Stop here for dry run
if [ $DRY_RUN -eq 1 ]; then
  exit
fi

#
# Populate network group with IPs
#

# Check if group exists
sudo ipset -q -L $NETGROUP > /dev/null 2>&1
if [ "$?" != 0 ]; then
  echo "Firewall network group $NETGROUP doesn't exist yet"
  logger -s "$SCRIPT_FILE_NAME [$NOW] Firewall network group $NETGROUP doesn't exist yet" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
  rm $LIST_FILE_PATH
  exit 1
fi

# Create temporary group
NEWGROUP=$NETGROUP-$$
sudo ipset create $NEWGROUP hash:net
if [ "$?" != 0 ]; then
  echo "There was an error trying to create temporary set"
  logger -s "$SCRIPT_FILE_NAME [$NOW] There was an error trying to create temporary set" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
  rm $LIST_FILE_PATH
  exit 1
fi

# Add IPs from list to temporary group
count=0;
for ip in `cat $LIST_FILE_PATH`; do
  sudo ipset -q -A $NEWGROUP $ip
  if [ "$?" != 0 ]; then
    echo "There was an error trying to add $ip. Skip!"
    logger -s "$SCRIPT_FILE_NAME [$NOW] There was an error trying to add $ip" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
    logger -s "$SCRIPT_FILE_NAME [$NOW] Skipping IP $ip" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
    # rm $LIST_FILE_PATH
    # exit 1
  fi
  let "count++"
done
rm $LIST_FILE_PATH

# Swap group to avoid down time
sudo ipset swap $NEWGROUP $NETGROUP
if [ "$?" != 0 ]; then
  echo "There was an error trying to swap temporary set"
  logger -s "$SCRIPT_FILE_NAME [$NOW] There was an error trying to swap temporary set" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
  exit 1
fi
# Destroy temporary group
sudo ipset destroy $NEWGROUP

END_TIME=`date +"%T"`
logger -s "$SCRIPT_FILE_NAME [$NOW] $START_TIME-$END_TIME: Successfully added $count entries to $NETGROUP" 2>> "$LOGGING_DIR/$LOG_FILE_NAME"
exit 0