Crufty script to generate IPSets blocklists, untested for the last few years

badips

#!/bin/sh

force=0
semidaily=0
quadaily=0
hexdaily=0
verbose=0
download=0
process=0

while getopts "fvdSQHh" opt
do
  case "$opt" in
    f) force=1;;
    S) semidaily=1;;
    Q) quadaily=1;;
    H) hexdaily=1;;
    v) verbose=1;;
    d) download=1 && force=1 && verbose=1;;
    h)
      echo >&2 \
      'usage: badips [[-v] [-S] [-Q] [-H]] | [[-f] [-v]] | [-d]

      -f  Force processing of all lists, and badips ipset.
      -v  Verbose, print download progress, and ipset information.
      -d  Download all lists, ignore modification times, and force
          processing with verbose output. Implies -f -v.
      -S  Process lists which should be updated twice a day.
      -Q  Process lists which should be updated four times a day.
      -H  Process lists which should be updated six times a day.
      '
      exit 1;;
  esac
done
shift `expr $OPTIND - 1`

mkdir -p /var/tmp/badips/lists
mkdir -p /var/tmp/badips/lists-preprocess
cd /var/tmp/badips/

if [ $verbose -eq 1 ]
  then
    curlopts='-4R --no-keepalive --compressed -m 60 --retry 2'
  else
    curlopts='-4RSs --no-keepalive --compressed -m 60 --retry 2'
fi

if [ $force -eq 1 ] || [ $semidaily -eq 1 ]
  then
    # At least an hour between fetches, or ip will be banned.
    # Slow turnover, suggested once per day.
    shd='./lists/spamhaus-drop.txt'
    shd=$([ $download -eq 1 ] && echo "-o $shd -z $shd" || echo "-o $shd")
    curl $curlopts $shd https://www.spamhaus.org/drop/drop.txt
    she='./lists/spamhaus-edrop.txt'
    she=$([ $download -eq 1 ] && echo "-o $she -z $she" || echo "-o $she")
    curl $curlopts $she https://www.spamhaus.org/drop/edrop.txt
    process=1
fi

if [ $force -eq 1 ] || [ $quadaily -eq 1 ]
  then
    # *tracker.abuse.net, download interval limits unknown.
    # Zeus BadIPs, excludes infected servers and free web hosts.
    ztbi='./lists/zeustracker-badips.txt'
    ztbi=$([ $download -eq 1 ] && echo "-o $ztbi -z $ztbi" || echo "-o $ztbi")
    curl $curlopts $ztbi https://zeustracker.abuse.ch/blocklist.php?download=badips
    # Palevo Tracker: Discontinued
    # ptib='./lists/palevotracker-ipblocklist.txt'
    # ptib=$([ $download -eq 1 ] && echo "-o $ptib -z $ptib" || echo "-o $ptib")
    # curl $curlopts $ptib https://palevotracker.abuse.ch/blocklists.php?download=ipblocklist
    # Feodo Tracker B addresses, only hosts "dedicated" to Feodo.
    ftbi='./lists/feodotracker-badips.txt'
    ftbi=$([ $download -eq 1 ] && echo "-o $ftbi -z $ftbi" || echo "-o $ftbi")
    curl $curlopts $ftbi https://feodotracker.abuse.ch/blocklist/?download=badips
    process=1
fi

if [ $force -eq 1 ] || [ $hexdaily -eq 1 ]
  then
    # Updated every 4 hours, download every 4 hours.
    # Can't get SSL to function, so -k argument to curl.
    tcfbipv4='./lists/team-cymru-fullbogons-ipv4.txt'
    tcfbipv4=$([ $download -eq 1 ] && echo "-o $tcfbipv4 -z $tcfbipv4" || echo "-o $tcfbipv4")
    curl $curlopts -k $tcfbipv4 https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt
    # Dshield
    # TODO: Use https://feeds.dshield.org/block.txt.asc signature.
    drblp='./lists-preprocess/dshield-recommended-blocklist.txt'
    drbl=$([ $download -eq 1 ] && echo "-o $drblp -z $drblp" || echo "-o $drblp")
    curl $curlopts $drbl https://www.dshield.org/block.txt
    # Need to change list to a compatible format, all ranges are class C (/24).
    sed_drbl='s_^(([0-9]{1,3}\.){3}[0-9]{1,3}).*_\1/24_p'
    sed -rn "$sed_drbl" "$drblp" | cat - > ./lists/dshield-recommended-blocklist.txt
    process=1
fi

sed_ips='s_^(([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?).*_\1_p'
sed_ipset='s_(^.*)_add badips-swap \1_p'
perl_dedup='BEGIN{our $c=Net::CIDR::Lite->new} our $c->add_any($_); END{$c->clean; print join("\n",$c->list)}'

all_ips=$(cat ./lists/* | sed -rn "$sed_ips")
dedup_ips=$(echo "$all_ips" | perl -WlanMNet::CIDR::Lite -e"$perl_dedup" | sed -rn "$sed_ipset")

sed_range='s_(^.*)_a:\1_p'
perl_dedup_range='BEGIN{our $c=Net::CIDR::Lite->new} our $c->add_any($_); END{$c->clean; print join("\n",$c->list_range)}'
dedup_ips_range=$(echo "$all_ips" | perl -WlanMNet::CIDR::Lite -e"$perl_dedup_range" | sed -rn "$sed_range")
echo "$dedup_ips_range" > ./badips-blocklist

if [ $force -eq 1 ] || [ $process -eq 1 ]
  then
    echo 'create badips hash:net -exist' > ./badips-restore
    echo 'create badips-swap hash:net -exist' >> ./badips-restore
    echo 'flush badips-swap' >> ./badips-restore
    echo "$dedup_ips" >> ./badips-restore
    echo 'swap badips-swap badips'  >> ./badips-restore
    echo 'destroy badips-swap' >> ./badips-restore

    ipset restore -file ./badips-restore
fi

if [ $verbose -eq 1 ]
  then
    raw_count=$(echo "$all_ips" | wc -l)
    dedup_count=$(echo "$dedup_ips" | wc -l)
    echo "Raw IP addresses: $raw_count"
    echo "Deduplicated IP addresses: $dedup_count"
    echo "Deduplication savings:" $(( $raw_count-$dedup_count ))
    echo "All IP addresses in badips ipset:" $(ipset list badips | grep -v ':' | wc -l)
    echo "Hosts in badips ipset:" $(ipset list badips | grep -Ev ':|/' | wc -l)
    echo "Subnets in badips ipset:" $(ipset list badips | grep '/' | wc -l)
    echo "### Terse badips ipset info ###"
    echo "$(ipset -t list badips)"
fi