ProBackup-nl/sub2rbl

## sub2rbl
#!/bin/sh
#
# sub2rbl for padavan firmware @ProBackup-nl
# ============
# Subscribes to IP or CIDR (netblock) RBLs; auto-integrates into iptables
# Purely for reduced memory consumption it creates two IP sets:
# 1. one for unicast IPs (sub2rbl)
# 2. the other one for CIDR (sub2rbl-net) blocks
# Realtime block lists contain both types
#
#   https://gist.github.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7
#

# Dependencies
# ============
# entware(-ng-3x)
# awk + xargs + mktemp
# wget-ssl || curl (>=v7.73.0) + ca-certificates (186k) needed if using HTTPS RBLs
# kernel modules xt_set, ip_set_hash_ip, ip_set_hash_net
# built-in-firmware: ipset, iptables, grep, lsmod, grep, sed, mv, rm
# optional: iprange

# Installation
# ============
# 0. mkdir -p /opt/usr/sbin;NAME=sub2rbl;wget -O /opt/usr/sbin/$NAME https://gist.githubusercontent.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7/raw/sub2rbl && chmod 755 /opt/usr/sbin/$NAME
# 1. Go to "Padavan Router > Customization > Scripts" web interface page and put following content to "Run After Router Started" field:
# modprobe ip_set_hash_ip
# modprobe ip_set_hash_net
# modprobe xt_set
# ipset create sub2rbl hash:ip maxelem 32768
# ipset create sub2rbl-net hash:net maxelem 32768
#
# 2. nano /opt/etc/init.d/S01system
# *) start
#		[ -s '/opt/etc/ipset.conf' ] && ipset -file /opt/etc/ipset.conf restore
#		func_start
#
# 3. Go to "Padavan Router > Administration > Services" web interface page and enable "Cron Daemon (Scheduler)"
#
# 4. Go to "Padavan Router > Administration > Services" web interface page and put following content to "Scheduler tasks (Crontab)" field:
# 1-59/13 * * * * /opt/usr/sbin/sub2rbl
#
# This will run the update process every 13 minutes past the hour + 1 minute
#
# 5. nano /opt/etc/init.d/S10iptables
# start|update)
#         # add iptables sub2rbl rules
#         [ -d '/opt/etc' ] || exit 0
#         iptables -I INPUT 1 -m set --match-set sub2rbl src -j DROP
#         iptables -I INPUT 1 -m set --match-set sub2rbl-net src -j DROP
#         iptables -I FORWARD 1 -m set --match-set sub2rbl src -j DROP
#         iptables -I FORWARD 1 -m set --match-set sub2rbl-net src -j DROP
#         ;;
#
# 6. ssh into your router and run the update process once to generate the configuration file
# sub2rbl -l 3 -f stdout
#
# 7. enhance your configuration by excluding your own IP-adresses and/or CIDR blocks by whitelisting them
# nano /opt/etc/config/sub2rbl.conf
#
# 8. run the update process once more
# sub2rbl -l 3 -f stdout

# Improvements / ToDo
# ===================
# - remove (very) old cached downloaded files when upstream is gone (current: on reboot)
# - process rbl lists according to their individual update frequency (even skip if-modified check)
# - whitelist from url

#
# Advanced settings
#
setName=sub2rbl
logLevel=1
logFacility=authpriv.notice
firewallTarget=DROP
firewallHookChains='INPUT:1 FORWARD:1'
netSetName="$setName"-net
logTag="sub2rbl[$$]"
ipsetArgs="maxelem 32768"
webGetCmd=guess
umask 0002 # no write access for other

if [ ! -d /tmp/$setName ] ; then
  mkdir -m 664 /tmp/$setName
fi

# if webGetCmd is set to guess (default), we'll try to autoselect
# between a few options for an TLS ready http client.
[ "$webGetCmd" = guess ] && {
  if [ -x /opt/bin/wget-ssl ] ; then
    #                            -q quiet
    #                             -T timeout (seconds)
    #                                  -t tries
    #                                       -O output document
    #                                      -N time-stamping, only save newer files
    #                                         -P set directory prefix to prefix
    webGetCmd="/opt/bin/wget-ssl -qT15 -t2 -N -P /tmp/$setName -O"  # GNU wget
  elif [ -x /opt/bin/wget ] ; then
    webGetCmd="/opt/bin/wget -qT15 -t2 -N -P /tmp/$setName -O"  # entware renamed wget-ssl to wget (wget became wget-nossl)
  elif [ -x /opt/bin/curl ] ; then
    #                        -s silent
    #                         -S show error
    #                          -R remote-time
    #                           -O output file and name file to remote
    #                            -m maximum transfer time (seconds)
    #                                                            -z time-conditional requires file, thus skipped
    webGetCmd="/opt/bin/curl -sSROm15 --output-dir /tmp/$setName -z"  # curl with mbedTLS - most lightweight option
  else
    # lastly, busybox wget - bad option as it fails on some TLS connections like spamhaus
    webGetCmd="/usr/bin/wget -qT15 -t2 -P /tmp/$setName -O"
  fi
}

# Functions
#
logLine() {
  [ $1 -gt $logLevel ] && return ; shift
  if [ "$logFacility" = "stdout" ] ; then echo "$@"
  elif [ "$logFacility" = "stderr" ] ; then echo "$@" >&2
  else logger -t "$logTag" -p "$logFacility" "$@"
  fi
}

# Arg: $1=logLevel, $2=setName, $3=setType (ip or net)
createSet() {
  local useLogLevel="$1" set="$2" setType="$3"
  ipset list -q -n "$set" >/dev/null || {
    logLine $useLogLevel "Creating ipset $set"
    ipset create "$set" hash:"$setType" $ipsetArgs
  }
}

# Arg: $1=logLevel, $2=setName
removeSet() {
  local useLogLevel=$1 set="$2"
  ipset list -q -n "$set" >/dev/null && {
    logLine $useLogLevel "Destroying ipset $set"
    ipset destroy "$set"
} }

# Arg: 1=$setName
setupIPTables() {
  local set="$1" x chain position
  for x in $firewallHookChains ; do
    chain="${x%:*}" ; position="${x#*:}"
    if [ $position -ge 0 ] && ! iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
      if [ $position = 0 ] ; then
        logLine 1 "Appending iptables rule for $set into chain $chain"
        iptables -A "$chain" -m set --match-set "$set" src -j "$firewallTarget"
      else
        logLine 1 "Inserting iptables rule at position $position for $set into chain $chain"
        iptables -I "$chain" $position -m set --match-set "$set" src -j "$firewallTarget"
  fi ; fi ; done
}

# Arg: 1=$setName
removeFromIPTables() {
  local set="$1" x chain position
  for x in $firewallHookChains ; do
    chain="${x%:*}" ; position="${x#*:}"
    if [ $position -ge 0 ] && iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
      logLine 1 "Removing iptables rule for $set from chain $chain"
      iptables -D "$chain" -m set --match-set "$set" src -j "$firewallTarget"
  fi ; done
}

printUsage() {
  cat <<-_EOF_
	Usage: sub2rbl [-C ...] [-f ...] [-j ...] [-l #] [-s ...] [-w] [URL ...]
	  -C ... firewall hook chains:positions (def: $firewallHookChains)
	  -f ... log facility (syslog facility or stdout/stderr) (def: $logFacility)
	  -j ... firewall target when matching the set (def: $firewallTarget)
	  -l #   log level - 0=off, 1=standard, 2=verbose (def: $logLevel)
	  -s ... ipset name
	  -w     wipe ipset and iptables rule (do not process RBLs)
	  URL(s) RBL URLs - using command line RBL URLs will override uci entries
	_EOF_
}

writeConfigFile() {
  cat > /opt/etc/config/sub2rbl.conf <<'_EOF_'
#
# sub2rbl-padavan - https://github.com/ProBackup-nl/sub2rbl-padavan
#
# Save this file at /opt/etc/config/sub2rbl.conf
# Indent using tab-characters (TSV: tab separated value list)
# Values must be quoted

	# Whitelist regex strings examples
	#	list	whitelist	'^10\.0\.[01]\..*$'
		list	whitelist	'^127\.0\.0\.0/8$'
		list	whitelist	'^192\.168\.0\.0/16$'

	# RBL URLs - some (but not all) will also support http
	# Prepend net: to the URL for CIDR (net) RBLs
	# Order: net: URL's first (allows skipping IPs already blocked with net:)

        ### https://www.spamhaus.org/drop - Spamhaus (Extended) Don't Route Or Peer List - spammer/cyber-crime net based RBLs
        ### check frequency: 12 hours
                list    rbl     'net:https://www.spamhaus.org/drop/drop.txt'
                list    rbl     'net:https://www.spamhaus.org/drop/edrop.txt'

        ### Bogons and also IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user
        ### WARNING: Rigorously test this filter because it contains 127. 192. 172. and 10. non routable space.
                list    rbl     'net:https://team-cymru.org/Services/Bogons/fullbogons-ipv4.txt'

        ### http://ipdeny.com - provides lists of IPs associated with countries.  Ex: CN
        # check frequency: 1 day
        #	list	rbl 'net:http://www.ipdeny.com/ipblocks/data/aggregated/cn-aggregated.zone'

	### http://rbldata.interserver.net/ - malware posts, ssh/ftp/email/modsec scans, spam and more
	### iprbl.txt = full (12K), standard ip.txt = last 7 days (2K), ipslim.txt = last 48 hours (1K)
        ### check frequency: 3 minutes
	## list	rbl	'https://sigs.interserver.net/iprbl.txt'
		list	rbl	'https://rbldata.interserver.net/ip.txt'
	## list	rbl	'https://rbldata.interserver.net/ipslim.txt'

	### https://www.blocklist.de/en - blacklists generated from fail2ban feedback, last 48 hours
        ### check frequency:  15 minutes
		list	rbl	'https://lists.blocklist.de/lists/ssh.txt'
	### All IPs which # are older then 2 month and have more then 5.000 attacks.
		list	rbl	'https://lists.blocklist.de/lists/strongips.txt'

_EOF_
}

if [ ! -f /opt/etc/config/sub2rbl.conf -o ! -s /opt/etc/config/sub2rbl.conf ] ; then
	writeConfigFile
	whitelist=''
	rblList=''
else
	# Read configuration from (uci style) configuration file /opt/etc/config/sub2rbl.conf
	whiteList=$(awk '!(/^\t*#/) && $2 == "whitelist" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf | xargs | sed -e 's/ /|/g')
	rblList=$(awk '!(/^\t*#/) && $2 == "rbl" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf)
fi


# Logic begins
#
wipe=false
# Pure ash getopts replacement by https://gist.github.com/jhurliman/ef18f8cd1880dc00365d
while [[ $# -gt 1 ]]; do
  arg="$1"
  case "$arg" in
    -C) firewallHookChains="$2"; shift ;;
    -f) logFacility="$2"; shift ;;
    -j) firewallTarget="$2"; shift ;;
    -l) logLevel="$2"; shift ;;
    -s) setName="$2"; shift ;;
    -w) wipe=true ;;
    *) printUsage
      exit 254
  esac
  shift
done

#before taking any action, first make sure that the required kernel modules are loaded:
modList='xt_set
ip_set_hash_ip
ip_set_hash_net'
#if not, load them using modprobe
#if loading fails, abort
for mod in $modList ; do
  if [ -z "lsmod | grep '^$mod '" ] ; then
    logLine 2 "Loading kernel module ($mod)"
    modprobe "$mod"
    wait 1
    if [ -z "lsmod | grep '^$mod '" ] ; then
      logLine 1 "Failure loading required kernel module ($mod)"
      exit 78
    fi
  fi
done

if $wipe ; then
  logLine 2 "Wiping sub2rbl from iptables and ipset"
  for tempSet in "$setName" "$netSetName" ; do
    removeFromIPTables "$tempSet"
    removeSet 1 "$tempSet"
  done
  # remove ipset backup file too
  [ -f /opt/etc/ipset.conf ] && rm /opt/etc/ipset.conf
  exit
fi

# Download updates when available
# WARNING: each rbl MUST use a unique file name
newFile=false
for rbl in $rblList ; do
  if [ ${rbl%%:*} = net ] ; then
    rbl="${rbl##net:}"
  fi
  logLine 2 "Retrieving RBL ($rbl)"
  fileName=/tmp/$setName/`basename $rbl`
  if ! $newFile; then
    [ -s $fileName ] && touch -r $fileName $fileName'.time'
  fi

  #logLine 3 "$webGetCmd $fileName $rbl"
  if ! errorMsg=`$webGetCmd "$fileName" "$rbl" 2>&1`; then
    logLine 1 "Error: retrieving RBL($rbl): $errorMsg"
    # ToDo: remove really old files (current = cache forever)
    continue
  fi

  if ! $newFile; then
    if [ ! -e $fileName'.time' -a -s $fileName ] ; then
      logLine 3 '= new file (not exists before and not empty after)'
      newFile=true
    elif [ -e $fileName'.time' -a $fileName'.time' -ot $fileName ] ; then
      logLine 3 '= modified'
      newFile=true
    elif [ -e $fileName'.time' -a ! -e $fileName ] ; then
      logLine 3 '= removed (exist before and not exists after)'
      newFile=true
    fi
    [ -e $fileName'.time' ] && rm $fileName'.time'
  fi
done

if ! $newFile; then
  logLine 1 "No updates, exit."
  exit 0
fi

swingSetName="${setName}_swing"
swingNetSetName="${netSetName}_swing"
intNetEntries=0
tmpFile="`/opt/bin/mktemp`"
tmpFile2="`/opt/bin/mktemp`"
tmpFile3="`/opt/bin/mktemp`"

# Process
# configure net based first then IP based to be able to remove overlaps
for rbl in $rblList ; do
  if [ ${rbl%%:*} = net ] ; then
    setType='net' 		# process net-based RBLs
    rbl="${rbl##net:}"
    tempSet="$netSetName"
  else
    setType='ip'		# default is IP-based (default)
    tempSet="$setName"
  fi
  fileName=/tmp/$setName/`basename $rbl`
  createSet 1 "$tempSet" $setType
  createSet 2 "$tempSet"_swing $setType
  #if ! errorMsg=`$webGetCmd "$tmpFile" "$rbl" 2>&1`; then

  # check that fileName exists and is not empty
  if [ -s $fileName ] ; then
    logLine 3 "Processing RBL ($rbl)"

    if [ $setType = ip ] ; then
      # autodetect the DRG sshpwauth report
      grep -Ev '^#' "$fileName" | head -1 | grep -q '^.*|.*|.*|.*|.*sshpwauth'
      if [ $? -eq 0 ] ; then
        logLine 3 "Loading DRG pwauth"
        sed -n 's/^.*|.*| *\([0-9\.][0-9\.]*\) .*$/\1/p' "$fileName" > "$tmpFile2"
      else
        logLine 3 "Loading plain IP list"
        sed -n 's/^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)[ 	]*.*$/\1/p' "$fileName" > "$tmpFile2"
      fi
      if [ -x /opt/bin/iprange ] ; then
        # remove IP's that match -net ranges
        # ToDo move to end of processing (after ranges are removed that match whitelisted IP's)
        ipset list $swingNetSetName | tail -n +9 > "$tmpFile"
        if [ -s "$tmpFile" -a -s "$tmpFile2" ] ; then
          logLine 3 "tmpFile (-net) and tmpFile2 not empty"
          [ $logLevel -eq 3 ] && setSizeBefore=`grep -c ^ "$tmpFile2"`
          /opt/bin/iprange -1 "$tmpFile2" --except "$tmpFile" > "$tmpFile3"
          mv "$tmpFile3" "$tmpFile2"
          [ $logLevel -eq 3 ] && setSize=`grep -c ^ "$tmpFile2"`
          [ $logLevel -eq 3 ] && logLine 3 "RBL ($rbl) reduced by $((setSizeBefore - setSize)) entries, matching $netSetName"
        fi
      fi
    elif [ $setType = net ] ; then
        logLine 3 "Loading plain Net list from $fileName"
        sed -n 's#^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*[/0-9]*\)[ 	]*.*$#\1#p' "$fileName" > "$tmpFile2"
    fi
    if [ -n "$whiteList" ] ; then
      logLine 3 "Applying whitelist"
      #    -E = extended regular expression (ERE) grep
      #     -v = invert match, to select non-matching lines
      grep -Ev "$whiteList" "$tmpFile2" > "$tmpFile"
      mv "$tmpFile" "$tmpFile2"
    fi
    # Prefix tmp file
    sed -i "s/^/add "$tempSet"_swing /" "$tmpFile2"
    [ $logLevel -gt 1 ] && setSizeBefore=`ipset save "$tempSet"_swing | grep -Ec '^add '`
    ipset restore -f "$tmpFile2" -\!
    setSize=`ipset save "$tempSet"_swing | grep -Ec '^add '`
    [ $logLevel -gt 1 ] && logLine 2 "RBL ($rbl) added $((setSize - setSizeBefore)) entries"
    [ $setType = net ] && intNetEntries=$((setSize))
  fi
done

logLine 1 "Added $setSize entries (net:$intNetEntries)"

if [ -x /opt/bin/iprange ] ; then
  logLine 3 '/opt/bin/iprange is executable'
  if [[ $intNetEntries -gt 0 ]] ; then
    logLine 2 'Optimizing '$netSetName'_swing ('$intNetEntries' entries)'
    ## Optimize, speed and memory (59% lower, Size in memory, in test case)
    # ipset list sub2rbl-net | tail -n +9 | iprange --ipset-reduce 20 --print-prefix 'add sub2rbl-net ' > /tmp/tmp.lasIymKg
    ipset list $netSetName'_swing' | tail -n +9 | /opt/bin/iprange --ipset-reduce 20 --print-prefix 'add '$netSetName'_swing ' > $tmpFile
    # Flush ...-net_swing and Load from tmpFile
    ipset flush $netSetName'_swing'
    ipset restore -f "$tmpFile"
fi; fi

rm -f "$tmpFile" "$tmpFile2" "$tmpFile3"

for tempSet in "$netSetName" "$setName" ; do
  if ipset list -q -n "$tempSet"_swing >/dev/null ; then
    setupIPTables "$tempSet"
    logLine 3 "Swinging set $tempSet"
    ipset swap "$tempSet" "$tempSet"_swing
    removeSet 2 "$tempSet"_swing
  fi
done

if [ ! -z "$rblList" ] ; then
	# save backup backup for restore on router restart
	ipset -file /opt/etc/ipset.conf save
fi
	#!/bin/sh
	#
	# sub2rbl for padavan firmware @ProBackup-nl
	# ============
	# Subscribes to IP or CIDR (netblock) RBLs; auto-integrates into iptables
	# Purely for reduced memory consumption it creates two IP sets:
	# 1. one for unicast IPs (sub2rbl)
	# 2. the other one for CIDR (sub2rbl-net) blocks
	# Realtime block lists contain both types
	#
	# https://gist.github.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7
	#

	# Dependencies
	# ============
	# entware(-ng-3x)
	# awk + xargs + mktemp
	# wget-ssl \|\| curl (>=v7.73.0) + ca-certificates (186k) needed if using HTTPS RBLs
	# kernel modules xt_set, ip_set_hash_ip, ip_set_hash_net
	# built-in-firmware: ipset, iptables, grep, lsmod, grep, sed, mv, rm
	# optional: iprange

	# Installation
	# ============
	# 0. mkdir -p /opt/usr/sbin;NAME=sub2rbl;wget -O /opt/usr/sbin/$NAME https://gist.githubusercontent.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7/raw/sub2rbl && chmod 755 /opt/usr/sbin/$NAME
	# 1. Go to "Padavan Router > Customization > Scripts" web interface page and put following content to "Run After Router Started" field:
	# modprobe ip_set_hash_ip
	# modprobe ip_set_hash_net
	# modprobe xt_set
	# ipset create sub2rbl hash:ip maxelem 32768
	# ipset create sub2rbl-net hash:net maxelem 32768
	#
	# 2. nano /opt/etc/init.d/S01system
	# *) start
	# [ -s '/opt/etc/ipset.conf' ] && ipset -file /opt/etc/ipset.conf restore
	# func_start
	#
	# 3. Go to "Padavan Router > Administration > Services" web interface page and enable "Cron Daemon (Scheduler)"
	#
	# 4. Go to "Padavan Router > Administration > Services" web interface page and put following content to "Scheduler tasks (Crontab)" field:
	# 1-59/13 * * * * /opt/usr/sbin/sub2rbl
	#
	# This will run the update process every 13 minutes past the hour + 1 minute
	#
	# 5. nano /opt/etc/init.d/S10iptables
	# start\|update)
	# # add iptables sub2rbl rules
	# [ -d '/opt/etc' ] \|\| exit 0
	# iptables -I INPUT 1 -m set --match-set sub2rbl src -j DROP
	# iptables -I INPUT 1 -m set --match-set sub2rbl-net src -j DROP
	# iptables -I FORWARD 1 -m set --match-set sub2rbl src -j DROP
	# iptables -I FORWARD 1 -m set --match-set sub2rbl-net src -j DROP
	# ;;
	#
	# 6. ssh into your router and run the update process once to generate the configuration file
	# sub2rbl -l 3 -f stdout
	#
	# 7. enhance your configuration by excluding your own IP-adresses and/or CIDR blocks by whitelisting them
	# nano /opt/etc/config/sub2rbl.conf
	#
	# 8. run the update process once more
	# sub2rbl -l 3 -f stdout

	# Improvements / ToDo
	# ===================
	# - remove (very) old cached downloaded files when upstream is gone (current: on reboot)
	# - process rbl lists according to their individual update frequency (even skip if-modified check)
	# - whitelist from url

	#
	# Advanced settings
	#
	setName=sub2rbl
	logLevel=1
	logFacility=authpriv.notice
	firewallTarget=DROP
	firewallHookChains='INPUT:1 FORWARD:1'
	netSetName="$setName"-net
	logTag="sub2rbl[$$]"
	ipsetArgs="maxelem 32768"
	webGetCmd=guess
	umask 0002 # no write access for other

	if [ ! -d /tmp/$setName ] ; then
	mkdir -m 664 /tmp/$setName
	fi

	# if webGetCmd is set to guess (default), we'll try to autoselect
	# between a few options for an TLS ready http client.
	[ "$webGetCmd" = guess ] && {
	if [ -x /opt/bin/wget-ssl ] ; then
	# -q quiet
	# -T timeout (seconds)
	# -t tries
	# -O output document
	# -N time-stamping, only save newer files
	# -P set directory prefix to prefix
	webGetCmd="/opt/bin/wget-ssl -qT15 -t2 -N -P /tmp/$setName -O" # GNU wget
	elif [ -x /opt/bin/wget ] ; then
	webGetCmd="/opt/bin/wget -qT15 -t2 -N -P /tmp/$setName -O" # entware renamed wget-ssl to wget (wget became wget-nossl)
	elif [ -x /opt/bin/curl ] ; then
	# -s silent
	# -S show error
	# -R remote-time
	# -O output file and name file to remote
	# -m maximum transfer time (seconds)
	# -z time-conditional requires file, thus skipped
	webGetCmd="/opt/bin/curl -sSROm15 --output-dir /tmp/$setName -z" # curl with mbedTLS - most lightweight option
	else
	# lastly, busybox wget - bad option as it fails on some TLS connections like spamhaus
	webGetCmd="/usr/bin/wget -qT15 -t2 -P /tmp/$setName -O"
	fi
	}

	# Functions
	#
	logLine() {
	[ $1 -gt $logLevel ] && return ; shift
	if [ "$logFacility" = "stdout" ] ; then echo "$@"
	elif [ "$logFacility" = "stderr" ] ; then echo "$@" >&2
	else logger -t "$logTag" -p "$logFacility" "$@"
	fi
	}

	# Arg: $1=logLevel, $2=setName, $3=setType (ip or net)
	createSet() {
	local useLogLevel="$1" set="$2" setType="$3"
	ipset list -q -n "$set" >/dev/null \|\| {
	logLine $useLogLevel "Creating ipset $set"
	ipset create "$set" hash:"$setType" $ipsetArgs
	}
	}

	# Arg: $1=logLevel, $2=setName
	removeSet() {
	local useLogLevel=$1 set="$2"
	ipset list -q -n "$set" >/dev/null && {
	logLine $useLogLevel "Destroying ipset $set"
	ipset destroy "$set"
	} }

	# Arg: 1=$setName
	setupIPTables() {
	local set="$1" x chain position
	for x in $firewallHookChains ; do
	chain="${x%:}" ; position="${x#:}"
	if [ $position -ge 0 ] && ! iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
	if [ $position = 0 ] ; then
	logLine 1 "Appending iptables rule for $set into chain $chain"
	iptables -A "$chain" -m set --match-set "$set" src -j "$firewallTarget"
	else
	logLine 1 "Inserting iptables rule at position $position for $set into chain $chain"
	iptables -I "$chain" $position -m set --match-set "$set" src -j "$firewallTarget"
	fi ; fi ; done
	}

	# Arg: 1=$setName
	removeFromIPTables() {
	local set="$1" x chain position
	for x in $firewallHookChains ; do
	chain="${x%:}" ; position="${x#:}"
	if [ $position -ge 0 ] && iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
	logLine 1 "Removing iptables rule for $set from chain $chain"
	iptables -D "$chain" -m set --match-set "$set" src -j "$firewallTarget"
	fi ; done
	}

	printUsage() {
	cat <<-_EOF_
	Usage: sub2rbl [-C ...] [-f ...] [-j ...] [-l #] [-s ...] [-w] [URL ...]
	-C ... firewall hook chains:positions (def: $firewallHookChains)
	-f ... log facility (syslog facility or stdout/stderr) (def: $logFacility)
	-j ... firewall target when matching the set (def: $firewallTarget)
	-l # log level - 0=off, 1=standard, 2=verbose (def: $logLevel)
	-s ... ipset name
	-w wipe ipset and iptables rule (do not process RBLs)
	URL(s) RBL URLs - using command line RBL URLs will override uci entries
	_EOF_
	}

	writeConfigFile() {
	cat > /opt/etc/config/sub2rbl.conf <<'_EOF_'
	#
	# sub2rbl-padavan - https://github.com/ProBackup-nl/sub2rbl-padavan
	#
	# Save this file at /opt/etc/config/sub2rbl.conf
	# Indent using tab-characters (TSV: tab separated value list)
	# Values must be quoted

	# Whitelist regex strings examples
	# list whitelist '^10\.0\.[01]\..*$'
	list whitelist '^127\.0\.0\.0/8$'
	list whitelist '^192\.168\.0\.0/16$'

	# RBL URLs - some (but not all) will also support http
	# Prepend net: to the URL for CIDR (net) RBLs
	# Order: net: URL's first (allows skipping IPs already blocked with net:)

	### https://www.spamhaus.org/drop - Spamhaus (Extended) Don't Route Or Peer List - spammer/cyber-crime net based RBLs
	### check frequency: 12 hours
	list rbl 'net:https://www.spamhaus.org/drop/drop.txt'
	list rbl 'net:https://www.spamhaus.org/drop/edrop.txt'

	### Bogons and also IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user
	### WARNING: Rigorously test this filter because it contains 127. 192. 172. and 10. non routable space.
	list rbl 'net:https://team-cymru.org/Services/Bogons/fullbogons-ipv4.txt'

	### http://ipdeny.com - provides lists of IPs associated with countries. Ex: CN
	# check frequency: 1 day
	# list rbl 'net:http://www.ipdeny.com/ipblocks/data/aggregated/cn-aggregated.zone'

	### http://rbldata.interserver.net/ - malware posts, ssh/ftp/email/modsec scans, spam and more
	### iprbl.txt = full (12K), standard ip.txt = last 7 days (2K), ipslim.txt = last 48 hours (1K)
	### check frequency: 3 minutes
	## list rbl 'https://sigs.interserver.net/iprbl.txt'
	list rbl 'https://rbldata.interserver.net/ip.txt'
	## list rbl 'https://rbldata.interserver.net/ipslim.txt'

	### https://www.blocklist.de/en - blacklists generated from fail2ban feedback, last 48 hours
	### check frequency: 15 minutes
	list rbl 'https://lists.blocklist.de/lists/ssh.txt'
	### All IPs which # are older then 2 month and have more then 5.000 attacks.
	list rbl 'https://lists.blocklist.de/lists/strongips.txt'

	_EOF_
	}

	if [ ! -f /opt/etc/config/sub2rbl.conf -o ! -s /opt/etc/config/sub2rbl.conf ] ; then
	writeConfigFile
	whitelist=''
	rblList=''
	else
	# Read configuration from (uci style) configuration file /opt/etc/config/sub2rbl.conf
	whiteList=$(awk '!(/^\t*#/) && $2 == "whitelist" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf \| xargs \| sed -e 's/ /\|/g')
	rblList=$(awk '!(/^\t*#/) && $2 == "rbl" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf)
	fi



	# Logic begins
	#
	wipe=false
	# Pure ash getopts replacement by https://gist.github.com/jhurliman/ef18f8cd1880dc00365d
	while [[ $# -gt 1 ]]; do
	arg="$1"
	case "$arg" in
	-C) firewallHookChains="$2"; shift ;;
	-f) logFacility="$2"; shift ;;
	-j) firewallTarget="$2"; shift ;;
	-l) logLevel="$2"; shift ;;
	-s) setName="$2"; shift ;;
	-w) wipe=true ;;
	*) printUsage
	exit 254
	esac
	shift
	done

	#before taking any action, first make sure that the required kernel modules are loaded:
	modList='xt_set
	ip_set_hash_ip
	ip_set_hash_net'
	#if not, load them using modprobe
	#if loading fails, abort
	for mod in $modList ; do
	if [ -z "lsmod \| grep '^$mod '" ] ; then
	logLine 2 "Loading kernel module ($mod)"
	modprobe "$mod"
	wait 1
	if [ -z "lsmod \| grep '^$mod '" ] ; then
	logLine 1 "Failure loading required kernel module ($mod)"
	exit 78
	fi
	fi
	done

	if $wipe ; then
	logLine 2 "Wiping sub2rbl from iptables and ipset"
	for tempSet in "$setName" "$netSetName" ; do
	removeFromIPTables "$tempSet"
	removeSet 1 "$tempSet"
	done
	# remove ipset backup file too
	[ -f /opt/etc/ipset.conf ] && rm /opt/etc/ipset.conf
	exit
	fi

	# Download updates when available
	# WARNING: each rbl MUST use a unique file name
	newFile=false
	for rbl in $rblList ; do
	if [ ${rbl%%:*} = net ] ; then
	rbl="${rbl##net:}"
	fi
	logLine 2 "Retrieving RBL ($rbl)"
	fileName=/tmp/$setName/`basename $rbl`
	if ! $newFile; then
	[ -s $fileName ] && touch -r $fileName $fileName'.time'
	fi

	#logLine 3 "$webGetCmd $fileName $rbl"
	if ! errorMsg=`$webGetCmd "$fileName" "$rbl" 2>&1`; then
	logLine 1 "Error: retrieving RBL($rbl): $errorMsg"
	# ToDo: remove really old files (current = cache forever)
	continue
	fi

	if ! $newFile; then
	if [ ! -e $fileName'.time' -a -s $fileName ] ; then
	logLine 3 '= new file (not exists before and not empty after)'
	newFile=true
	elif [ -e $fileName'.time' -a $fileName'.time' -ot $fileName ] ; then
	logLine 3 '= modified'
	newFile=true
	elif [ -e $fileName'.time' -a ! -e $fileName ] ; then
	logLine 3 '= removed (exist before and not exists after)'
	newFile=true
	fi
	[ -e $fileName'.time' ] && rm $fileName'.time'
	fi
	done

	if ! $newFile; then
	logLine 1 "No updates, exit."
	exit 0
	fi

	swingSetName="${setName}_swing"
	swingNetSetName="${netSetName}_swing"
	intNetEntries=0
	tmpFile="`/opt/bin/mktemp`"
	tmpFile2="`/opt/bin/mktemp`"
	tmpFile3="`/opt/bin/mktemp`"

	# Process
	# configure net based first then IP based to be able to remove overlaps
	for rbl in $rblList ; do
	if [ ${rbl%%:*} = net ] ; then
	setType='net' # process net-based RBLs
	rbl="${rbl##net:}"
	tempSet="$netSetName"
	else
	setType='ip' # default is IP-based (default)
	tempSet="$setName"
	fi
	fileName=/tmp/$setName/`basename $rbl`
	createSet 1 "$tempSet" $setType
	createSet 2 "$tempSet"_swing $setType
	#if ! errorMsg=`$webGetCmd "$tmpFile" "$rbl" 2>&1`; then

	# check that fileName exists and is not empty
	if [ -s $fileName ] ; then
	logLine 3 "Processing RBL ($rbl)"

	if [ $setType = ip ] ; then
	# autodetect the DRG sshpwauth report
	grep -Ev '^#' "$fileName" \| head -1 \| grep -q '^.\|.\|.\|.\|.*sshpwauth'
	if [ $? -eq 0 ] ; then
	logLine 3 "Loading DRG pwauth"
	sed -n 's/^.\|.\| \([0-9\.][0-9\.]\) .*$/\1/p' "$fileName" > "$tmpFile2"
	else
	logLine 3 "Loading plain IP list"
	sed -n 's/^ \([1-9][0-9]\.[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9]\)[ ].*$/\1/p' "$fileName" > "$tmpFile2"
	fi
	if [ -x /opt/bin/iprange ] ; then
	# remove IP's that match -net ranges
	# ToDo move to end of processing (after ranges are removed that match whitelisted IP's)
	ipset list $swingNetSetName \| tail -n +9 > "$tmpFile"
	if [ -s "$tmpFile" -a -s "$tmpFile2" ] ; then
	logLine 3 "tmpFile (-net) and tmpFile2 not empty"
	[ $logLevel -eq 3 ] && setSizeBefore=`grep -c ^ "$tmpFile2"`
	/opt/bin/iprange -1 "$tmpFile2" --except "$tmpFile" > "$tmpFile3"
	mv "$tmpFile3" "$tmpFile2"
	[ $logLevel -eq 3 ] && setSize=`grep -c ^ "$tmpFile2"`
	[ $logLevel -eq 3 ] && logLine 3 "RBL ($rbl) reduced by $((setSizeBefore - setSize)) entries, matching $netSetName"
	fi
	fi
	elif [ $setType = net ] ; then
	logLine 3 "Loading plain Net list from $fileName"
	sed -n 's#^ \([1-9][0-9]\.[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][/0-9]\)[ ].$#\1#p' "$fileName" > "$tmpFile2"
	fi
	if [ -n "$whiteList" ] ; then
	logLine 3 "Applying whitelist"
	# -E = extended regular expression (ERE) grep
	# -v = invert match, to select non-matching lines
	grep -Ev "$whiteList" "$tmpFile2" > "$tmpFile"
	mv "$tmpFile" "$tmpFile2"
	fi
	# Prefix tmp file
	sed -i "s/^/add "$tempSet"_swing /" "$tmpFile2"
	[ $logLevel -gt 1 ] && setSizeBefore=`ipset save "$tempSet"_swing \| grep -Ec '^add '`
	ipset restore -f "$tmpFile2" -\!
	setSize=`ipset save "$tempSet"_swing \| grep -Ec '^add '`
	[ $logLevel -gt 1 ] && logLine 2 "RBL ($rbl) added $((setSize - setSizeBefore)) entries"
	[ $setType = net ] && intNetEntries=$((setSize))
	fi
	done

	logLine 1 "Added $setSize entries (net:$intNetEntries)"

	if [ -x /opt/bin/iprange ] ; then
	logLine 3 '/opt/bin/iprange is executable'
	if [[ $intNetEntries -gt 0 ]] ; then
	logLine 2 'Optimizing '$netSetName'_swing ('$intNetEntries' entries)'
	## Optimize, speed and memory (59% lower, Size in memory, in test case)
	# ipset list sub2rbl-net \| tail -n +9 \| iprange --ipset-reduce 20 --print-prefix 'add sub2rbl-net ' > /tmp/tmp.lasIymKg
	ipset list $netSetName'_swing' \| tail -n +9 \| /opt/bin/iprange --ipset-reduce 20 --print-prefix 'add '$netSetName'_swing ' > $tmpFile
	# Flush ...-net_swing and Load from tmpFile
	ipset flush $netSetName'_swing'
	ipset restore -f "$tmpFile"
	fi; fi

	rm -f "$tmpFile" "$tmpFile2" "$tmpFile3"

	for tempSet in "$netSetName" "$setName" ; do
	if ipset list -q -n "$tempSet"_swing >/dev/null ; then
	setupIPTables "$tempSet"
	logLine 3 "Swinging set $tempSet"
	ipset swap "$tempSet" "$tempSet"_swing
	removeSet 2 "$tempSet"_swing
	fi
	done

	if [ ! -z "$rblList" ] ; then
	# save backup backup for restore on router restart
	ipset -file /opt/etc/ipset.conf save
	fi