Skip to content

Instantly share code, notes, and snippets.

@mkorthof

mkorthof/ipset-logpat.sh

Last active Jan 18, 2018
Embed
What would you like to do?
creates ipset sets matching certain patterns in httpd logs
#!/bin/bash
# ipset-logpat
# searches httpd access logs for pattern, whoises matching ip's and uses
# ip blocks to create ipset set. also adds iptables rules to log and reject
# requires: iptables, ipset, aggregate (optional)
# other useful ipset commands: ipset list [-terse], ipset destroy
# more info:
# https://mattwilcox.net/web-development/unexpected-ddos-blocking-china-with-ipset-and-iptables
# https://github.com/jordanrinke/ipsets-persistent
# http://forums.debian.net/viewtopic.php?f=5&t=127437
# logs: you can use (...) syntax (array) for wildcards/globs like * or ? e.g.:
# logs=(/var/log/lighttpd/access.log*)
logs=(/var/log/lighttpd/access.log{,.1})
pattern="badhost.com"
whoisobj="^route:"
setname="badhost"
ipsout="/etc/iptables/${setname}.txt"
curdate="$( date +%y%m%d%H%M%S )"
stdlog="/var/log/ipset-${setname}.log"
func_help () {
cat <<-EOF
ipset-logpattern
Usage:
"$0 [--cron|--quiet] [--iplog|--ipset|--iptables|--all]"
[-c|--cron] cron: log stdout/err to "$stdlog"
[-q|--quiet] quiet: dont output to stdout/err
[-l|--iplog] iplog: create "$ipsout" using logs
[-i|--ipset] ipset: ipset create "$setname"
[-t|--ipset] iptables: insert iptables rule for ipset
[-a|--all] all: all of the above (default w/o args)
Settings:
variables can be edited in script or by using these arguments:
[--logs|--pattern|--whoisobj|--setname|--ipsout|--stdlog] <value>
Example:
"$0 -c --pattern "bad host" --setname badhost --ipsout /etc/badhosts.txt
EOF
}
std="/dev/stdin"
opt_ips="-quiet"
# md5sum "${ipsout}"* | grep -v "${ipsout}$" | grep "$( md5sum "$ipsout" | awk '{ print $1 }' )" \
# && echo "OK" || echo "NOK"
func_ipl () {
[ -s "$ipsout" ] && mv "$ipsout" "$ipsout.$curdate" || { [ -f $ipsout ] && rm "$ipsout"; }
for i in $( zgrep "$pattern" "${logs[@]}" | cut -d":" -f2 | cut -d" " -f1 | sort -u ); do
/sbin/ipset test "$opt_ips" "$setname" "$i" || { whois "$i" | grep "$whoisobj"; }
done | awk '{ print $(NF) }' | sort -u >>"$ipsout"
if /usr/bin/which aggregate >/dev/null 2>&1; then
/usr/bin/aggregate -q < "$ipsout" >"${ipsout}.$$.tmp" && mv "${ipsout}.$$.tmp" "$ipsout"
fi
if [ -s "$ipsout" ]; then logc="OK - $( wc -l < "$ipsout" ) lines"
else logc="NOK"; [ -f $ipsout ] && rm "$ipsout"; fi
echo "$( date +%F\ %T ) iplog: create ip block list \"$ipsout\" - $logc"
}
func_ips () {
/sbin/ipset create -exist "$opt_ips" "$setname" hash:net && ipsc="OK" || ipsc="NOK"
echo "$( date +%F\ %T ) ipset: create set \"$setname\" - $ipsc"
for i in ${ipsout}*; do
while read l; do /sbin/ipset add -exist "$opt_ips" "$setname" "$l"; done < "$i" && \
ipsa="OK" || ipsa="NOK"
echo "$( date +%F\ %T ) ipset: add entries from \"$i\" to \"$setname\" - $ipsa"
done
}
func_ipt () {
# with port: -A INPUT -p tcp -m tcp --dport 80 -m set --match-set $setname src -j LOGIPS
# iptables rc: 0 chain already exists, 1 chain doesnt exist, grep match rc: 0 (exists)
rulenum=3
rules=(
'-N LOGIPS'
'-A LOGIPS -m limit --limit 10/min -j LOG --log-prefix "IPS REJECT: " --log-level 6'
'-A LOGIPS -j REJECT --reject-with icmp-port-unreachable'
'-I INPUT '"$rulenum"' -p tcp -m set --match-set '"$setname"' src -j LOGIPS'
)
rejrule="-I INPUT $rulenum -p tcp -m set --match-set $setname src -j REJECT --reject-with icmp-port-unreachable"
i=0; while [ "$i" -lt "${#rules[@]}" ]; do
act="$( echo ${rules[$i]} | cut -d" " -f1 )"; chain="$( echo ${rules[$i]} | cut -d" " -f2 )"
if [ "$act" = "-N" ]; then
/sbin/iptables -n -L "$chain" >/dev/null 2>&1; rc="$?"
elif [ "$act" = "-A" ] || [ "$act" = "-I" ]; then
target="$( echo "${rules[$i]}" | sed "s/-A\?I\? ${chain}.*\?-j \([A-Z]\+\)\( .*\|$\)/\1/" )"
/sbin/iptables -n -L "$chain" | grep -q "^${target}"; rc="$?"
fi
if [ "$rc" -eq 0 ]; then res="already exists"; else eval /sbin/iptables "${rules[$i]}" && res="OK" || res="NOK"; fi
if [ "$act" = "-N" ]; then action="add"; target="new"
echo "$( date +%F\ %T ) iptables: create new $chain chain - $res"
else
if [ "$act" = "-A" ]; then action="append"
elif [ "$act" = "-I" ]; then action="insert"; fi
echo "$( date +%F\ %T ) iptables: $action $target rule to $chain chain - $res"
fi
i=$((i+1))
done
/sbin/iptables -n -L "${rules[0]/#-? /}" >/dev/null 2>&1 || { \
echo "$( date +%F\ %T ) iptables: could not create chain ${rules[0]/#-? /}, try replacement rule..."
eval /sbin/iptables "$rejrule" && res="OK" || res="NOK"
echo "$( date +%F\ %T ) iptables: insert REJECT rule to INPUT chain - $res"
}
unset act chain target rc res
}
# testing using $# 0 and (*) instead of the 'if' statement below
# if [ $# = 0 ]; { func_ipl; func_ips; func_ipt; } >>"$std" 2>&1; exit 0; fi
for (( ; $# >= 0; )); do
case $1 in
(start|restart|reload|force-reload) { func_ips; func_ipt; }; exit 0 ;;
(flush) /sbin/ipset flush "$setname"; exit 0 ;;
(stop) echo "Automatic flushing disabled, use \"flush\" instead of \"stop\""; exit 0; ;;
(-c|--cron) std="$stdlog"; shift ;;
(-q|--quiet) std="/dev/null"; shift ;;
(-d|--debug) unset opt_ips; shift ;;
(--logs) shift; logs="$1"; shift ;;
(--whoisobj) shift; whoisobj="$1"; shift ;;
(--pattern) shift; pattern="$1"; shift ;;
(--setname) shift; setname="$1"; shift ;;
(--ipsout) shift; ipsout="$1"; shift ;;
(--stdlog) shift; stdlog="$1"; shift ;;
(-a|--all) { func_ipl; func_ips; func_ipt; } >>"$std" 2>&1; exit 0 ;;
(-l|--iplog|--iplogs) func_ipl >>"$std" 2>&1; shift; exit 0 ;;
(-i|--ips|--ipset) func_ips >>"$std" 2>&1; shift; exit 0 ;;
(-t|--ipt|--iptables) func_ipt >>"$std" 2>&1; shift; exit 0 ;;
(--) shift; break ;;
(-h|--help|-?|help) func_help; exit 0 ;;
(*) { func_ipl; func_ips; func_ipt; } >>"$std" 2>&1; exit 0 ;;
esac
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment