Content Filtering on EdgeOS/VyOS

adblock.sh

# Auto blocking ads, malware, etc on EdgeOS/VyOS
#
# Blatant rip-off/integration of https://gist.github.com/OnlyInAmerica/75e200886e02e7562fa1 
# and http://www.bsdnow.tv/tutorials/dnsmasq 

# any requests to blocked domains will instead route here
pixelserv_ip="10.0.0.1"

final_list='/etc/dnsmasq.d/dnsmasq.adlist.conf'
tmp_list="$final_list.tmp"

# Slowly build file from all lists
curl -s -d mimetype=plaintext -d hostformat=unixhosts http://pgl.yoyo.org/adservers/serverlist.php? | sort > $tmp_list
curl -s http://winhelp2002.mvps.org/hosts.txt | grep -v "#" | grep -v "127.0.0.1" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | sort >> $tmp_list
curl -s https://adaway.org/hosts.txt | grep -v "#" | grep -v "::1" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> $tmp_list
curl -s http://hosts-file.net/.%5Cad_servers.txt | grep -v "#" | grep -v "::1" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> $tmp_list
curl -s http://www.malwaredomainlist.com/hostslist/hosts.txt | grep -v "#" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $3}' | grep -v '^\\' | grep -v '\\$' | sort >> $tmp_list
curl -s http://adblock.gjtech.net/?format=unix-hosts | grep -v "#" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> $tmp_list
curl -s http://someonewhocares.org/hosts/hosts | grep -v "#" | sed '/^$/d' | sed 's/\ /\\ /g' | grep -v '^\\' | grep -v '\\$' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> $tmp_list

# This is a 30+MB file... Just a warning
curl -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ http://adblock.mahakala.is/ | grep -v "#" | awk '{print $2}' | sort >> $tmp_list

# Sort the aggregated results and remove any duplicates before formatting
# it how dnsmasq wants it:
# 
# incoming: 101com.com
#			101order.com
#			123found.com
# 				...
# desired:	address=/101com.com/10.0.0.1 		# this is the pixelserv_ip
#			address=/101order.com/10.0.0.1
# 			address=/123found.com/10.0.0.1
# 				...

# Be patient here - it has many MBs of text to edit
cat $tmp_list | sort | uniq | sed 's/^/address=\//;s/$/\/10.0.0.1/' > $tmp_list

# TODO: Modify this original to work with what I am doing above instead (remove from whitelist file instead of single entries below)
# cat $tmp_list | sed $'s/\r$//' | sort | uniq | sed '/^$/d' | awk -v "IP=$pixelserv_ip" '{sub(/\r$/,""); print IP" "$0}' > $final_list

# strip out just a couple desired sites (temporary)
if [ -f "$tmp_list" ]
then
        sed -i -e '/googleadservices\.com/d' $tmp_list
        sed -i -e '/google\-analytics\.com/d' $tmp_list
        sed -i -e '/googlesyndication\.com/d' $tmp_list
        sed -i -e '/analytics\.google\.com/d' $tmp_list
        mv $tmp_list $final_list
else
        echo "Error building the ad list, please try again."
        exit
fi

/etc/init.d/dnsmasq force-reload