grab and sort hosts to block for dnsmasq (this existed way before pihole)

hostsupdater.sh

#!/bin/bash

# We create a new /tmp/block file first, then append to it
# the 'sleep 1's are inserted because we suffered from slow/delayed disk cache writes
# which often caused this script to skip or not entirely finish pasting lists

wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/notracking/hosts-blocklists/master/hostnames.txt > /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://adaway.org/hosts.txt >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/yous/YousList/master/hosts.txt  >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k http://sysctl.org/cameleon/hosts >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://www.malwaredomainlist.com/hostslist/hosts.txt >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt >> /tmp/block
sleep 1
wget -qO - --limit-rate=1500k https://hosts-file.net/ad_servers.txt >> /tmp/block
sleep 1
#shit
#wget -qO - --limit-rate=1500k https://mirror1.malwaredomains.com/files/justdomains >> /tmp/block
#sleep 1

# Let's clean up all the mess from these sources
sed -i 's/\t/ /g' /tmp/block
sleep 1
sed -i 's/[[:space:]]*#.*$//g' /tmp/block
sleep 1
sed -i 's/  \+/ /g' /tmp/block
sleep 1
sed -i '/::/d' /tmp/block
sleep 1

# 0.0.0.0 resolves faster in my firefox speed tests
sed -i 's/127.0.0.1/0.0.0.0/g' /tmp/block
sleep 1

# We're on a linux device here, so correct linebreaks
dos2unix -q -n /tmp/block /tmp/bloc
sleep 1

# gotta love uniq (sorting and removing doubles)
sort /tmp/bloc | uniq -u -f1 > /tmp/blo
sleep 1

# we use our own stuff out of /etc/hosts, so cut the local crap ..
sed -i '/localhost/d' /tmp/blo
sed -i '/127.0.0./d' /tmp/blo
sed -i '/255.255.255.255/d' /tmp/blo
sed -i '/0.0.0.*0.0.0/d' /tmp/blo
sleep 1

# *allow* a couple of decent ad link-throughs, because they're tested for security and allow certain payments
for domain in `cat whitelist` ; do sed -i.bak "/$domain/d" /tmp/blo ; done

sleep 1

# we need to insert our LAN names in /etc/hosts at the top
sed -i '1r loopbacklocal' /tmp/blo
sleep 1

mv -f /tmp/blo /etc/hosts

# Still looking for a way to maintain dns cache while reloading /etc/hosts file into dnsmasq, for now it's:
killall -1 dnsmasq

# Let's make sure next run is a clean one
rm -rf /tmp/block
rm -rf /tmp/bloc

exit 0

Part of my DNSmasq configuration file (i.e. /etc/dnsmasq.conf )

# This mitigates a Critical GLibc Vulnerability, see http://dankaminsky.com/2016/02/20/skeleton/
# You must set your client (or dhcp-server) to use ONLY your dnsmasq resolver for DNS, no secondaries!
edns-packet-max=2048

domain-needed
# Never forward addresses in the non-routed address spaces.
bogus-priv

# Change this line if you want dns to get its upstream servers from
# somewhere other that /etc/resolv.conf
#resolv-file=

no-resolv
no-poll

all-servers
#server=1.1.1.1@ens3
server=192.168.1.1@ens3
server=146.185.176.36@ens3

interface=lo
bind-interfaces
listen-address=192.168.1.11,127.0.0.1

# If you want dnsmasq to provide only DNS service on an interface,
# configure it as shown above, and then use the following line to
# disable DHCP and TFTP on it.
no-dhcp-interface=lo

#log all dns queries
#log-queries
# write logs to stdout
#log-facility=-
log-async=25

# concurrent DNS queries. (default = 150)
dns-forward-max=200

# Cached number of domains
cache-size=10000

# cache non existent domain results and for how long (in seconds)
neg-ttl=3600

# keep dns resolving server available for local devices only
local-ttl=3

Inside /etc/crontab (about once every week turns out to be enough, most lists aren't even updated with that frequency):

55 4 */6 * *  root  /somepath/hostsupdater.sh >> /var/log/hosts.log 2>&1

the file named whitelist is in the same dir as the .sh script, and currently holds these for us because we need them to work:

googleadservices.com
dartsearch.net
econda-monitor.de
tradedoubler.com
ad.atdmt.com
doubleclick.net
pagead.l.doubleclick.net
amazon-adsystem.com
analytics.twitter.com
dasch.pl