Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@KrisLowet
Last active March 17, 2024 02:08
Show Gist options
  • Star 6 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save KrisLowet/675ba34e682c6d2afbc53fc317b41e85 to your computer and use it in GitHub Desktop.
Save KrisLowet/675ba34e682c6d2afbc53fc317b41e85 to your computer and use it in GitHub Desktop.
#!/bin/bash
# Paths and file names of the hosts list and the result.
hosts_list='dnsblock_hosts_'$(date '+%Y%m%d')'.txt' # One FQDN per line in file.
result='dnsblock_result_'$(date '+%Y%m%d')'.csv' # CSV result file.
#
# Download the most recent hosts lists and combine to one file.
#
echo -e "### Downloading hosts lists."
# List 1. Source: Abuse.ch URLhaus. URLhaus is a project from abuse.ch with the goal of sharing malicious URLs that are being used for malware distribution.
wget --quiet -O "dnsblock_test_list_1.txt" "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-hosts.txt"
# List 2. Source: cert.pl. List of malicious domains.
wget --quiet -O "dnsblock_test_list_2.txt" "https://hole.cert.pl/domains/domains.txt"
cat dnsblock_test_list_1.txt dnsblock_test_list_2.txt > dnsblock_test_list_concat.txt
sed -i '/^[[:blank:]]*#/d;s/#.*//' dnsblock_test_list_concat.txt # Remove comments.
sed -i 's/^0.0.0.0 //; /^#.*$/d; /^ *$/d; s/\t*//g' "dnsblock_test_list_concat.txt" # Remove '0' IPs; comments, empty space lines, and tabs.
sort dnsblock_test_list_concat.txt | uniq > $hosts_list # Create a list with unique hosts.
totalhosts=$(wc -l < $hosts_list)
timeexpected=`echo "$totalhosts" / 1.6 / 3600 | bc -l | awk '{printf("%.0f \n",$1)}'`
echo -e "### Hosts to test: $totalhosts"
#
# Define IP address of the nameserver used for lookups.
#
ns0_sp='Cloudflare unfiltered'
ns0_ip='1.1.1.1'
ns1_sp='Google Public DNS'
ns1_ip='8.8.8.8'
ns2_sp='ControlD Malware'
ns2_ip='76.76.2.1'
ns3_sp='Norton ConnectSafe'
ns3_ip='199.85.126.10'
ns4_sp='UltraDNS Threat Protection'
ns4_ip='156.154.70.2'
ns5_sp='Quad9'
ns5_ip='9.9.9.9'
ns6_sp='Cloudflare for Families'
ns6_ip='1.1.1.2'
ns7_sp='dns0.eu'
ns7_ip='193.110.81.0'
ns8_sp='dns0.eu ZERO'
ns8_ip='193.110.81.9'
ns9_sp='CleanBrowsing Security Filter'
ns9_ip='185.228.169.9'
ns10_sp='Comodo Secure DNS'
ns10_ip='8.26.56.26'
echo -e ", $ns0_sp - $ns0_ip,$ns1_sp - $ns1_ip,$ns2_sp - $ns2_ip,$ns3_sp - $ns3_ip,$ns4_sp - $ns4_ip,$ns5_sp - $ns5_ip,$ns6_sp - $ns6_ip,$ns7_sp - $ns7_ip,$ns8_sp - $ns8_ip,$ns9_sp - $ns9_ip,$ns10_sp - $ns10_ip" > "$result";
#
# Take the average ping to the nameservers.
#
echo "### Checking average ping to nameservers."
ip0=`ping -c 10 -q $ns0_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip1=`ping -c 10 -q $ns1_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip2=`ping -c 10 -q $ns2_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip3=`ping -c 10 -q $ns3_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip4=`ping -c 10 -q $ns4_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip5=`ping -c 10 -q $ns5_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip6=`ping -c 10 -q $ns6_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip7=`ping -c 10 -q $ns7_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip8=`ping -c 10 -q $ns8_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip9=`ping -c 10 -q $ns9_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
ip10=`ping -c 10 -q $ns10_ip | grep ^rtt | cut -d ' ' -f 4 | cut -d '/' -f 1`
echo -en "PING (ms),$ip0,$ip1,$ip2,$ip3,$ip4,$ip5,$ip6,$ip7,$ip8,$ip9,$ip10\n" >> "$result";
#
# Test a list of safe hosts to ensure that the nameservers are responding well.
#
echo "### Testing safe hosts to ensure nameservers are responding."
safe_hosts=( nexxwave.be nasa.gov google.com cloudflare.com microsoft.com ) # Declare a list with some known safe domains. Separated with a space.
for domain in "${safe_hosts[@]}"
do
ip0=`dig @$ns0_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip1=`dig @$ns1_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip2=`dig @$ns2_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip3=`dig @$ns3_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip4=`dig @$ns4_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip5=`dig @$ns5_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip6=`dig @$ns6_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip7=`dig @$ns7_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip8=`dig @$ns8_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip9=`dig @$ns9_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip10=`dig @$ns10_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
echo -e "Testing $domain";
echo -en "$domain (safe domain),$ip0,$ip1,$ip2,$ip3,$ip4,$ip5,$ip6,$ip7,$ip8,$ip9,$ip10\n" >> "$result";
done
#
# Do the test.
#
echo "### Start test of hosts at $(date). This will take ~$timeexpected hours."
echo -e "\n" >> "$result";
echo -e "Domain name, $ns0_sp - $ns0_ip,$ns1_sp - $ns1_ip,$ns2_sp - $ns2_ip,$ns3_sp - $ns3_ip,$ns4_sp - $ns4_ip,$ns5_sp - $ns5_ip,$ns6_sp - $ns6_ip,$ns7_sp - $ns7_ip,$ns8_sp - $ns8_ip,$ns9_sp - $ns9_ip,$ns10_sp - $ns10_ip" >> "$result";
while IFS= read -r domain
do
if [ "$domain" != "" ]; then # Ensure the line is not empty.
ip0=`dig @$ns0_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`; # IP address lookup with unfiltered DNS with a longer timeout.
if [ -n "$ip0" ] && [ "$ip0" != "0.0.0.0" ] && [ "$ip0" != "127.0.0.1" ]; then # Only run additional lookups if the domain is returning a valid IP address.
ip1=`dig @$ns1_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip2=`dig @$ns2_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip3=`dig @$ns3_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip4=`dig @$ns4_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip5=`dig @$ns5_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip6=`dig @$ns6_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip7=`dig @$ns7_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip8=`dig @$ns8_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip9=`dig @$ns9_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
ip10=`dig @$ns10_ip +noadflag +noedns +short $domain | grep '^[.0-9]*$' | tail -n1`
# Blank out any localhost or 0'ed IP addresses.
if [ "$ip1" = "127.0.0.1" ] || [ "$ip1" = "0.0.0.0" ]; then
ip1=""
fi
if [ "$ip2" = "127.0.0.1" ] || [ "$ip2" = "0.0.0.0" ]; then
ip2=""
fi
if [ "$ip3" = "127.0.0.1" ] || [ "$ip3" = "0.0.0.0" ]; then
ip3=""
fi
if [ "$ip4" = "127.0.0.1" ] || [ "$ip4" = "0.0.0.0" ]; then
ip4=""
fi
if [ "$ip5" = "127.0.0.1" ] || [ "$ip5" = "0.0.0.0" ]; then
ip5=""
fi
if [ "$ip6" = "127.0.0.1" ] || [ "$ip6" = "0.0.0.0" ]; then
ip6=""
fi
if [ "$ip7" = "127.0.0.1" ] || [ "$ip7" = "0.0.0.0" ]; then
ip7=""
fi
if [ "$ip8" = "127.0.0.1" ] || [ "$ip8" = "0.0.0.0" ]; then
ip8=""
fi
if [ "$ip9" = "127.0.0.1" ] || [ "$ip9" = "0.0.0.0" ]; then
ip9=""
fi
if [ "$ip10" = "127.0.0.1" ] || [ "$ip10" = "0.0.0.0" ]; then
ip10=""
fi
echo -e "Testing $domain";
echo -en "$domain,$ip0,$ip1,$ip2,$ip3,$ip4,$ip5,$ip6,$ip7,$ip8,$ip9,$ip10\n" >> "$result";
fi
sleep 0.1 # Don't hammer the public resolver.
fi
done < "$hosts_list"
echo "### End test of hosts at $(date)"
@WFANG12719
Copy link

This is a great script and good to know the result. (from Eric Sauvageau's retweet). I had a quick test of it on my router, and I found each domain took about 25 seconds to complete. So, roughly it would take me about 38 days to run the whole list. If it could be optimized to adapt the multi-threaded environment, it will significantly improve the efficiency.

@KrisLowet
Copy link
Author

25 seconds for 10 DNS requests is a long time. Maybe you can choose only the DNS resolvers of your interest and not all of them.

@WFANG12719
Copy link

Thanks, Kris. Yes, I selected some and add another two. BTW, I made some changes on this script and realized "multi thread" by using the Linux File Descriptor (FD) . I tested it on my router ( 4* ARMv8 ,2G RAM) with 12 DNS servers and it could finish within 6-8 hours. I tried 100 sessions on my router, it worked well. (first, I tested 20 threads, then 50, then 80, 100, CPU was about 80% but RAM usage was 40%)

@SkewedZeppelin
Copy link

always nice to see different approaches, I made a similar script recently
https://github.com/divestedcg/circumnavigator/blob/master/test.sh
https://divested.dev/misc/circumnavigator.txt

@WFANG12719
Copy link

@SkewedZeppelin , great!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment