Script for downloading list of host and IP blocks

hosts2dnscrypt.py

#!/usr/bin/env python


# Takes hosts file and converts to DNSCrypt blacklist-domains file
# so you could block malicious hosts on DNSCrypt
# (when alternating to DNSCrypt instead of dnsmasq in my scenario)
# hosts file from https://github.com/StevenBlack/hosts
# See https://dnscrypt.org/ IP/domain names blocking

import re
import os

badguys_pattern = re.compile(
    '^0.0.0.0(\s*|\t*)(.*)\n|^127.0.0.1(\s*|\t*)(.*)\n')
localhost_pattern = re.compile(
    '^0.0.0.0.*localhost\.localdomain.*$|^(?!\#)(.*)localhost.*$|^(?!\#).*broadcasthost.*$|^0.0.0.0.*local.*$')
comment_pattern = re.compile('#(.*)\n')

# os.unlink("blacklist-domains")
# touch blacklist-domains first if it doesn't exist
output = open("blacklist-domains", "w")

with open('hosts', 'r') as f:
    for line in f.readlines():
        if re.match(localhost_pattern, line):
            pass
        elif re.match(comment_pattern, line):
            pass
        else:
            m = badguys_pattern.match(line)
            if m:
                if m.group(2) is not None:
                    output.write(m.group(2) + "\n")

output.close()

# print "if using DNSCrypt-OSXClient on Mac"
# print "cp blacklist-domains /Library/Application\ Support/DNSCrypt/control/"
# print "sudo chown rrj:wheel /Library/Application\ Support/DNSCrypt/control/blacklist-domains"
# print "cp /Library/Application\ Support/DNSCrypt/control/blacklist-domains /Library/Application\ Support/DNSCrypt/control/blacklist-domains.tmp"
# print "Also update blocklist-ips from emerging threats"

hosts2hosts6.py

#!/usr/bin/env python
# Takes amalgamated hosts file and converts to file which could be imported to Murus Firewall
# so you could block malicious hosts via firewall rather then /etc/hosts file
# See

import os
import re

badguys_pattern = re.compile(
    '^0.0.0.0(\s*|\t*)(.*)\n|^127.0.0.1(\s*|\t*)(.*)\n')
localhost_pattern = re.compile(
    '^0.0.0.0.*localhost\.localdomain.*$|^(?!\#)(.*)localhost.*$|^(?!\#).*broadcasthost.*$|^0.0.0.0.*local.*$')
comment_pattern = re.compile('#(.*)\n')

out_file = "hosts6"

os.unlink(out_file)
# touch out_file first if it doesn't exist
output = open(out_file, "w")

with open('hosts', 'r') as f:
    for line in f.readlines():
        if re.match(localhost_pattern, line):
            pass
        elif re.match(comment_pattern, line):
            pass
            # output.write(line)
        else:
            m = badguys_pattern.match(line)
            if m:
                if m.group(2) is not None:
                    output.write("::\t" + m.group(2) + "\n")

output.close()

pl.rrj.totblok.plist

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>Label</key>
    <string>pl.rrj.totblok</string>
    <key>ProgramArguments</key>
    <array>
                	<string>/usr/local/bin/totblok</string>
    </array>
    <key>StartInterval</key>
    <integer>39939</integer>
    <key>StandardErrorPath</key>
    <string>/Users/rrj/Library/Logs/pl.rrj.totblok</string>
    <key>StandardOutPath</key>
    <string>/Users/rrj/Library/Logs/pl.rrj.totblok</string>
</dict>
</plist>

totblok.sh

#!/usr/bin/env zsh
# All-in-one. Total block.
# Download and merge lists of bad IPs and hosts
# Update DNSCrypt OSX client and dnsmasq

# DNSCrypt_DIR="/Library/Application Support/DNSCrypt/control"
DNSCrypt_DIR='/usr/local/etc/dnscrypt-proxy'
Totblock_DIR=$HOME/Documents/Python/totblok
Hosts_DIR=$HOME/Documents/Python/hosts

try_http_query() {
  exec alarmer 5 curl -L --max-redirs 5 -4 -m 5 --connect-timeout 5 -s \
    http://warriornl.darktech.org 2>/dev/null | \
  fgrep -c 'Success' > /dev/null 2>&1
}

case "$1" in
    # honor --noupdate option (for adding host to blacklist without re-downloading sources)
    noupdate|--noupdate|-n)
        NOUPDATE='--noupdate';;
    # honor --skipstatichosts option (for creating host files for outside hosts)
    skipstatichosts|--skipstatichosts|-s)
        STATICHOST='--skipstatichosts';;
    # include apple hosts - black out Apple
    blackapple|--blackapple|-a)
        BLACKAPPLE='TRUE';;
    # help 
    help|--help|-h)
        echo 'totblok.sh \n\tdownload lists of malicious hosts and IPs then refresh dnscrypt-proxy'
        echo 'totblok -s --skipstatichosts \n\tcall updatehosts.py with --skipstatichosts option == '
        echo 'totblok -n --noupdate \n\tcall updatehosts.py with --noupdate option == omit the standard section, at the top containing lines like 127.0.0.1 localhost'
        exit 0;;
    *)
        NOUPDATE='';
        STATICHOST='';;
esac

# output date and time for logfile
echo $(date +"%c")

# if not NOUPDATE check and wait for internet connection 
if [ -z $NOUPDATE ]; then
    local COUNTER=0
    while true; do
        let COUNTER=$((COUNTER+1))
        CONN=$(networksetup -getairportnetwork en0)
        if [[ $COUNTER -gt 16 ]]
        then
            echo "WiFi not available. Exiting..."
            exit 0
        elif [[ "$CONN" =~ .*Wi-Fi\ power\ is\ currently\ off.* ]]
        then
            sleep $COUNTER
        elif [[ "$CONN" =~ .*You\ are\ not\ associated.* ]] 
        then
            sleep $COUNTER       
        else
            echo "$COUNTER: $CONN"
            echo "WiFi is available. Checking internet connection"
            if try_http_query 
            then
                echo "Looks like we have WiFi and VPN on. Proceeding.."
                break
            else
                echo "No internet connection available. VPN is off? ..\nWaiting..."
                sleep $COUNTER
            fi
        fi
    done
fi
# set -x
cd "$Hosts_DIR"
# python3 "$Hosts_DIR"/updateHostsFile.py --noupdate --extensions porn social
python3 $Hosts_DIR/updateHostsFile.py --auto $NOUPDATE $STATICHOST --extensions porn social
if [ $? -eq 0 ]; then
    cp $Hosts_DIR/hosts $Totblock_DIR/hosts
    # dnsmasq
    cd $Totblock_DIR
    # python3 $Totblock_DIR/hosts2hosts6.py 
    # cp $Totblock_DIR/hosts /usr/local/etc/dnsmasq/hosts.dnsmasq
    # cp $Totblock_DIR/hosts6 /usr/local/etc/dnsmasq/hosts6.dnsmasq
    # DNSCrypt domains
    python3 $Totblock_DIR/hosts2dnscrypt.py
    cd $DNSCrypt_DIR
    cp $Totblock_DIR/blacklist-domains $DNSCrypt_DIR
    # cp blacklist-domains blacklist-domains.tmp
else
    echo 'updateHostsFile.py failed. ?'
fi
# Firehol IP's --> dnscrypt-proxy blacklist-ips
# cd /tmp
# /usr/local/bin/wget -q https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset
curl -Lso /tmp/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset &>/dev/null
if [ -f /tmp/firehol_level1.netset ]; then
        cd $DNSCrypt_DIR
        mv /tmp/firehol_level1.netset $DNSCrypt_DIR/blacklist-ips.tmp
        chown rrj:admin ./blacklist-ips.tmp
        rm -f $DNSCrypt_DIR/blacklist-ips
        mv $DNSCrypt_DIR/blacklist-ips.tmp $DNSCrypt_DIR/blacklist-ips
else
        echo 'Download from Firehol failed. No worries, we will retry next time.'
fi
# reload dnscrypt-proxy plugins (implies blacklists reload) via luanchctl, via SIGHUP
touch $DNSCrypt_DIR/reload
# check if files updated OK --> log
cd $DNSCrypt_DIR
ls -l
cd $HOME/Documents
# notify user
if [ -z $NOUPDATE ]; then

    /usr/local/bin/terminal-notifier -title "totblok.sh" \
        -message "totblok finished updating blacklists. find more in log" \
        -appIcon $HOME/Pictures/Icons/flat-osx/Apps-Tower.icns

fi