Unbound domain block list generator

unbound-ad-generator.py

#!/usr/bin/python3

import argparse
import re
import shutil
import sys
import tempfile
import time
import urllib.request as urlrequest
from pathlib import Path

UTF_8: str = 'utf-8'

DNSMASQ_DOMAINS_REGEX = r'^address=/(.*)/'
HOSTS_REGEX = r'^(?:0.0.0.0|::) (.*)'
DOMAINS_REGEX = r'^(.*)'

# https://firebog.net
# https://filterlists.com
# https://github.com/EnergizedProtection/block

SOURCES: dict = {
    'https://raw.githubusercontent.com/notracking/hosts-blocklists/master/domains.txt': DNSMASQ_DOMAINS_REGEX,
    'https://raw.githubusercontent.com/notracking/hosts-blocklists/master/hostnames.txt': HOSTS_REGEX,
    'https://raw.githubusercontent.com/deathbybandaid/piholeparser/master/Subscribable-Lists/ParsedBlacklists/RUAdListCounters.txt': DOMAINS_REGEX,
    'https://v.firebog.net/hosts/Easyprivacy.txt': DOMAINS_REGEX,
    'https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt': DOMAINS_REGEX,
    'https://raw.githubusercontent.com/EnergizedProtection/block/master/basic/formats/domains.txt': DOMAINS_REGEX
}


def parse_nottracking(url: str, r: str, s: set):
    with urlrequest.urlopen(url) as response:
        if response.getcode() == 200:
            for line in response:
                # remove comments
                if not line.startswith(b'#'):
                    domain_match_groups = re.match(
                        r, line.decode(UTF_8)).groups()
                    if len(domain_match_groups) == 1:
                        domain = domain_match_groups[0]
                        if '..' in domain:
                            # e.g. foobar..duckdns.org
                            print(f'Invalid domain: {domain}')
                            continue
                        s.add(domain)
        else:
            exit(f'Domains list response code: "{response.getcode()}"')


def main(out_path: str):
    unblock_rules = set()

    for (url, r) in SOURCES.items():
        print(f'Load: {url}')
        parse_nottracking(url, r, unblock_rules)

    print('Generate output file')
    with tempfile.TemporaryFile() as out:
        out.write(b'###############################################\n')
        out.write((f'#File generated at: {time.asctime()}\n').encode(UTF_8))
        out.write((f'#Entries count: {len(unblock_rules)}\n').encode(UTF_8))
        out.write(b'###################[DOMAINS]###################\n')

        for rule in sorted(unblock_rules):
            out.write((f'local-zone: "{rule}" static\n').encode(UTF_8))

        out.seek(0)

        with open(out_path, 'w+b') as real_out:
            shutil.copyfileobj(out, real_out)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Unbound adblock list generator')
    parser.add_argument(
        '--out', default=str(Path.home().joinpath('unbound_adblock.conf')), help='Output file path')

    args = parser.parse_args()

    main(args.out)