Generates salt pillar sls with adv domains

download_latest_lists.py

import os
import re
import http.client


connection = http.client.HTTPConnection('pgl.yoyo.org')
connection.request('GET', '/as/serverlist.php?hostformat=one-line&mimetype=plaintext')
response = connection.getresponse()

res_lines = response.read().decode().split(os.linesep)
domains = set()

for line in res_lines:
    line = line.strip()
    if line.startswith('#'):
        continue
    domains.update(set(map(lambda s: s.strip(), line.split(','))))

non_valid = re.compile('[^A-Z0-9\.-]+', re.IGNORECASE)
for item in domains:
    if non_valid.search(item) is not None:
        raise RuntimeError('Found invalid characters in domain from external list `%s`' % item)
    chunks = item.split('.')
    if len(chunks) == 1:
        raise RuntimeError('Top level domain found `%s`' % item)
    for chunk in chunks:
        if len(chunk) == 0:
            raise RuntimeError('Wrong domain chunk found in `%s`' % item)

with open(os.path.join(os.path.dirname(__file__), 'init.sls'), 'w') as f:
    f.write('\n'.join([
        '#!py',
        'def run():',
        '    return {',
        '        "adv_hostnames": [',
        '             %s' % ',\n'.join(map(lambda x: '"%s"' % x, domains)),
        '         ]',
        '    }'
    ]))