A downloader script to fetch the latest comodo waf rulebase from waf.comodo.com

download_comodo_waf_rules.py

#!/usr/bin/env python3
""" download_comodo_waf_rules.py - A very simple downloader script 

    This script updates the ModSecurity rulebase from waf.comodo.com automagically.
    It downloads the latest rulebase for both apache and nginx as tarbal and extracts the rules to a directory.
    
    To get started, replace the AUTH_USERNAME and AUTH_PASSWORD variables with your own credentials.
    (Create at https://accounts.comodo.com/cwaf/management/signup)

"""

import sys
import os
import re
import tarfile

if sys.version_info.major == 3:
    from http.cookiejar import FileCookieJar
    from urllib.parse import urlencode
    from urllib.request import build_opener, HTTPCookieProcessor, Request
else:
    from urllib import urlencode
    from urllib2 import build_opener, HTTPCookieProcessor, Request
    from cookielib import FileCookieJar


AUTH_USERNAME = 'you@example.com'
AUTH_PASSWORD = 'P4ssW0rd'
AUTH_TOKEN_URL = 'https://waf.comodo.com/user/auth'

BASE_DIR = '/tmp/rule_base'


def login(opener):
    auth_regex = b'<input id="authenticity_token" name="authenticity_token" type="hidden" value="(.*?)" />'
    auth_page = opener.open(AUTH_TOKEN_URL).read()
    auth_token = re.findall(auth_regex, auth_page, re.DOTALL)[0]

    credentials = urlencode({
        'login': AUTH_USERNAME,
        'password': AUTH_PASSWORD,
        'authenticity_token': auth_token,
        'commit': 'LOGIN'
    }).encode('ascii')

    request = Request(AUTH_TOKEN_URL, data=credentials)
    request.add_header("Content-Type", "application/x-www-form-urlencoded")
    request.add_header("Referer", AUTH_TOKEN_URL)
    request.get_method = lambda: "POST"
    opener.open(request)


def get_revisions(opener):
    revisions = {}
    base_url = 'https://waf.comodo.com/user/cwaf_revisions?source={source}'

    for webserver, source in (('apache', 0), ('nginx', 2)):
        data = opener.open(base_url.format(source=source))
        content = data.readlines()

        if webserver not in revisions:
            revisions[webserver] = []

        for line in content:
            line = line.decode('utf-8')
            match = re.findall('<option selected="selected" value="(.*?)">.*?</option>', line, re.DOTALL)
            if match and 'cwaf_revision_id' in line:
                revision = match[0]
                revisions[webserver] = dict(source=source, revision=int(revision))
    return revisions


def download_revision(opener, name, data):
    filename = '{}/{}_waf.tgz'.format(BASE_DIR, name)
    url = 'https://waf.comodo.com/user/cwaf_revisions/{revision}/download?full_version=1&source={source}'.format(
        revision=data['revision'],
        source=data['source'],
    )
    data = opener.open(url)
    with open(filename, 'wb') as fh:
        for line in data.readlines():
            fh.write(line)
    return filename


def download_waf_rules():
    cookiejar = FileCookieJar("cookies")
    opener = build_opener(HTTPCookieProcessor(cookiejar))

    login(opener=opener)
    data = get_revisions(opener=opener)

    downloaded_files = []

    for item in data:
        if not data[item]:
            print('Failed to retrieve WAF revisions from Comodo')
            sys.exit(1)

        print('Downloading WAF rules for {}'.format(item))

        filename = download_revision(
            opener=opener,
            name=item,
            data=data[item],
        )
        downloaded_files.append(filename)
    return downloaded_files


def unpack_waf_rules(package):
    if not os.path.isfile(package):
        return None

    destination = os.path.join(BASE_DIR, package.replace('.tgz', ''))
    if not os.path.isdir(destination):
        os.mkdir(destination)

    print('Unpacking {}'.format(package))

    tar = tarfile.open(package)
    tar.extractall(path=destination)
    tar.close()


if __name__ == "__main__":
    if not os.path.isdir(BASE_DIR):
        os.mkdir(BASE_DIR)

    files = download_waf_rules()

    for package in files:
        unpack_waf_rules(package=package)
        os.unlink(package)