jake!

jake

#!/usr/bin/env python2

import argparse
import contextlib
import logging
import re
import os
import sys
import types

from StringIO import StringIO

from parsimonious.grammar import Grammar


log = logging.getLogger(__name__)

# FIXME: add escaped grave

STR_GRAMMAR = r'''
script = block*
block = code / text
code  = grave text grave
text = ~"[A-Z 0-9\s\!\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\"\_\{\|\}\~\\\\]+"i
grave = "`"
'''

GRAMMAR = Grammar(STR_GRAMMAR)


class Config:
    comment_char = '#'
    dump_lookups = True
    dynamic_config_callable_name = 'jake_get_config'
    namespace_template_separator = '~~~'
    verbose = False
    strip_extra_whitespace = True
    lint = False
    dump_debug = False


@contextlib.contextmanager
def redir_stdout():
    old = sys.stdout
    new = StringIO()
    sys.stdout = new

    yield new

    sys.stdout = old


class Template(object):
    def __init__(self, template_path, namespace=None):
        self.namespace = namespace or Namespace()

        template = open(template_path).read()

        try:
            self.namespace_str, self.template = template.split(
                '\n{}\n'.format(Config.namespace_template_separator))
        except ValueError:
            self.template = template
            self.namespace_str = ''

    @classmethod
    def render(cls, name):
        template = cls(name)
        template.execute_namespace()
        return template.render_template()

    def run(self):
        self.execute_namespace()
        self.execute_template()

    def execute_namespace(self):
        if self.namespace_str:
            namespace = self.namespace

            exec self.namespace_str in namespace.module_namespace

            if namespace.config is None and Config.dynamic_config_callable_name in self.namespace.module_namespace:
                config_callable = self.namespace.module_namespace[Config.dynamic_config_callable_name]
                namespace.config = config_callable()

    def create_script(self):
        script = StringIO()

        for is_code, text in self.transform():
            if is_code:
                # gross approximation for printing values when necessary
                if len(text.split('\n')) == 1:
                    script.write('\nprint ({} or ""),\n'.format(text))
                else:
                    script.write(text)
            else:
                script.write('\nprint """{}""",\n'.format(text))

        return script.getvalue()

    def execute_script(self, script):
        if Config.dump_debug:
            sys.stderr.write(script)
            sys.stderr.flush()
        exec script in self.namespace

    def lint_script(self, script):
        script = self.namespace_str + script
        import subprocess
        try:
            # we ignore all E-class warnings because they're non-fatal,
            # style-based things.
            proc = subprocess.Popen(
                ['flake8', '--ignore', 'E', '-'],
                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT
            )
        except OSError as e:
            print('# Could not lint source: {}\n'.format(str(e)))
            return

        stdout, stderr = proc.communicate(script)

        if stdout:
            for line in stdout.splitlines():
                m = re.match(r".* undefined name '(.+)'", line)
                if m:
                    name, = m.groups(0)
                    if name in self.namespace.lookups:
                        continue
                    else:
                        try:
                            self.namespace[name]
                        except KeyError:
                            self.namespace.lookups.pop(-1)
                        else:
                            self.namespace.lookups.pop(-1)
                            continue

                name, line = line.split(':', 1)
                print('# {}'.format(line))
            print('')

    def render_template(self):
        self.namespace.freeze()

        script = self.create_script()
        with redir_stdout() as config:
            if Config.lint:
                self.lint_script(script)
            self.execute_script(script)
        output = config.getvalue()
        pretty_output = self.dump_pretty_output(output)

        return pretty_output

    def dump_pretty_output(self, output):
        pretty_output = StringIO()

        # prefix all the dynamic values that were looked up as comments at the
        # top of the file.
        if Config.dump_lookups:
            for name in sorted(self.namespace.lookups):
                value = self.namespace[name]

                if isinstance(value, types.FunctionType):
                    continue

                pretty_output.write('%s %s = %s\n' % (Config.comment_char, name, value))
            pretty_output.write('\n')

        if Config.strip_extra_whitespace:
            output = re.sub(r' \n', '\n', output)
            output = re.sub(r'  ', ' ', output)
            output = re.sub(r'\n\n+', '\n\n', output)

        pretty_output.write(output)

        return pretty_output.getvalue()

    def transform(self):
        parsed = GRAMMAR.parse(self.template)

        for block in parsed:
            assert len(block.children) == 1

            expr_name = block.children[0].expr_name

            text = block.text
            if expr_name == 'code':
                yield True, text[1:-1]
            else:
                yield False, text


class Namespace(dict):
    def __init__(self, config=None):
        self.config = config
        self.module_namespace = {}
        self.local_namespace = {}

        self._frozen = False
        self.lookups = []

    def freeze(self):
        self._frozen = True

    def __getitem__(self, key):
        self.lookups.append(key)

        try:
            if isinstance(self.config, dict):
                value = self.config[key]
            else:
                value = getattr(self.config, key)
        except (AttributeError, KeyError) as e:
            if self.config is not None:
                log.debug("Could not retrieve %s from config object", key)
        else:
            return value

        if key in self.module_namespace:
            value = self.module_namespace[key]
        elif key in self.local_namespace:
            value = self.local_namespace[key]
        else:
            raise KeyError(key)
        return value

    def __setitem__(self, key, value):
        if self._frozen:
            self.module_namespace[key] = value
        else:
            self.local_namespace[key] = value


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('templates', metavar='template', nargs='+',
                        help='Paths to templates to generate.')
    parser.add_argument('--no-dump-lookups', action='store_false',
                        default=True,
                        help="If given, don't dump variable lookups to beginning of config file as comments")
    parser.add_argument('--verbose', action='store_true',
                        help="Debugging output. Does not affect template output")
    parser.add_argument('--namespace-template-separator', type=str,
                        default=Config.namespace_template_separator,
                        help="Change separator between namespace and template separator.")
    parser.add_argument('--comment-char', type=str,
                        default=Config.comment_char,
                        help="Change comment character to use for dumped variable name lookups.")
    parser.add_argument('--dynamic-config-callable-name', type=str,
                        default=Config.dynamic_config_callable_name,
                        help="Change name of the callable to look for for retrieving dynamic configuration.")
    parser.add_argument('--output-to-dir',
                        default=None,
                        help="Rather than write to stdout, write templates to given directory. Filenames will be that of the template")
    parser.add_argument('--no-strip-extra-whitespace',
                        action='store_false',
                        default=Config.strip_extra_whitespace,
                        help="Don't remove trailing whitespace, extra newlines and double spaces.")
    parser.add_argument('--lint',
                        action='store_true',
                        default=Config.lint,
                        help="Lint script before it is executed. Warnings will be inserted toward the top of the file")
    parser.add_argument('--dump-debug',
                        action='store_true',
                        default=Config.dump_debug,
                        help="Dump script that is about to be executed.")
    return parser.parse_args()


def main():
    options = parse_args()

    Config.comment_char = options.comment_char
    Config.dynamic_config_callable_name = options.dynamic_config_callable_name
    Config.dump_lookups = options.no_dump_lookups
    Config.namespace_template_separator = options.namespace_template_separator
    Config.verbose = options.verbose
    Config.strip_extra_whitespace = options.no_strip_extra_whitespace
    Config.lint = options.lint
    Config.dump_debug = options.dump_debug

    for filename in options.templates:
        output = Template.render(filename)
        if options.output_to_dir:
            output_path = os.path.join(options.output_to_dir, os.path.basename(filename))

            with open(output_path, 'w') as f:
                f.write(output)
        else:
            sys.stdout.write(output)

if __name__ == '__main__':
    main()

jake invocation

$ ./bin/jake examples/squid.conf

squid.conf

class jake_get_config:
    direct_proxy_enabled = False
    transparent_proxy_enabled = False
    smp_enabled = True
    cache_mem = 256000
    disk_caches = []
    ssl_bump_server_first = False
    ssl_bump_enabled = False


import subprocess


def directive(command, *args, **kwargs):
    text = '%s %s %s' % (command, ' '.join(map(str, args)), ' '.join('%s=%s' % kv for kv in kwargs.iteritems()))
    print text


def http_port(port, *args, **kwargs):
    args = set(args)

    if kwargs.pop('ssl_bump', False):
        args.add('ssl-bump')
        kwargs['generate-host-certificates'] = 'on'
        kwargs['cert'] = '/path/to/cert'
        kwargs['key'] = '/path/to/key'

    https = kwargs.pop('https', False)

    return directive('http_port' if not https else 'https_port', port, *args, **kwargs)

def https_port(port, *args, **kwargs):
    return http_port(port, https=True, *args, **kwargs)


def workers(enabled):
    if enabled:
        cpu_count = len(subprocess.check_output(['grep', 'processor', '/proc/cpuinfo']).splitlines())
        return directive('workers', max(cpu_count/2, 1))


def cache_dir(config):
    if not config.enabled:
        return

    args = []


    if config.cache_type in ('ufs', 'aufs', 'diskd'):
        args.append('/var/cache/ufs/%d' % config.pk)
        args += ['%s' % (config.cache_size / 1024), '16', '256']
    else:
        args.append('/var/cache/rock/%d' % config.pk)
        args.append('%s' % (config.cache_size / 1024))

    if config.no_store:
        args.append('no-store')

    kwargs = {
        'min-size': config.min_value_object_size,
        'min-size': config.max_value_object_size,
    }

    return directive('cache_dir', config.cache_type, *args, **kwargs)


def ssl_bump(server_first, enabled):
    if not enabled:
        return

    if server_first:
        mode = 'server-first'
    else:
        mode = 'client-first'

    return directive('ssl_bump', mode)

~~~
acl localnet src 10.0.0.0/8
acl localnet src 172.16.0.0/12
acl localnet src 192.168.0.0/16
acl localnet src fc00::/7
acl localnet src fe80::/10


acl SSL_ports port 443
acl Safe_ports port 80
acl Safe_ports port 443
acl CONNECT method CONNECT

http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports

http_access allow localnet
http_access allow localhost

http_access allow localhost manager
http_access deny manager

# INSERT RULES HERE

http_access deny all

cache_effective_user proxy

coredump_dir /var/cache/squid

`
if direct_proxy_enabled:
    http_port(direct_proxy_port, ssl_bump=direct_ssl)

if transparent_proxy_enabled:
    http_port(65080, 'intercept')

    if transparent_ssl:
        https_port(65443, 'intercept', ssl_bump=transparent_ssl)

`

`workers(smp_enabled)`

cache_mem `cache_mem` bytes

`
for dir in disk_caches:
    if smp_enabled and dir.cache_type != 'rock':
        continue
    cache_dir(dir)
`

url_rewriter_program /usr/libexec/url_rewriter

url_rewrite_access allow all

`ssl_bump(ssl_bump_server_first, enabled=ssl_bump_enabled)`

refresh_pattern ^ftp: 1440 20% 10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern . 0 20% 4320

squid.conf output

# cache_mem = 256000
# direct_proxy_enabled = False
# disk_caches = []
# smp_enabled = True
# ssl_bump_enabled = False
# ssl_bump_server_first = False
# transparent_proxy_enabled = False

acl localnet src 10.0.0.0/8
acl localnet src 172.16.0.0/12
acl localnet src 192.168.0.0/16
acl localnet src fc00::/7
acl localnet src fe80::/10

acl SSL_ports port 443
acl Safe_ports port 80
acl Safe_ports port 443
acl CONNECT method CONNECT

http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports

http_access allow localnet
http_access allow localhost

http_access allow localhost manager
http_access deny manager

# INSERT RULES HERE

http_access deny all

cache_effective_user proxy

coredump_dir /var/cache/squid

workers 4

cache_mem 256000 bytes

url_rewriter_program /usr/libexec/url_rewriter

url_rewrite_access allow all

refresh_pattern ^ftp: 1440 20% 10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern . 0 20% 4320