Skip to content

Instantly share code, notes, and snippets.

@nhoad

nhoad/jake

Created Feb 25, 2015
Embed
What would you like to do?
jake!
#!/usr/bin/env python2
import argparse
import contextlib
import logging
import re
import os
import sys
import types
from StringIO import StringIO
from parsimonious.grammar import Grammar
log = logging.getLogger(__name__)
# FIXME: add escaped grave
STR_GRAMMAR = r'''
script = block*
block = code / text
code = grave text grave
text = ~"[A-Z 0-9\s\!\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\"\_\{\|\}\~\\\\]+"i
grave = "`"
'''
GRAMMAR = Grammar(STR_GRAMMAR)
class Config:
comment_char = '#'
dump_lookups = True
dynamic_config_callable_name = 'jake_get_config'
namespace_template_separator = '~~~'
verbose = False
strip_extra_whitespace = True
lint = False
dump_debug = False
@contextlib.contextmanager
def redir_stdout():
old = sys.stdout
new = StringIO()
sys.stdout = new
yield new
sys.stdout = old
class Template(object):
def __init__(self, template_path, namespace=None):
self.namespace = namespace or Namespace()
template = open(template_path).read()
try:
self.namespace_str, self.template = template.split(
'\n{}\n'.format(Config.namespace_template_separator))
except ValueError:
self.template = template
self.namespace_str = ''
@classmethod
def render(cls, name):
template = cls(name)
template.execute_namespace()
return template.render_template()
def run(self):
self.execute_namespace()
self.execute_template()
def execute_namespace(self):
if self.namespace_str:
namespace = self.namespace
exec self.namespace_str in namespace.module_namespace
if namespace.config is None and Config.dynamic_config_callable_name in self.namespace.module_namespace:
config_callable = self.namespace.module_namespace[Config.dynamic_config_callable_name]
namespace.config = config_callable()
def create_script(self):
script = StringIO()
for is_code, text in self.transform():
if is_code:
# gross approximation for printing values when necessary
if len(text.split('\n')) == 1:
script.write('\nprint ({} or ""),\n'.format(text))
else:
script.write(text)
else:
script.write('\nprint """{}""",\n'.format(text))
return script.getvalue()
def execute_script(self, script):
if Config.dump_debug:
sys.stderr.write(script)
sys.stderr.flush()
exec script in self.namespace
def lint_script(self, script):
script = self.namespace_str + script
import subprocess
try:
# we ignore all E-class warnings because they're non-fatal,
# style-based things.
proc = subprocess.Popen(
['flake8', '--ignore', 'E', '-'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
except OSError as e:
print('# Could not lint source: {}\n'.format(str(e)))
return
stdout, stderr = proc.communicate(script)
if stdout:
for line in stdout.splitlines():
m = re.match(r".* undefined name '(.+)'", line)
if m:
name, = m.groups(0)
if name in self.namespace.lookups:
continue
else:
try:
self.namespace[name]
except KeyError:
self.namespace.lookups.pop(-1)
else:
self.namespace.lookups.pop(-1)
continue
name, line = line.split(':', 1)
print('# {}'.format(line))
print('')
def render_template(self):
self.namespace.freeze()
script = self.create_script()
with redir_stdout() as config:
if Config.lint:
self.lint_script(script)
self.execute_script(script)
output = config.getvalue()
pretty_output = self.dump_pretty_output(output)
return pretty_output
def dump_pretty_output(self, output):
pretty_output = StringIO()
# prefix all the dynamic values that were looked up as comments at the
# top of the file.
if Config.dump_lookups:
for name in sorted(self.namespace.lookups):
value = self.namespace[name]
if isinstance(value, types.FunctionType):
continue
pretty_output.write('%s %s = %s\n' % (Config.comment_char, name, value))
pretty_output.write('\n')
if Config.strip_extra_whitespace:
output = re.sub(r' \n', '\n', output)
output = re.sub(r' ', ' ', output)
output = re.sub(r'\n\n+', '\n\n', output)
pretty_output.write(output)
return pretty_output.getvalue()
def transform(self):
parsed = GRAMMAR.parse(self.template)
for block in parsed:
assert len(block.children) == 1
expr_name = block.children[0].expr_name
text = block.text
if expr_name == 'code':
yield True, text[1:-1]
else:
yield False, text
class Namespace(dict):
def __init__(self, config=None):
self.config = config
self.module_namespace = {}
self.local_namespace = {}
self._frozen = False
self.lookups = []
def freeze(self):
self._frozen = True
def __getitem__(self, key):
self.lookups.append(key)
try:
if isinstance(self.config, dict):
value = self.config[key]
else:
value = getattr(self.config, key)
except (AttributeError, KeyError) as e:
if self.config is not None:
log.debug("Could not retrieve %s from config object", key)
else:
return value
if key in self.module_namespace:
value = self.module_namespace[key]
elif key in self.local_namespace:
value = self.local_namespace[key]
else:
raise KeyError(key)
return value
def __setitem__(self, key, value):
if self._frozen:
self.module_namespace[key] = value
else:
self.local_namespace[key] = value
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('templates', metavar='template', nargs='+',
help='Paths to templates to generate.')
parser.add_argument('--no-dump-lookups', action='store_false',
default=True,
help="If given, don't dump variable lookups to beginning of config file as comments")
parser.add_argument('--verbose', action='store_true',
help="Debugging output. Does not affect template output")
parser.add_argument('--namespace-template-separator', type=str,
default=Config.namespace_template_separator,
help="Change separator between namespace and template separator.")
parser.add_argument('--comment-char', type=str,
default=Config.comment_char,
help="Change comment character to use for dumped variable name lookups.")
parser.add_argument('--dynamic-config-callable-name', type=str,
default=Config.dynamic_config_callable_name,
help="Change name of the callable to look for for retrieving dynamic configuration.")
parser.add_argument('--output-to-dir',
default=None,
help="Rather than write to stdout, write templates to given directory. Filenames will be that of the template")
parser.add_argument('--no-strip-extra-whitespace',
action='store_false',
default=Config.strip_extra_whitespace,
help="Don't remove trailing whitespace, extra newlines and double spaces.")
parser.add_argument('--lint',
action='store_true',
default=Config.lint,
help="Lint script before it is executed. Warnings will be inserted toward the top of the file")
parser.add_argument('--dump-debug',
action='store_true',
default=Config.dump_debug,
help="Dump script that is about to be executed.")
return parser.parse_args()
def main():
options = parse_args()
Config.comment_char = options.comment_char
Config.dynamic_config_callable_name = options.dynamic_config_callable_name
Config.dump_lookups = options.no_dump_lookups
Config.namespace_template_separator = options.namespace_template_separator
Config.verbose = options.verbose
Config.strip_extra_whitespace = options.no_strip_extra_whitespace
Config.lint = options.lint
Config.dump_debug = options.dump_debug
for filename in options.templates:
output = Template.render(filename)
if options.output_to_dir:
output_path = os.path.join(options.output_to_dir, os.path.basename(filename))
with open(output_path, 'w') as f:
f.write(output)
else:
sys.stdout.write(output)
if __name__ == '__main__':
main()
$ ./bin/jake examples/squid.conf
class jake_get_config:
direct_proxy_enabled = False
transparent_proxy_enabled = False
smp_enabled = True
cache_mem = 256000
disk_caches = []
ssl_bump_server_first = False
ssl_bump_enabled = False
import subprocess
def directive(command, *args, **kwargs):
text = '%s %s %s' % (command, ' '.join(map(str, args)), ' '.join('%s=%s' % kv for kv in kwargs.iteritems()))
print text
def http_port(port, *args, **kwargs):
args = set(args)
if kwargs.pop('ssl_bump', False):
args.add('ssl-bump')
kwargs['generate-host-certificates'] = 'on'
kwargs['cert'] = '/path/to/cert'
kwargs['key'] = '/path/to/key'
https = kwargs.pop('https', False)
return directive('http_port' if not https else 'https_port', port, *args, **kwargs)
def https_port(port, *args, **kwargs):
return http_port(port, https=True, *args, **kwargs)
def workers(enabled):
if enabled:
cpu_count = len(subprocess.check_output(['grep', 'processor', '/proc/cpuinfo']).splitlines())
return directive('workers', max(cpu_count/2, 1))
def cache_dir(config):
if not config.enabled:
return
args = []
if config.cache_type in ('ufs', 'aufs', 'diskd'):
args.append('/var/cache/ufs/%d' % config.pk)
args += ['%s' % (config.cache_size / 1024), '16', '256']
else:
args.append('/var/cache/rock/%d' % config.pk)
args.append('%s' % (config.cache_size / 1024))
if config.no_store:
args.append('no-store')
kwargs = {
'min-size': config.min_value_object_size,
'min-size': config.max_value_object_size,
}
return directive('cache_dir', config.cache_type, *args, **kwargs)
def ssl_bump(server_first, enabled):
if not enabled:
return
if server_first:
mode = 'server-first'
else:
mode = 'client-first'
return directive('ssl_bump', mode)
~~~
acl localnet src 10.0.0.0/8
acl localnet src 172.16.0.0/12
acl localnet src 192.168.0.0/16
acl localnet src fc00::/7
acl localnet src fe80::/10
acl SSL_ports port 443
acl Safe_ports port 80
acl Safe_ports port 443
acl CONNECT method CONNECT
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow localnet
http_access allow localhost
http_access allow localhost manager
http_access deny manager
# INSERT RULES HERE
http_access deny all
cache_effective_user proxy
coredump_dir /var/cache/squid
`
if direct_proxy_enabled:
http_port(direct_proxy_port, ssl_bump=direct_ssl)
if transparent_proxy_enabled:
http_port(65080, 'intercept')
if transparent_ssl:
https_port(65443, 'intercept', ssl_bump=transparent_ssl)
`
`workers(smp_enabled)`
cache_mem `cache_mem` bytes
`
for dir in disk_caches:
if smp_enabled and dir.cache_type != 'rock':
continue
cache_dir(dir)
`
url_rewriter_program /usr/libexec/url_rewriter
url_rewrite_access allow all
`ssl_bump(ssl_bump_server_first, enabled=ssl_bump_enabled)`
refresh_pattern ^ftp: 1440 20% 10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern . 0 20% 4320
# cache_mem = 256000
# direct_proxy_enabled = False
# disk_caches = []
# smp_enabled = True
# ssl_bump_enabled = False
# ssl_bump_server_first = False
# transparent_proxy_enabled = False
acl localnet src 10.0.0.0/8
acl localnet src 172.16.0.0/12
acl localnet src 192.168.0.0/16
acl localnet src fc00::/7
acl localnet src fe80::/10
acl SSL_ports port 443
acl Safe_ports port 80
acl Safe_ports port 443
acl CONNECT method CONNECT
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow localnet
http_access allow localhost
http_access allow localhost manager
http_access deny manager
# INSERT RULES HERE
http_access deny all
cache_effective_user proxy
coredump_dir /var/cache/squid
workers 4
cache_mem 256000 bytes
url_rewriter_program /usr/libexec/url_rewriter
url_rewrite_access allow all
refresh_pattern ^ftp: 1440 20% 10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern . 0 20% 4320
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment