|
#!/usr/bin/env python2 |
|
|
|
import argparse |
|
import contextlib |
|
import logging |
|
import re |
|
import os |
|
import sys |
|
import types |
|
|
|
from StringIO import StringIO |
|
|
|
from parsimonious.grammar import Grammar |
|
|
|
|
|
log = logging.getLogger(__name__) |
|
|
|
# FIXME: add escaped grave |
|
|
|
STR_GRAMMAR = r''' |
|
script = block* |
|
block = code / text |
|
code = grave text grave |
|
text = ~"[A-Z 0-9\s\!\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\"\_\{\|\}\~\\\\]+"i |
|
grave = "`" |
|
''' |
|
|
|
GRAMMAR = Grammar(STR_GRAMMAR) |
|
|
|
|
|
class Config: |
|
comment_char = '#' |
|
dump_lookups = True |
|
dynamic_config_callable_name = 'jake_get_config' |
|
namespace_template_separator = '~~~' |
|
verbose = False |
|
strip_extra_whitespace = True |
|
lint = False |
|
dump_debug = False |
|
|
|
|
|
@contextlib.contextmanager |
|
def redir_stdout(): |
|
old = sys.stdout |
|
new = StringIO() |
|
sys.stdout = new |
|
|
|
yield new |
|
|
|
sys.stdout = old |
|
|
|
|
|
class Template(object): |
|
def __init__(self, template_path, namespace=None): |
|
self.namespace = namespace or Namespace() |
|
|
|
template = open(template_path).read() |
|
|
|
try: |
|
self.namespace_str, self.template = template.split( |
|
'\n{}\n'.format(Config.namespace_template_separator)) |
|
except ValueError: |
|
self.template = template |
|
self.namespace_str = '' |
|
|
|
@classmethod |
|
def render(cls, name): |
|
template = cls(name) |
|
template.execute_namespace() |
|
return template.render_template() |
|
|
|
def run(self): |
|
self.execute_namespace() |
|
self.execute_template() |
|
|
|
def execute_namespace(self): |
|
if self.namespace_str: |
|
namespace = self.namespace |
|
|
|
exec self.namespace_str in namespace.module_namespace |
|
|
|
if namespace.config is None and Config.dynamic_config_callable_name in self.namespace.module_namespace: |
|
config_callable = self.namespace.module_namespace[Config.dynamic_config_callable_name] |
|
namespace.config = config_callable() |
|
|
|
def create_script(self): |
|
script = StringIO() |
|
|
|
for is_code, text in self.transform(): |
|
if is_code: |
|
# gross approximation for printing values when necessary |
|
if len(text.split('\n')) == 1: |
|
script.write('\nprint ({} or ""),\n'.format(text)) |
|
else: |
|
script.write(text) |
|
else: |
|
script.write('\nprint """{}""",\n'.format(text)) |
|
|
|
return script.getvalue() |
|
|
|
def execute_script(self, script): |
|
if Config.dump_debug: |
|
sys.stderr.write(script) |
|
sys.stderr.flush() |
|
exec script in self.namespace |
|
|
|
def lint_script(self, script): |
|
script = self.namespace_str + script |
|
import subprocess |
|
try: |
|
# we ignore all E-class warnings because they're non-fatal, |
|
# style-based things. |
|
proc = subprocess.Popen( |
|
['flake8', '--ignore', 'E', '-'], |
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, |
|
stderr=subprocess.STDOUT |
|
) |
|
except OSError as e: |
|
print('# Could not lint source: {}\n'.format(str(e))) |
|
return |
|
|
|
stdout, stderr = proc.communicate(script) |
|
|
|
if stdout: |
|
for line in stdout.splitlines(): |
|
m = re.match(r".* undefined name '(.+)'", line) |
|
if m: |
|
name, = m.groups(0) |
|
if name in self.namespace.lookups: |
|
continue |
|
else: |
|
try: |
|
self.namespace[name] |
|
except KeyError: |
|
self.namespace.lookups.pop(-1) |
|
else: |
|
self.namespace.lookups.pop(-1) |
|
continue |
|
|
|
name, line = line.split(':', 1) |
|
print('# {}'.format(line)) |
|
print('') |
|
|
|
def render_template(self): |
|
self.namespace.freeze() |
|
|
|
script = self.create_script() |
|
with redir_stdout() as config: |
|
if Config.lint: |
|
self.lint_script(script) |
|
self.execute_script(script) |
|
output = config.getvalue() |
|
pretty_output = self.dump_pretty_output(output) |
|
|
|
return pretty_output |
|
|
|
def dump_pretty_output(self, output): |
|
pretty_output = StringIO() |
|
|
|
# prefix all the dynamic values that were looked up as comments at the |
|
# top of the file. |
|
if Config.dump_lookups: |
|
for name in sorted(self.namespace.lookups): |
|
value = self.namespace[name] |
|
|
|
if isinstance(value, types.FunctionType): |
|
continue |
|
|
|
pretty_output.write('%s %s = %s\n' % (Config.comment_char, name, value)) |
|
pretty_output.write('\n') |
|
|
|
if Config.strip_extra_whitespace: |
|
output = re.sub(r' \n', '\n', output) |
|
output = re.sub(r' ', ' ', output) |
|
output = re.sub(r'\n\n+', '\n\n', output) |
|
|
|
pretty_output.write(output) |
|
|
|
return pretty_output.getvalue() |
|
|
|
def transform(self): |
|
parsed = GRAMMAR.parse(self.template) |
|
|
|
for block in parsed: |
|
assert len(block.children) == 1 |
|
|
|
expr_name = block.children[0].expr_name |
|
|
|
text = block.text |
|
if expr_name == 'code': |
|
yield True, text[1:-1] |
|
else: |
|
yield False, text |
|
|
|
|
|
class Namespace(dict): |
|
def __init__(self, config=None): |
|
self.config = config |
|
self.module_namespace = {} |
|
self.local_namespace = {} |
|
|
|
self._frozen = False |
|
self.lookups = [] |
|
|
|
def freeze(self): |
|
self._frozen = True |
|
|
|
def __getitem__(self, key): |
|
self.lookups.append(key) |
|
|
|
try: |
|
if isinstance(self.config, dict): |
|
value = self.config[key] |
|
else: |
|
value = getattr(self.config, key) |
|
except (AttributeError, KeyError) as e: |
|
if self.config is not None: |
|
log.debug("Could not retrieve %s from config object", key) |
|
else: |
|
return value |
|
|
|
if key in self.module_namespace: |
|
value = self.module_namespace[key] |
|
elif key in self.local_namespace: |
|
value = self.local_namespace[key] |
|
else: |
|
raise KeyError(key) |
|
return value |
|
|
|
def __setitem__(self, key, value): |
|
if self._frozen: |
|
self.module_namespace[key] = value |
|
else: |
|
self.local_namespace[key] = value |
|
|
|
|
|
def parse_args(): |
|
parser = argparse.ArgumentParser() |
|
parser.add_argument('templates', metavar='template', nargs='+', |
|
help='Paths to templates to generate.') |
|
parser.add_argument('--no-dump-lookups', action='store_false', |
|
default=True, |
|
help="If given, don't dump variable lookups to beginning of config file as comments") |
|
parser.add_argument('--verbose', action='store_true', |
|
help="Debugging output. Does not affect template output") |
|
parser.add_argument('--namespace-template-separator', type=str, |
|
default=Config.namespace_template_separator, |
|
help="Change separator between namespace and template separator.") |
|
parser.add_argument('--comment-char', type=str, |
|
default=Config.comment_char, |
|
help="Change comment character to use for dumped variable name lookups.") |
|
parser.add_argument('--dynamic-config-callable-name', type=str, |
|
default=Config.dynamic_config_callable_name, |
|
help="Change name of the callable to look for for retrieving dynamic configuration.") |
|
parser.add_argument('--output-to-dir', |
|
default=None, |
|
help="Rather than write to stdout, write templates to given directory. Filenames will be that of the template") |
|
parser.add_argument('--no-strip-extra-whitespace', |
|
action='store_false', |
|
default=Config.strip_extra_whitespace, |
|
help="Don't remove trailing whitespace, extra newlines and double spaces.") |
|
parser.add_argument('--lint', |
|
action='store_true', |
|
default=Config.lint, |
|
help="Lint script before it is executed. Warnings will be inserted toward the top of the file") |
|
parser.add_argument('--dump-debug', |
|
action='store_true', |
|
default=Config.dump_debug, |
|
help="Dump script that is about to be executed.") |
|
return parser.parse_args() |
|
|
|
|
|
def main(): |
|
options = parse_args() |
|
|
|
Config.comment_char = options.comment_char |
|
Config.dynamic_config_callable_name = options.dynamic_config_callable_name |
|
Config.dump_lookups = options.no_dump_lookups |
|
Config.namespace_template_separator = options.namespace_template_separator |
|
Config.verbose = options.verbose |
|
Config.strip_extra_whitespace = options.no_strip_extra_whitespace |
|
Config.lint = options.lint |
|
Config.dump_debug = options.dump_debug |
|
|
|
for filename in options.templates: |
|
output = Template.render(filename) |
|
if options.output_to_dir: |
|
output_path = os.path.join(options.output_to_dir, os.path.basename(filename)) |
|
|
|
with open(output_path, 'w') as f: |
|
f.write(output) |
|
else: |
|
sys.stdout.write(output) |
|
|
|
if __name__ == '__main__': |
|
main() |