scared.py

import ast
import re

split = re.compile(r'(({{|{%)(.*?)(%}|}}))').split

class Visit:
    def __init__(self, funcname):
        self.names = []
        self.funcname = funcname

    def _yield(self, node):
        return ast.Expr(ast.Yield(node))

    def _string(self, node):
        return ast.Call(
            func=ast.Name('str', ctx=ast.Load()),
            args=[node],
            keywords=[],
        )

    def visit(self, node):
        if not isinstance(node, Node):
            raise TypeError()
        node_name = node.__class__.__name__
        method = getattr(self, f'visit_{node_name}', self.visit_generic)
        return method(node)

    def visit_generic(self, node):
        raise ValueError(f"Unknown node type {node.__class__.__name__!r}")

    def visit_Template(self, node):
        _inner_body = [self.visit(x) for x in node.value]
        body = [
            ast.FunctionDef(
                name='_inner',
                args=ast.arguments(
                    args=[],
                    vararg=None,
                    kwonlyargs=[],
                    kw_defaults=[],
                    kwarg=None,
                    defaults=[]
                ),
                body=_inner_body,
                decorator_list=[],
                returns=None
            ),
            ast.Return(
                value=ast.Call(
                    func=ast.Attribute(
                        value=ast.Str(s=''),
                        attr='join',
                        ctx=ast.Load()
                    ),
                    args=[
                        ast.Call(
                            func=ast.Name(id='_inner', ctx=ast.Load()),
                            args=[],
                            keywords=[]
                        )
                    ],
                    keywords=[]
                )
            )
        ]
        tree = ast.Interactive(body=[
            ast.FunctionDef(
                name=self.funcname,
                args=ast.arguments(
                    args=[ast.arg(arg=n, annotation=None) for n in self.names],
                    vararg=None,
                    kwonlyargs=[],
                    kw_defaults=[],
                    kwarg=None,
                    defaults=[]
                ),
                body=body,
                decorator_list=[],
                returns=None
            )]
        )
        return ast.fix_missing_locations(tree)

    def visit_String(self, node):
        return self._yield(ast.Str(node.value))

    def visit_Name(self, node):
        self.names.append(node.value)
        return self._yield(self._string(ast.Name(id=node.value, ctx=ast.Load())))

    def visit_For(self, node):
        self.names.append(node.name)
        iter_ = ast.Name(id=node.name, ctx=ast.Load())
        target = ast.Name(id=node.target, ctx=ast.Store())
        body = list(self.visit(x) for x in node.value)
        if node.target in self.names:
            del self.names[self.names.index(node.target)]
        orelse = []
        return ast.For(target=target, iter=iter_, body=body, orelse=orelse)


class Node:
    def __init__(self, value):
        self.value = value

    def __repr__(self):
        return f"{self.__class__.__name__}({repr(self.value)})"

class Template(Node):
    pass

class String(Node):
    pass

class Name(Node):
    pass

class For(Node):
    def __init__(self, value, *, name, target):
        self.value = value
        self.name = name
        self.target = target

    def __repr__(self):
        return f"{self.__class__.__name__}({self.value!r}, name={self.name!r}, target={self.target!r})"

def tokenize(instring):
    toks = iter(split(instring))
    while True:
        try:
            tok = next(toks)
        except StopIteration:
            break
        if not tok.startswith('{{') and not tok.startswith('{%'):
            yield ('text', tok)
            continue
        open_brace = next(toks)
        exp = next(toks).strip()
        next(toks)
        if open_brace == '{{':
            yield ('name', exp)
            continue
        yield ('statement', exp)

def _parse(stream, end=None):
    for token in stream:
        if token[0] == 'statement' and token[1] == end:
            break
        if token[0] == 'text':
            yield String(token[1])
            continue
        if token[0] == 'name':
            yield Name(token[1])
            continue
        kind = token[1].partition(' ')[0]
        endkind = f'end{kind}'
        if kind == 'for':
            match = re.match(r"^for\s+(.*?)\s+in\s+(.*?)$", token[1])
            if not match:
                raise ValueError(f"Parse Error: {token[1]!r}")
            target, name = match.groups()
            name = name
            yield For(
                list(_parse(stream, end=endkind)),
                name=name,
                target=target
            )

def parse(stream):
    return Template(list(_parse(stream)))

def build(source, funcname):
    namespace = {}
    visitor = Visit(funcname)
    tokens = tokenize(source)
    parse_tree = parse(tokens)
    ast_tree = visitor.visit(parse_tree)
    exec(compile(ast_tree, '<template>', 'single'), {}, namespace)
    return namespace.get(funcname)


if __name__ == '__main__':
    template = """\
this is a test of the template tokenizer.

{{ hello }}, World!

<ul>
{% for foo in foos %}<li>{{ foo }}</li>
{% endfor %}</ul>
"""

    func = build(template, 'func')
    print(func('world', [1, 2, 3]))