Recursive Jinja Templates (based on Ansible)

jinja-recursive-templates.py

#!/usr/bin/env python3

# Stolen from Ansible, thus licensed under GPLv3+.
# https://github.com/ansible/ansible/blob/13c28664ae0817068386b893858f4f6daa702052/lib/ansible/template/__init__.py#L661
# https://github.com/ansible/ansible/blob/13c28664ae0817068386b893858f4f6daa702052/lib/ansible/template/vars.py#L33

from collections.abc import Mapping, Sequence
from hashlib import sha1
from jinja2.nativetypes import NativeEnvironment
from jinja2.runtime import Macro

import re
import logging

import jinja2.nativetypes

from itertools import chain
from itertools import islice

logger = logging.getLogger(__name__)

# https://github.com/pallets/jinja/blob/35c69e9ec3597a88977ddd27628ecd9318e04737/src/jinja2/nativetypes.py#L14-L38
def native_concat(values):
    """Return a native Python type from the list of compiled nodes. If
    the result is a single node, its value is returned. Otherwise, the
    nodes are concatenated as strings.
    :param values: Iterable of outputs to concatenate.
    """
    head = list(islice(values, 2))

    if not head:
        return None

    if len(head) == 1:
        raw = head[0]
        if not isinstance(raw, str):
            return raw
    else:
        raw = "".join([str(v) for v in chain(head, values)])

    return raw

# monkey-patch because in jinja 2.10.x it's used in NativeCodeGenerator
# https://github.com/pallets/jinja/blob/6cc49a789afb025b750fafb13727092c9324e5ff/jinja2/nativetypes.py#L124
jinja2.nativetypes.native_concat = native_concat

class Templar:

    def __init__(self, data, environment=None, env_globals={}):

        logger.debug("Templar data %r" % data)

        self._data = data
        self._cache = {}
        self._environment = environment or NativeEnvironment()

        self._environment.globals["dict"] = dict

        for key, value in env_globals.items():
            self._environment.globals[key] = value

        self.MARKER = ("{{", "}}")
        self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % self.MARKER)

    def template(self, variable):

        logger.debug("template %s" % variable)

        sha1_hash = sha1(str(variable).encode("utf-8")).hexdigest()

        if sha1_hash in self._cache:
            logger.debug("cached %s %s" % (variable, sha1_hash))
            return self._cache[sha1_hash]
        else:
            logger.debug("not cached %s %s" % (variable, sha1_hash))

        if isinstance(variable, str):

            for marker in self.MARKER:
                if not marker in variable:
                    logger.debug("not template %s" % variable)
                    return variable

            t = self._environment.from_string(variable)
            ctx = t.new_context(CustomVars(self, self._data, t.globals), shared=True)
            rf = t.root_render_func(ctx)

            ret = native_concat(rf)

        elif isinstance(variable, Sequence):
            ret = [self.template(v) for v in variable]
        elif isinstance(variable, Mapping):
            d = {}
            for k in variable.keys():
                d[k] = self.template(variable[k])
            ret = d
        else:
            if not isinstance(variable, (int, type(None), Macro)):
                logger.warn("fall-through %s!" % variable)
            ret = variable

        logger.debug("%s rendered to %s (%s)" %(variable, ret, type(ret)))

        self._cache[sha1_hash] = ret

        return ret

class CustomVars(Mapping):

    def __init__(self, templar, data, globals):
        self._data = data
        self._globals = globals
        self._templar = templar

    def __contains__(self, k):
        logger.debug("check %s" % k)
        return k in self._data or k in self._globals

    def __iter__(self):
        keys = set()
        keys.update(self._data, self._globals)
        return iter(keys)

    def __len__(self):
        keys = set()
        keys.update(self._data, self._globals)
        return len(keys)

    def __getitem__(self, varname):

        logger.debug("get %s" % varname)

        if varname in self._globals:
            variable = self._globals[varname]
        else:
            variable = self._data[varname]

        logger.debug("%s = %s" % (varname, variable))

        return self._templar.template(variable)

z_output.txt

$ python3 jinja-recursive-templates.py
template {{ foo }}
not cached {{ foo }} d38b23d97fb4433a81e59db55f41dd73196d39d5 {}
check foo
get foo
foo = {'baz': '{{ bar }}', 'trulla': ['{{ bar }}']}
template {'baz': '{{ bar }}', 'trulla': ['{{ bar }}']}
not cached {'baz': '{{ bar }}', 'trulla': ['{{ bar }}']} 25178f51986b77b3decc081f38473c24a1208801 {}
template {{ bar }}
not cached {{ bar }} 4b5ad647900164d59f47a12c9a10f476c486b423 {}
check bar
get bar
bar = 42
template 42
not cached 42 92cfceb39d57d914ed8b14d0e37643de0797ae56 {}
not template 42
{{ bar }} rendered to 42
template ['{{ bar }}']
not cached ['{{ bar }}'] b148e286a698f487859a326965dab36fd88eddb6 {'4b5ad647900164d59f47a12c9a10f476c486b423': '42'}
template {{ bar }}
cached {{ bar }} 4b5ad647900164d59f47a12c9a10f476c486b423
['{{ bar }}'] rendered to ['42']
{'baz': '{{ bar }}', 'trulla': ['{{ bar }}']} rendered to {'baz': '42', 'trulla': ['42']}
{{ foo }} rendered to {'baz': '42', 'trulla': ['42']}
template result: {'baz': '42', 'trulla': ['42']}