Iterative rendering of Jinja2 template, with external actions, using Twisted.

twisted_render_jinja2.py

"""Rendering of Jinja2 templates, with external actions, using Twisted.

Combines external actions for Jinja2 [1] with Twisted's deferreds for
cooperative iterator consumption [2] to render template. The template prepares,
executes and formats the response from an HTTP request.

[1] https://gist.github.com/aliles/8417271
[2] https://gist.github.com/aliles/8454244
"""
import os
import random
import string
import sys

from twisted.internet import defer, reactor, task, threads
from twisted.python import log
import jinja2
import jinja2.ext
import requests


class HTTPExtension(jinja2.ext.Extension):
    """HTTP method extension for Jinja2 templates.

        {% http METHOD URL into NAME as TYPE %}

    Defers control out of template to the calling scope for exection of HTTP
    request. A Mutable object is also passed out to allow external scope to
    pass result back into template context.
    """

    tags = set(['http'])

    mutable = type('Mutable', (object,), {'__slots__': ('value',)})

    def __init__(self, environment):
        super(HTTPExtension, self).__init__(environment)

    def parse(self, parser):
        proxy = "_" + "".join(random.choice(string.letters) for i in range(16))
        lineno = parser.stream.next().lineno
        method = parser.stream.expect('name').value
        url = parser.parse_expression()
        _ = parser.stream.expect('name:into')
        target = parser.stream.next().value
        _ = parser.stream.expect('name:as')
        content = parser.stream.expect('name').value
        return [
            # Creates new mutable proxy object inside template context.
            jinja2.nodes.Assign(
                jinja2.nodes.Name(proxy, 'store'),
                jinja2.nodes.Call(
                    self.attr('mutable', lineno=lineno),
                    [], [], None, None)).set_lineno(lineno),
            # Pass control out to calling scope.
            jinja2.nodes.CallBlock(
                self.call_method('action', [
                    jinja2.nodes.Name(proxy, 'load'),
                    jinja2.nodes.Const(method),
                    url,
                    jinja2.nodes.Const(content)
                ]), [], [], []).set_lineno(lineno),
            # Retrives value and assigns it to final result target name.
            jinja2.nodes.Assign(
                jinja2.nodes.Name(target, 'store'),
                jinja2.nodes.Getattr(
                    jinja2.nodes.Name(proxy, 'load'), 'value', 'load')
                ).set_lineno(lineno)
        ]

    def action(self, proxy, method, url, content, caller):
        """Defer HTTP request and processed to background thread"""
        def closure():
            request = getattr(requests, method)
            response = request(url)
            data = getattr(response, content)
            if callable(data):
                data = data()
            proxy.value = data
        deferred = threads.deferToThread(closure)
        return deferred


def iterate(iterator):
    """Consume next value from iterator

    If the next value is not a deferred, process the value with process() and
    return a new deferred for the next value.

    If the next value is a deferred add process() as a callback followed by an
    anonymous function for a new deferred to process the next value.
    """
    try:
        value = iterator.next()
        if not isinstance(value, defer.Deferred):
            process(value)
            return task.deferLater(reactor, 0, iterate, iterator)
        else:
            value.addCallback(lambda _: task.deferLater(reactor, 0, iterate, iterator))
            return value
    except StopIteration:
        return


def process(result):
    """Process template output"""
    sys.stdout.write(result)


def render_deferred(template, context=None):
    """Create new iterator for rendering template with context"""
    iterator = template.generate({} if context is None else context)
    deferred = task.deferLater(reactor, 0, iterate, iter(iterator))
    return deferred


if __name__ == '__main__':
    # Example of Jinja2 template code.
    SOURCE = """{% set email = 'foo@foo.com' %}
{% set url = 'http://haveibeenpwned.com/api/breachedaccount/' + email|urlencode %}
{% http get url into result as json %}
{% for site in result %}
    {{ site }}
{% endfor %}
"""

    # Create Environment, with the HTTP extension, and load sample template
    # code.
    env = jinja2.Environment(trim_blocks=True, lstrip_blocks=True,
            extensions=[HTTPExtension])
    tmpl = env.from_string(SOURCE)

    # Render template using deferred, stopping the reactor when template
    # rendering complete.
    deferred = render_deferred(tmpl)
    deferred.addErrback(log.err)
    deferred.addCallback(lambda result: reactor.stop())
    reactor.run()