API endpoint templating using Jinja2 templates and Twisted.Web.

api_endpoint_templating.py

"""API endpoint templating using Jinja2.

Rendering of Jinja2 templates, with external actions, using Twisted.Web.

Combining external actions for Jinja2 [1] with Twisted's deferreds inside the
Twisted's web framework. Jinja2 templates are iteratively processed [2] to
render an API response [3]. The template prepares, executes and formats the
response from an HTTP request.

When run, the application looks up an email address provided as a GET paramater
on Have I Been Pwned, returning a text response with the names of datasets that
have leaked private information registered with the address. For example:

    http://localhost:8080/pwned?email=foo@foo.com

This example will return the text "Adobe Gawker Stratfor".

[1] https://gist.github.com/aliles/8417271
[2] https://gist.github.com/aliles/8454244
[3] https://gist.github.com/aliles/8471660
[4] https://haveibeenpwned.com/
"""
import os
import random
import string
import sys

from twisted.internet import defer, reactor, task, threads
from twisted.python import log
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
import jinja2
import jinja2.ext
import requests


class HTTPExtension(jinja2.ext.Extension):
    """HTTP method extension for Jinja2 templates.

        {% http METHOD URL into NAME as TYPE %}

    Defers control out of template to the calling scope for exection of HTTP
    request. A Mutable object is also passed out to allow external scope to
    pass result back into template context.
    """

    tags = set(['http'])

    mutable = type('Mutable', (object,), {'__slots__': ('value',)})

    def __init__(self, environment):
        super(HTTPExtension, self).__init__(environment)

    def parse(self, parser):
        proxy = "_" + "".join(random.choice(string.letters) for i in range(16))
        lineno = parser.stream.next().lineno
        method = parser.stream.expect('name').value
        url = parser.parse_expression()
        _ = parser.stream.expect('name:into')
        target = parser.stream.next().value
        _ = parser.stream.expect('name:as')
        content = parser.stream.expect('name').value
        return [
            # Creates new mutable proxy object inside template context.
            jinja2.nodes.Assign(
                jinja2.nodes.Name(proxy, 'store'),
                jinja2.nodes.Call(
                    self.attr('mutable', lineno=lineno),
                    [], [], None, None)).set_lineno(lineno),
            # Pass control out to calling scope.
            jinja2.nodes.CallBlock(
                self.call_method('action', [
                    jinja2.nodes.Name(proxy, 'load'),
                    jinja2.nodes.Const(method),
                    url,
                    jinja2.nodes.Const(content)
                ]), [], [], []).set_lineno(lineno),
            # Retrives value and assigns it to final result target name.
            jinja2.nodes.Assign(
                jinja2.nodes.Name(target, 'store'),
                jinja2.nodes.Getattr(
                    jinja2.nodes.Name(proxy, 'load'), 'value', 'load')
                ).set_lineno(lineno)
        ]

    def action(self, proxy, method, url, content, caller):
        """Defer HTTP request and processed to background thread"""
        def closure():
            request = getattr(requests, method)
            response = request(url)
            data = getattr(response, content)
            if callable(data):
                data = data()
            proxy.value = data
        deferred = threads.deferToThread(closure)
        return deferred



class Endpoint(Resource):
    """API endpoint web resource for Twisted web applications.

    Uses Jinja2 as a templating engine for API endpoints, iteratively
    processing the template.
    """

    isLeaf = True

    def __init__(self, source, environment):
        self.source = source
        self.environment = environment
        self.template = environment.from_string(source)

    def render_GET(self, request):
        """Process GET requests for endpoint resource.

        Iteratively processes the Jinja2 template, returning the templates
        output in chucnked responses to the client.
        """
        deferred = self._render(self.template, request)
        deferred.addCallback(lambda _: request.finish())
        deferred.addErrback(log.err)
        return NOT_DONE_YET

    def _iterate(self, iterator, request):
        """Consume next value from iterator

        If the next value is not a deferred, process the value with process() and
        return a new deferred for the next value.

        If the next value is a deferred add process() as a callback followed by an
        anonymous function for a new deferred to process the next value.
        """
        try:
            value = iterator.next()
            if not isinstance(value, defer.Deferred):
                request.write(value.encode('utf8'))
                return task.deferLater(reactor, 0, self._iterate, iterator, request)
            else:
                value.addCallback(lambda _: task.deferLater(reactor, 0, self._iterate, iterator, request))
                return value
        except StopIteration:
            return

    def _render(self, template, request):
        """Create new iterator for rendering template with context"""
        iterator = template.generate({'request': request})
        deferred = task.deferLater(reactor, 0, self._iterate, iter(iterator), request)
        return deferred


if __name__ == '__main__':
    # Example of Jinja2 template code.
    source = """{% set email = request.args.email[0] %}
{% set url = 'http://haveibeenpwned.com/api/breachedaccount/' + email|urlencode %}
{% http get url into result as json %}
{% for site in result %}
    {{ site }}
{% endfor %}
"""

    # Create Environment, with the HTTP extension, and load sample template
    # code.
    env = jinja2.Environment(trim_blocks=True, lstrip_blocks=True,
            extensions=[HTTPExtension])

    # Construct resource structure for Twisted web application.
    root = Resource()
    root.putChild("pwned", Endpoint(source, env))
    factory = Site(root)

    # Listen on port 8080 for HTTP requests and start the Twisted reactor.
    reactor.listenTCP(8080, factory)
    reactor.run()