jinja-unicode.py

class EnsureUnicodeExtension(Extension):
    """Ensures all strings in Jinja are unicode"""

    @staticmethod
    def ensure_unicode(s):
        """Converts a bytestring to unicode. Must be registered as a filter!"""
        if isinstance(s, str):
            return s.decode('utf-8')
        return s

    def filter_stream(self, stream):
        # The token stream looks like this:
        # ------------------------
        # variable_begin {{
        # name           event
        # dot            .
        # name           getTitle
        # lparen         (
        # rparen         )
        # pipe           |
        # name           safe
        # variable_end   }}
        # ------------------------
        # Intercepting the end of the actual variable is hard but it's rather easy to get the end of
        # the variable tag or the start of the first filter. As filters are optional we need to check
        # both cases. If we inject the code before the first filter we *probably* don't need to run
        # it again later assuming our filters are nice and only return unicode. If that's not the
        # case we can simply remove the `variable_done` checks.

        variable_done = False
        for token in stream:
            if token.type == 'pipe':
                # Inject our filter call before the first filter
                yield Token(token.lineno, 'pipe', '|')
                yield Token(token.lineno, 'name', 'ensure_unicode')
                variable_done = True
            elif token.type == 'variable_end':
                if not variable_done:
                    # Inject our filter call if we haven't injected it right after the variable
                    yield Token(token.lineno, 'pipe', '|')
                    yield Token(token.lineno, 'name', 'ensure_unicode')
                variable_done = False
            yield token