Skip to content

Instantly share code, notes, and snippets.

@bdraco
Forked from voscausa/Jinja module loader.md
Created August 10, 2020 14:37
Show Gist options
  • Save bdraco/474b8a46f27c67c3a9124442c51fbe4b to your computer and use it in GitHub Desktop.
Save bdraco/474b8a46f27c67c3a9124442c51fbe4b to your computer and use it in GitHub Desktop.
Jinja2 compiled templates module loader for App Engine Pyhton 2.7.
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, with_statement
import logging
import zipfile
import StringIO
import marshal
import imp
import struct
import time
from jinjaenv import models, jinja2env
def code_gen(entity, env=None):
""" Jinja module code """
eid = entity.key.id()
logging.info('compiled : ' + eid)
name = eid.split('.')[0]
raw = env.compile(entity.content, name=name, filename=name + '.html', raw=True)
i = 0
py_code = ''
indentation = " " * 4
for line in raw.split("\n"):
if i < 2:
py_code += (line + "\n")
else:
if i == 2:
py_code += "__jinja_template__ = None\n"
py_code += "def run(environment, jinja_template):\n"
py_code += indentation + "global __jinja_template__\n"
py_code += indentation + "__jinja_template__ = jinja_template\n"
py_code += (indentation + line + "\n")
i += 1
py_code += indentation + "return (name, blocks, root, debug_info)\n"
return py_code
def compile_callback(entity, env):
""" NDB map async callback """
py_code = code_gen(entity, env)
code_object = compile(py_code.encode(), ('%s.%s' % (entity.key.parent().id(),
entity.key.id())).encode(), 'exec')
entity.blob = marshal.dumps(code_object)
return entity.put_async()
def compile_runtimes(site_key, env):
""" Compile runtimes HTML and TXT templates into Python code objects """
logging.info('compiling runtime html templates')
env = jinja2env.JinjaEnv.get_env(site_key.id())
query = models.Runtimes.query(ancestor=site_key)
future = query.map_async(lambda entity: compile_callback(entity, env))
future.get_result()
def zip_compiled(site_key, response):
""" Download zip Runtimes compiled Python files : .pyc """
updates = []
env = jinja2env.JinjaEnv.get_env(site_key.id())
output = StringIO.StringIO()
with zipfile.ZipFile(output, 'w') as z:
for each in models.RunTimes.query(ancestor=site_key)
member_name = site_key.id() + '.' + each.key.id().split('.')[0] + '.pyc'
# compile again, to make sure we have the latest version
code_object = compile(code_gen(each, env).encode(),
('%s.%s' % (site_key.id(), each.key.id())).encode(), 'exec')
each.blob = marshal.dumps(code_object)
updates.append(each)
timestamp = struct.pack(b'f', time.mktime(each.modified.timetuple()))
z.writestr(member_name.encode(), imp.get_magic() + timestamp + each.blob)
ndb.put_multi(updates)
response.headers[b'Content-Type'] = b'multipart/x-zip'
response.headers[b'Content-Disposition'] = str('attachment; filename=%s-compiled-templates.zip'
% site_key.id())
response.out.write(output.getvalue())

Jinja compiled templates module loader

This code is part of a Jinja CMS for Google App Engine Python 2.7 and NDB datastore

A Jinja enviroment is created for every CMS site: site_key_id = 'example

The modules are created using compiler.py The resulting code objects are stored in the dadastore using Kind Runtimes and a BlobProperty

The modules can also be saved / downloaded as .pyc in a zip archive: <site_key_id>-compiled-templates.zip If this zip archive is part of an app engine project, the module loader does not need the Runtimes blobs to run the modules.

The zip archive folder name is part of the env_variables in the app.yaml : JINJACMS_LOADPY

class Runtimes(ndb.Model):
    site = ndb.ComputedProperty(lambda self: self.key.parent().id())
    content = ndb.TextProperty(default='')                # html or txt template content
    blob = ndb.BlobProperty(default=None)                 # code object
    modified = ndb.DateTimeProperty(auto_now=True)
    created = ndb.DateTimeProperty(auto_now_add=True)
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import imp
import sys
import os
from google.appengine.ext import ndb
import zipfile
import marshal
import logging
from jinja2 import Environment
from jinja2.exceptions import TemplateNotFound
# Loading templates from Python modules from datastore or a zip archive in zip_dir
# Based on : https://groups.google.com/group/pocoo-libs/browse_thread/thread/748b0d2024f88f64
def init_env(site_key_id):
""" initialize Jinja environment : using the module loader
JINJACMS_LOADPY in app.yaml env_variables, value : compiled or NONE
zip archive with .pyc, name : <site_key_id>-compiled-templates.zip """
if 'JINJACMS_LOADPY' not in os.environ or os.environ['JINJACMS_LOADPY'].lower() in ['', 'none']:
path = None
logging.info('%s RUNTIME_MODULE loader initialized, using Python code objects' % site_key_id)
else:
path = os.path.join(os.path.dirname(sys.modules['appengine_config'].__file__),
os.environ['JINJACMS_LOADPY'], site_key_id + '-compiled-templates.zip')
logging.info('%s RUNTIME_MODULE loader initialized, using zip in %s' % (site_key_id, path))
loader = FileSystemModuleLoader(site_key_id, path)
return Environment(auto_reload=False, loader=loader)
class ModuleLoader(object):
"""Base mixin class for loaders that use pre-parsed Jinja2 templates stored as Python code. """
def get_module(self, environment, template):
raise TemplateNotFound(template)
def load(self, environment, filename, j_globals=None):
""" Loads a pre-compiled template, stored as Python code in a template module. """
if j_globals is None:
j_globals = {'environment': environment}
t = object.__new__(environment.template_class)
module = self.get_module(environment, filename)
name, blocks, root, debug_info = module.run(environment, t)
t.environment = environment
t.globals = j_globals
t.name = name
t.filename = filename
t.blocks = blocks
# render function and module
t.root_render_func = root
t._module = None
# debug and loader helpers
t._debug_info = debug_info
t._uptodate = lambda: True
return t
class FileSystemModuleLoader(ModuleLoader):
""" Load compiled Jinja templates from the datastore (code objects)
or .py from a zip archive for the CMS sites(site_keys) """
def __init__(self, site_key_id, path=None):
self.site_key_id = site_key_id
self.site_key = ndb.Key('CmsSites', site_key_id)
if path:
try:
self._zf = zipfile.ZipFile(os.path.join(path), 'r')
except IOError, e:
logging.warning('Zip archive not found, path : %s. Using Runtimes datastore.'
' Exception : %s' % (path, str(e)))
self._zf = None
else:
self._zf = None
# fake module : <site_key_id>
mod = imp.new_module(site_key_id)
mod.__loader__ = self
mod.__file__ = "[fake module %r]" % site_key_id
mod.__path__ = []
sys.modules[site_key_id] = mod
def get_module(self, environment, template):
""" Convert the template to a module name and load the code """
mod_name = '%s.%s' % (self.site_key_id, template.replace('.html', '')
.replace('.txt', '').replace('/', '.'))
if mod_name in sys.modules:
return sys.modules[mod_name]
logging.info('load Jinja template module : ' + mod_name)
try:
if self._zf:
# ignore first 8 bytes with magic and timestamp of .pyc
module_code = marshal.loads(self._zf.read(mod_name + '.pyc')[8:])
else:
module_code = marshal.loads(ndb.Key('Runtimes', template,
parent=self.site_key).get().blob)
module = imp.new_module(mod_name)
exec module_code in module.__dict__
sys.modules[mod_name] = module
return module
except (ImportError, AttributeError, KeyError):
logging.error('load failed : ' + mod_name)
raise TemplateNotFound(mod_name)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment