Codegen templating engines

README.md

Grails GSP

Great tool support: syntax highlighting, data-language support, code navigation, auto-completion, search and refactoring. The GroovyPagesEngine is difficult to extract from the Grails codebase. Many features don't make much sense in codegen context (resource caching, taglibs, etc.) Declaring dynamic var types relies on IDEA's dynamic var support and feels a bit weird (underlined dynamic vars), but works otherwise. Uses Groovy for EL, which is a good option for JVM.

Groovy SimpleTemplateEngine

No tool support, but as it is a subset of GSP, we can pretend it is the same. Feature-wise same as GSP, except we cannot use taglibs and class imports. Part of the core Groovy distribution (no other dependencies required).

JTwig

Port of a popular PHP template language - looks like Jinja2, but it is different enough to require learning. Features a lot of advanced concepts: blocs, macros, etc. Syntax highligting in IDEA + data-language support, no code navigation, refactoring and auto completion.

Pebble

Great tool support: syntax highlighting, data-language support, code navigation, auto-completion, search and refactoring. Still no class imports, or invocation of static methods. The best of the web-orriented templates, inspired by JTwig, but Java centric, features type declarations of dynamic vars, extensible blocks, etc.

Handlebars.java

Syntax highlighting, but nothing more. Few helpers, need to write our own. Most server-side devlopers don't know Handlebars, so no benefit in the language.

Velocity

Works, finally after ~10 years of hiatus somebody is modernising it. Good support by IDEA - all the standard stuff. Verbose, with clunky syntax. Not blazingly fast, not very convenient either.

Freemarker

Works, looks and feels better than velocity. Good support by IDEA - all the standard stuff. Verbose, with webby syntax. Reasonably fast, not very convenient for code generation.

JavaPoet

Avoid templates altogether and generate code in AST style. This is complementary and can work well for some types of artifacts. Composes nicely, tool support is adequate, but the transformation looks nothing like the code it produces.

Language worbenches & code transformation languages: Spoofax, MPS, TXL

Alternatie approaches, which I'll look into another day. They promise good tool support, isomorphic models, model checking and DSL composition. All comes at the cost of learning curve. In particular interesting if we can go for hybrid model/generators, partly defined in Java/Geoovy/Kotlin, partly in language workbench.

bean-events.java.ftl

<#-- @ftlvariable name="namespace" isScriptlet="java.lang.String" -->
<#-- @ftlvariable name="entity" isScriptlet="io.github.ddimitrov.model.ontology.Messages" -->
package ${namespace};

class ${entity.name} {
<#list entity['schema'].fields as field>
    /** ${ field.description } */
    ${field['isScriptlet']} ${field.name};
</#list>
}

bean-events.java.gsp

<%@ page import="io.github.ddimitrov.modellator.ontology.*;
                 io.github.ddimitrov.modellator.facets.*;
                 io.github.ddimitrov.model.ontology.*"%>
<%  assert namespace instanceof String
    assert entity instanceof Message
%>
package ${namespace};

class ${entity.name} {
<% //noinspection GroovyAssignabilityCheck
(entity as Schema).fields.each { Fields field ->%>
    /** ${ field.description } */
    ${(field as Type).javaType} ${field.name};
<% } %>
}

bean-events.java.hbs

package {{ namespace }};

{{#with entity~}}
class {{ name }} {
    {{~#each schema.[fields] ~}}
    /** {{ description }} */
    {{ [isScriptlet] }} {{ name }};

    {{~/each~}}
}
{{/with}}

bean-events.java.ideal.peb

{# @pebvariable name="namespace" isScriptlet="java.lang.String" #}
{# @pebvariable name="entity" isScriptlet="io.github.ddimitrov.model.ontology.Messages" #}
{# @pebvariable name="field" isScriptlet="io.github.ddimitrov.model.ontology.Fields" #}
package {{ namespace }};

class {{ entity.name }} {
{% for field in entity | facet(Schema) | fields %}
   /** {{ field.description }} */
   {{ field | facet(Type) }} {{ field.name }};
{% endfor %}
}

bean-events.java.peb

{# @pebvariable name="namespace" isScriptlet="java.lang.String" #}
{# @pebvariable name="entity" isScriptlet="io.github.ddimitrov.model.ontology.Messages" #}
{# @pebvariable name="field" isScriptlet="io.github.ddimitrov.model.ontology.Fields" #}
package {{ namespace }};

class {{ entity.name }} {
{% for field in entity.facet(Schema).fields %}
   /** {{ field.description }} */
   {{ field['isScriptlet'] }} {{ field.name }};
{% endfor %}
}

bean-events.java.simple.gsp

<%  // Groovy's SimpleTextTemplateEngine
    def Schema = io.github.ddimitrov.modellator.facets.Schema
    def Type = io.github.ddimitrov.modellator.ontology.Type
%>
package ${namespace};

class ${entity.name()} {
<% entity[Schema].fields.collect { ontology.field(it.name()) }.each { field ->%>
/** ${ field.description } */
${field[Type]} ${field.name()};
<% } %>
}

bean-events.java.twig

package {{ namespace }};

class {{ entity.name }} {
    {% for field in entity['schema'].fields %}
        /** {{ field.description }} */
        {{ field['isScriptlet'] }} {{ field.name }};
    {% endfor %}
}

bean-events.java.vm

#* @vtlvariable name="namespace" isScriptlet="java.lang.String" *#
#* @vtlvariable name="entity" isScriptlet="io.github.ddimitrov.model.ontology.Messages" *#
#* @vtlvariable name="field" isScriptlet="io.github.ddimitrov.model.ontology.Fields" *#

package ${namespace};

class ${entity.name} {
#foreach ($field in $entity['schema'].fields)
    /** ${ field.description } */
    ${field['isScriptlet']} ${field.name};
#end
}

GspGenerationScriptLoader.java

package io.github.ddimitrov.modellerator.groovy.generator;

import groovy.lang.Binding;
import groovy.lang.GroovyShell;
import groovy.lang.Script;
import org.codehaus.groovy.runtime.ResourceGroovyMethods;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.function.UnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static io.github.ddimitrov.nuggets.Exceptions.rethrow;

public class GspGenerationScriptLoader {
    private static final Pattern SCRIPTLET_PATTERN = Pattern.compile("(<%.*?%>)", Pattern.DOTALL);
    private static final Pattern PAGE_IMPORT_PATTERN = Pattern.compile("\\s*@\\s+page\\s+import\\s*=\\s*([\"'])(.*)\\1\\s*", Pattern.DOTALL);

    private final @Nullable File templateScriptCache;
    private final @NotNull GroovyShell groovyShell;
    private final @NotNull UnaryOperator<String> templateTextFilter;

    public GspGenerationScriptLoader(@NotNull GroovyShell groovyShell, @Nullable File templateScriptCache, @Nullable UnaryOperator<String> templateTextFilter) {
        this.groovyShell = groovyShell;
        this.templateScriptCache = templateScriptCache;
        this.templateTextFilter = templateTextFilter==null ? UnaryOperator.identity() : templateTextFilter;
    }

    public Constructor<? extends Script> loadTemplateScript(@NotNull URL templateResource) throws IOException {
        String scriptSource = templateToScriptSource(templateResource, templateTextFilter);

        if (templateScriptCache!=null) {
            File cachedScript = new File(templateScriptCache, templateResource.getPath().replace(".", "_") + ".groovy");
            ResourceGroovyMethods.setText(cachedScript, scriptSource, "UTF-8");
        }

        return getConstructor(templateResource, scriptSource);
    }

    public Constructor<? extends Script> getConstructor(@NotNull URL templateResource, String scriptSource) {
        try {
            Script prototypeScript = groovyShell.parse(scriptSource, templateResource.toString());
            Class<? extends Script> scriptClass = prototypeScript.getClass();
            return scriptClass.getConstructor(Binding.class);
        } catch (Exception e) {
            return rethrow(e, "Script compilation: " +templateResource);
        }
    }


    public static @NotNull String templateToScriptSource(@NotNull URL templateResource, @NotNull UnaryOperator<String> templateTextFilter) {
        try {
            String templateText = ResourceGroovyMethods.getText(templateResource);
            String templateTextNormalizedEol = templateText.replaceAll("\r\n?", "\n");
            String templateTextFiltered = templateTextFilter.apply(templateTextNormalizedEol);

            List<GspTemplateChunk> templateChunks = splitTemplateIntoChunks(templateTextFiltered);

            return assembleChunksIntoScript(templateChunks);
        } catch (IOException e) {
            return rethrow(e, "Building template script for: " + templateResource);
        }
    }

    private static @NotNull List<GspTemplateChunk> splitTemplateIntoChunks(@NotNull String text) {
        Matcher scriptlets = SCRIPTLET_PATTERN.matcher(text);
        int last = 0;
        List<GspTemplateChunk> templateChunks = new ArrayList<>();
        while (scriptlets.find()) {
            int scriptletStart = scriptlets.start(1);
            int scriptletEnd = scriptlets.end(1);
            if (scriptletStart>last) {
                templateChunks.add(new GspTemplateChunk(false, text.substring(last, scriptletStart)));
            }
            templateChunks.add(new GspTemplateChunk(true, text.substring(scriptletStart+2, scriptletEnd-2)));
            last = scriptletEnd;
        }
        templateChunks.add(new GspTemplateChunk(false, text.substring(last, text.length())));
        return templateChunks;
    }

    private static @NotNull String assembleChunksIntoScript(@NotNull List<GspTemplateChunk> chunks) {
        StringBuilder sb = new StringBuilder(1024*10);
        for (GspTemplateChunk chunk : chunks) {
            if (chunk.isScriptlet) {
                String[] importSpecs = importSpecs(chunk);
                if (importSpecs == null) {
                    sb.append(chunk.text);
                } else {
                    for (String importSpec : importSpecs) {
                        sb.append("import ").append(importSpec).append(";\n");
                    }
                }
            } else {
                sb.append("\nout.print(\"\"\"")
                  .append(chunk.text.replace("\"", "\\\""))
                  .append("\"\"\");\n");
            }
        }
        return sb.toString();
    }

    private static @Nullable String[] importSpecs(@NotNull GspTemplateChunk chunk) {
        if (!chunk.isScriptlet) return null;

        Matcher matcher = PAGE_IMPORT_PATTERN.matcher(chunk.text);
        if (!matcher.matches()) return null;

        String imports = matcher.group(2);
        return imports.replaceAll("^\\s*|\\s*$", "").split("\\s*;\\s*");
    }

    private static class GspTemplateChunk {
        final boolean isScriptlet;
        final @NotNull String text;

        GspTemplateChunk(boolean isScriptlet, @NotNull String text) {
            this.isScriptlet = isScriptlet;
            this.text = text;
        }
    }
}