with_tag.py

# type: ignore
"""
A SLY lexer and parser for a Liquid `with` tag.

expr        : ID ":" value { "," ID ":" value }

value       : literal
            | path

literal     : FLOAT 
            | INTEGER
            | STRING
            | TRUE
            | FALSE
            | NIL
            | NULL
            | EMPTY
            | BLANK

path        : ID { prop }

prop        : bracketed
            | dotted
        
bracketed   : "[" elem "]"

dotted      : DOT ID

elem        : path
            | INTEGER
            | STRING
"""

from sly import Lexer
from sly import Parser

from liquid.ast import Node
from liquid.parse import expect
from liquid.tag import Tag

from liquid.expression import FloatLiteral
from liquid.expression import IntegerLiteral
from liquid.expression import StringLiteral
from liquid.expression import Identifier
from liquid.expression import IdentifierPathElement
from liquid.expression import TRUE
from liquid.expression import FALSE
from liquid.expression import NIL
from liquid.expression import BLANK
from liquid.expression import EMPTY

from liquid.exceptions import LiquidSyntaxError

from liquid.token import TOKEN_EXPRESSION
from liquid.token import TOKEN_EOF


class WithTagLexer(Lexer):
    tokens = {
        FLOAT,
        DOT,
        INTEGER,
        ID,
        TRUE,
        FALSE,
        NIL,
        NULL,
        EMPTY,
        BLANK,
        STRING,
    }

    literals = {"[", "]", ",", ":"}
    ignore = " \t\r"

    DOT = r"\."

    @_(r"-?\d+\.(?!\.)\d*")
    def FLOAT(self, t):
        t.value = float(t.value)
        return t

    @_(r"-?\d+")
    def INTEGER(self, t):
        t.value = int(t.value)
        return t

    @_(r"'")
    def SINGLEQUOTE(self, t):
        self.begin(SingleQuoteStringLexer)

    @_(r"\"")
    def DOUBLEQUOTE(self, t):
        self.begin(DoubleQuoteStringLexer)

    ID = r"\w[\w\-]*\??"
    ID["true"] = TRUE
    ID["false"] = FALSE
    ID["nil"] = NIL
    ID["null"] = NULL
    ID["empty"] = EMPTY
    ID["blank"] = BLANK

    @_(r"\n+")
    def ignore_newline(self, t):
        self.lineno += t.value.count("\n")

    def error(self, t):
        raise LiquidSyntaxError(f"unexpected {t.value[0]!r}", self.lineno)


class SingleQuoteStringLexer(Lexer):
    tokens = {ESCAPE, SINGLEQUOTE, STRING}

    STRING = r"[^\\']+"

    @_(r"\\'")
    def ESCAPE(self, t):
        t.value = "'"
        return t

    @_(r"'")
    def SINGLEQUOTE(self, t):
        self.begin(WithTagLexer)

    def error(self, t):
        raise LiquidSyntaxError(f"unexpected {t.value[0]!r}", self.lineno)


class DoubleQuoteStringLexer(Lexer):
    tokens = {ESCAPE, DOUBLEQUOTE, STRING}

    STRING = r'[^\\"]+'

    @_(r'\\"')
    def ESCAPE(self, t):
        t.value = '"'
        return t

    @_(r'"')
    def DOUBLEQUOTE(self, t):
        self.begin(WithTagLexer)

    def error(self, t):
        raise LiquidSyntaxError(f"unexpected {t.value[0]!r}", self.lineno)


class WithTagParser(Parser):
    tokens = WithTagLexer.tokens

    @_('ID ":" value { "," ID ":" value }')
    def expr(self, p):
        return {p.ID0: p.value0, **{k: v for k, v in zip(p.ID1, p.value1)}}

    @_("literal")
    def value(self, p):
        return p.literal

    @_("path")
    def value(self, p):
        assert isinstance(p.path, Identifier)
        return p.path

    @_("FLOAT")
    def literal(self, p):
        return FloatLiteral(p.FLOAT)

    @_("INTEGER")
    def literal(self, p):
        return IntegerLiteral(p.INTEGER)

    @_("STRING")
    def literal(self, p):
        return StringLiteral(p.STRING)

    @_("TRUE")
    def literal(self, p):
        return TRUE

    @_("FALSE")
    def literal(self, p):
        return FALSE

    @_(
        "NIL",
        "NULL",
    )
    def literal(self, p):
        return NIL

    @_("EMPTY")
    def literal(self, p):
        return EMPTY

    @_("BLANK")
    def literal(self, p):
        return BLANK

    @_("ID { prop }")
    def path(self, p):
        return Identifier([p.ID, *p.prop])

    @_(
        "bracketed",
        "dotted",
    )
    def prop(self, p):
        return p[0]

    @_('"[" elem "]"')
    def bracketed(self, p):
        return p.elem

    @_("DOT ID")
    def dotted(self, p):
        return p.ID

    @_(
        "INTEGER",
        "STRING",
    )
    def elem(self, p):
        return IdentifierPathElement(p[0])

    @_("path")
    def elem(self, p):
        return p.path

    def error(self, p):
        raise LiquidSyntaxError(
            f"unexpected {p.value[0]!r}",
            linenum=p.lineno,
        )


class WithNode(Node):
    def __init__(self, tok, args, block):
        self.tok = tok
        self.args = args
        self.block = block

    def render_to_output(self, context, buffer):
        namespace = {k: v.evaluate(context) for k, v in self.args.items()}

        with context.extend(namespace):
            self.block.render(context, buffer)


class WithTag(Tag):
    name = "with"
    end = "endwith"

    def __init__(self, env):
        super().__init__(env)
        self.parser = get_parser(self.env)

    def parse(self, stream):
        # Remember the first token. We'll pass this to `WithNode` so
        # it can report render-time errors with line numbers.
        tok = stream.current

        # Assert that an expression immediately follows the "tag" token.
        stream.next_token()
        expect(stream, TOKEN_EXPRESSION)

        # Use our SLY lexer to tokenize the current token's value.
        tokens = WithTagLexer().tokenize(stream.current.value)
        # Use our SLY parser to parse those tokens to a dictionary of
        # keyword arguments.
        args = WithTagParser().parse(tokens)

        # Parse the tag's block using the parser attached to the
        # current environment.
        stream.next_token()
        block = self.parser.parse_block(stream, (self.end, TOKEN_EOF))

        return WithNode(tok, args, block)

An alternative implementation of the with tag demonstrated in Python Liquid's docs. This implementation uses SLY to tokenize and parse with expressions.