Proposed lexer API

lexer.js

const matching = {
  '{': '}',
  '}': '{',
};

export const tokenize = (lexEngine) => {
  const { start, get, take, end, setType } = lexEngine; // token fns
  const { peekChar, pushState, popState, match, matchable, allMatched } = lexEngine;

  const literal = (n) => {
    start('literal');
    take(n);
    return end();
  };
  const ctx = {
    escaped: false,
  };

  const keywords = new Set(['import']);

  return {
    bare() {
      const char = peekChar();
      if (char === '`') {
        literal(1);
        pushState('templateString');
      } else if (matchable(char)) {
        match(literal(1));

        if (char === '}' && allMatched()) popState();
      } else if (/[a-zA-Z$_]/.test(char)) {
        start('identifier');
        while (/[a-zA-Z0-9$_]/.test(peekChar())) {
          take(1);
        }
        if (keywords.has(get())) {
          setType('keyword');
        }
        end();
      }
    },

    templateString() {
      const char = peekChar();
      const { escaped } = ctx;
      ctx.escaped = false;

      if (char === '\\' && !escaped) {
        ctx.escaped = true;
      }

      if (char === '`') {
        end();
        popState();
        return 'bare';
      } else if (char === '$' && peekChar() === '{' && !escaped) {
        end();
        literal(2);
        pushState('bare');
        match('{');
      } else {
        start('string');
      }
    },
  };
};

const lexer = new Lexer(tokenize, { matching });
lexer.lex(' abc '); // [ Identifier`abc` ]
lexer.lex('`abc`'); // [ Literal`\``, String`abc`, Literal`\`` ]
lexer.lex('`"${name}"`'); // [ Literal`\``, String`"`, Literal`${`, Identifier`name`, Literal`}`, String`"`, Literal`\`` ]
lexer.lex('Identifier`String`'); // [ Identifier`Identifier`, Literal`\``, String`String`, Literal`\`` ]