2-layer lexer for IDEA

LayeredLexer.java

package cz.juzna.latte.lexer;

import com.intellij.lexer.Lexer;
import com.intellij.lexer.LookAheadLexer;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;

/**
 * Uses base-lexer to get big chunks, and several of them (typesToLex) are then lexed into smaller tokens using sub-lexer.
 *
 * Useful in some templating languages, where you lex-out whole chunks of your language, which is then lexed by another lexer.
 *
 * @author Jan Dolecek <juzna.cz@gmail.com>
 * @see http://devnet.jetbrains.net/message/5450157
 */
public class LayeredLexer extends LookAheadLexer {
    protected Lexer subLexer;
    protected TokenSet typesToLex;

    /**
     * @param baseLexer  Lexer which creates big chunks
     * @param subLexer   Lexer which is applied to chunks from previous lexer
     * @param typesToLex Only these chunks from base-lexer are processed
     */
    public LayeredLexer(Lexer baseLexer, Lexer subLexer, TokenSet typesToLex) {
        super(baseLexer);
        this.subLexer = subLexer;
        this.typesToLex = typesToLex;
    }

    @Override
    protected void lookAhead(Lexer baseLexer) {
        if(typesToLex.contains(baseLexer.getTokenType())) {
            // Parse all sub tokens
            IElementType subToken;
            subLexer.start(baseLexer.getBufferSequence(), baseLexer.getTokenStart(), baseLexer.getTokenEnd());
            while((subToken = subLexer.getTokenType()) != null) {
                addToken(subLexer.getTokenEnd(), subToken);
                subLexer.advance();
            }

            baseLexer.advance();

        } else {
            addToken(baseLexer.getTokenType());
            baseLexer.advance();
        }
    }
}