Skip to content

Instantly share code, notes, and snippets.

@electrum
Created June 14, 2018 16:45
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save electrum/4ba3da9e7bbf0e47de1b2c2a3f71fed2 to your computer and use it in GitHub Desktop.
Save electrum/4ba3da9e7bbf0e47de1b2c2a3f71fed2 to your computer and use it in GitHub Desktop.
ANTLR print used tokens
package com.facebook.presto.sql.parser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.NotSetTransition;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.testng.annotations.Test;
public class TestTokens
{
@Test
public void testUnusedTokens()
{
IntervalSet tokens = new IntervalSet();
ATN atn = SqlBaseParser._ATN;
for (ATNState state : atn.states) {
for (Transition transition : state.getTransitions()) {
IntervalSet label = transition.label();
if (label != null) {
if (transition instanceof NotSetTransition) {
label = label.complement(IntervalSet.of(Token.MIN_USER_TOKEN_TYPE, atn.maxTokenType));
}
tokens.addAll(label);
}
}
}
for (Integer token : tokens.toSet()) {
String name = SqlBaseLexer.VOCABULARY.getDisplayName(token);
System.out.println(name);
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment