Skip to content

Instantly share code, notes, and snippets.

@pfmiles
Created June 22, 2011 10:53
Show Gist options
  • Save pfmiles/1039860 to your computer and use it in GitHub Desktop.
Save pfmiles/1039860 to your computer and use it in GitHub Desktop.
Hand write java Iterable lexer code skeleton
/**
* This is a skeleton code to implement a iterable lexer. Which is able to be processed in a convenient
* for-each loop...
*/
public class ActionLexer implements Iterable<ActionToken> {
private Reader reader;
// 19 status total
private static final int[] stats = new int[19];
static {
for (int i = 0; i < 19; i++)
stats[i] = i;
}
// if eof token returned
private boolean eofReturned = false;
// char buffer, stores next n chars
private char[] cb = new char[1024];
// current char index, -1 when no char in buffer
private int p = -1;
// current valid char length in buffer
private int validLength = 0;
// null char
private static final char nil = 0xffff;
// the stateless, singleton iterator
private final Iterator<ActionToken> iter = new Iterator<ActionToken>() {
public boolean hasNext() {
return p != -1 || !eofReturned;
}
public ActionToken next() {
// first invoke
if (validLength == 0) {
readToBuffer();
}
int status = stats[0];
// currently matched string
StringBuilder sb = new StringBuilder();
// is final state (about to return a matched token)
boolean fnl = false;
ActionToken matchedToken = null;
while (!fnl) {
char c = currentChar();
switch (status) {
// your hand-write DFA code goes here...
case 0:
if (isBlankChar(c)) {
// ignore any BLANK chars
p++;
} else if (')' == c) {
sb.append(cb[p++]);
status = 18;
} else if ('(' == c) {
......
}
}
return matchedToken;
}
// return the current char
private char currentChar() {
if (p > validLength - 1 && validLength != -1) {
readToBuffer();
}
if (validLength == -1) {
return nil;
} else {
return cb[p];
}
}
private void readToBuffer() {
try {
validLength = reader.read(cb);
if (validLength != -1) {
p = 0;
} else {
p = -1;
}
} catch (IOException e) {
throw new ActionException(e);
}
}
public void remove() {
throw new UnsupportedOperationException("Not supported.");
}
};
/**
* lexing upon specified string
*
* @param code
*/
public ActionLexer(String code) {
this.reader = new CharArrayReader(code.toCharArray());
}
/**
* lexing upon given reader
*
* @param reader
*/
public ActionLexer(Reader reader) {
this.reader = reader;
}
public Iterator<ActionToken> iterator() {
return this.iter;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment