Skip to content

Instantly share code, notes, and snippets.

@esnya
Created January 13, 2015 08:13
Show Gist options
  • Save esnya/868433e0f4975ec882cb to your computer and use it in GitHub Desktop.
Save esnya/868433e0f4975ec882cb to your computer and use it in GitHub Desktop.
import std.algorithm;
import std.exception;
import std.stdio;
import std.range;
import std.traits;
import std.typecons;
class InvalidCharacerException : Exception {
this(string msg, string file = __FILE__, uint line = __LINE__) {
super(msg, file, line);
}
}
auto lexChar(alias Char, Range)(Range range) if (isInputRange!Range) {
return range.until!(a => a != Char)();
}
auto lexPlus(Range)(Range range) {
return lexChar!('+', Range)(range);
}
auto lexMinus(Range)(Range range) {
return lexChar!('-', Range)(range);
}
auto lexStar(Range)(Range range) {
return lexChar!('*', Range)(range);
}
auto lexSlash(Range)(Range range) {
return lexChar!('/', Range)(range);
}
auto lexLParen(Range)(Range range) {
return lexChar!('(', Range)(range);
}
auto lexRParen(Range)(Range range) {
return lexChar!(')', Range)(range);
}
auto lexNumeric(Range)(Range range) if (isInputRange!Range) {
return range.until!(a => !std.uni.isNumber(a))();
}
struct Tokenizer(Args...) {
alias Lex = Args;
alias TokenType = typeof(Args[1]);
}
struct Token(Type, Range) {
Type type;
Range value;
}
struct Tokenize(Tokenizer, Range) if (isForwardRange!Range) {
alias Token = .Token!(Tokenizer.TokenType, ElementType!Range[]);
private Range _range;
private Token _front;
private bool _empty;
this(Range range) {
_range = range.save();
_empty = _range.empty();
if (!_empty) {
popFront();
}
}
auto front() const {
return _front;
}
auto popFront() in {
assert(!empty);
} body {
if (_range.empty) {
_empty = true;
} else {
foreach (i, lex; Tokenizer.Lex) {
static if (i % 2 == 0) {
auto lexed = lex(_range);
if (!lexed.empty) {
_front = Token(Tokenizer.Lex[i + 1], lexed.array);
_range.popFrontN(_front.value.length);
return;
}
}
}
throw new InvalidCharacerException(_range);
}
}
auto empty() const {
return _empty;
}
Tokenize!(Tokenizer, Range) save() {
return Tokenize!(Tokenizer, Range)(_range);
}
}
auto tokenize(Tokenizer, Range)(Range range) {
return Tokenize!(Tokenizer, Range)(range);
}
enum TokenType {
Numeric, Plus, Minus, Star, Slash, LParen, RParen,
}
unittest {
auto range = "12+20*3-4*(22+3)";
alias tokenizer = Tokenizer!(
lexNumeric, TokenType.Numeric,
lexPlus, TokenType.Plus,
lexMinus, TokenType.Minus,
lexStar, TokenType.Star,
lexSlash, TokenType.Slash,
lexLParen, TokenType.LParen,
lexRParen, TokenType.RParen);
auto tokens = tokenize!(tokenizer)(range);
static assert(isInputRange!(typeof(tokens)));
//static assert(isForwardRange!(typeof(tokens)));
writeln(tokens);
assert(equal(tokens, [
Token!(TokenType, dchar[])(TokenType.Numeric, "12"d.dup),
Token!(TokenType, dchar[])(TokenType.Plus, "+"d.dup),
Token!(TokenType, dchar[])(TokenType.Numeric, "20"d.dup),
Token!(TokenType, dchar[])(TokenType.Star, "*"d.dup),
Token!(TokenType, dchar[])(TokenType.Numeric, "3"d.dup),
Token!(TokenType, dchar[])(TokenType.Minus, "-"d.dup),
Token!(TokenType, dchar[])(TokenType.Numeric, "4"d.dup),
Token!(TokenType, dchar[])(TokenType.Star, "*"d.dup),
Token!(TokenType, dchar[])(TokenType.LParen, "("d.dup),
Token!(TokenType, dchar[])(TokenType.Numeric, "22"d.dup),
Token!(TokenType, dchar[])(TokenType.Plus, "+"d.dup),
Token!(TokenType, dchar[])(TokenType.Numeric, "3"d.dup),
Token!(TokenType, dchar[])(TokenType.RParen, ")"d.dup),
]));
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment