Skip to content

Instantly share code, notes, and snippets.

@yaraki
Last active August 29, 2015 14:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save yaraki/8d5356c530c7c2c2796a to your computer and use it in GitHub Desktop.
Save yaraki/8d5356c530c7c2c2796a to your computer and use it in GitHub Desktop.
S-expression lexer in C++11
#include <iostream>
#include <sstream>
namespace ya {
void each_token(std::istream& input, void (*proc)(const std::string& token))
{
std::stringstream buffer;
auto send = [&buffer, &proc]() {
if (0 < buffer.tellp()) {
proc(buffer.str());
buffer.str("");
return true;
}
return false;
};
while (!input.eof()) {
char c = input.get();
switch (c) {
case EOF:
case ' ':
case '\r':
case '\n':
case '\t':
case '\v':
send();
break;
case '(':
case ')':
case '[':
case ']':
case '\'':
case '`':
if (send()) {
input.unget();
} else {
buffer << c;
send();
}
break;
case ',':
if (send()) {
input.unget();
} else {
buffer << c;
if ('@' == input.peek()) {
buffer << (char) input.get();
}
send();
}
break;
case '\"':
if (send()) {
input.unget();
} else {
buffer << c;
bool escaped = false;
while (!input.eof()) {
char c = input.get();
buffer << c;
if (!escaped && c == '"') {
send();
break;
} else if (c == '\\') {
escaped = true;
} else {
escaped = false;
}
}
}
break;
default:
buffer << c;
break;
}
}
send();
}
}
int main()
{
std::stringstream input("`(a ,b ,@(c) \"d e\")");
ya::each_token(input, [](const std::string& token) {
std::cout << token << std::endl;
});
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment