tokenizerTest.cpp

vector<string> specials{"{", "}", "[", "]", "(", ")"};
vector<string> keywords{"if", "else"};

enum class TokenType{Symbol, Keyword, Special, Numeric};

bool isKeyword(const std::string& mValue) { return contains(keywords, mValue); }
bool isSpecial(const std::string& mValue) { return contains(specials, mValue); }
bool isNumeric(const std::string& mValue) { for(const auto& c : mValue) if(!isdigit(c)) return false; return true; }

struct Token
{
	TokenType type;
	string value;

	Token(const string& mValue) : value(mValue)
	{
		if(isKeyword(mValue)) type = TokenType::Keyword;
		else if(isSpecial(mValue)) type = TokenType::Special;
		else if(isNumeric(mValue)) type = TokenType::Numeric;
		else type = TokenType::Symbol;
	}
};

struct Tokenizer
{
	bool isNonTokenChar(char mChar)		{ return iscntrl(mChar) || isspace(mChar); }
	bool isInitialTokenChar(char mChar)	{ return isalnum(mChar); }

	void emitToken(vector<char>& mCurrentToken, vector<Token>& mTarget)
	{
		if(mCurrentToken.empty()) return;

		string value{begin(mCurrentToken), end(mCurrentToken)};
		mTarget.emplace_back(value);
		mCurrentToken.clear();
	}

	vector<string> tokenize(const string& mString)
	{
		vector<Token> result;
		vector<char> currentToken;

		for(auto itr(begin(mString)); itr != end(mString); ++itr)
		{
			char c{*itr};

			if(isNonTokenChar(c)) emitToken(currentToken, result);
			else
			{
				if(!isInitialTokenChar(c)) emitToken(currentToken, result);
				currentToken.push_back(c);
			}
		}

		vector<string> fakeResult;
		for(const auto& t : result) fakeResult.push_back("{" + toStr((int)(t.type)) + "} '" + t.value + "'");
		return fakeResult;
	}
};


int main()
{
	Tokenizer t;
	for(const auto& s : t.tokenize("dhjmiofhjifafmnselse if else{ iff  banana15 15elses 1500 eelse else")) log(s);
}