Simple Tokenizer

fjs_tokenizer.cpp

#include "fjs_tokenizer.h"

List<Symbol> tokenize(char* str) {
	
	string* temp = new string();
	string* code = new string(str);
	List<Symbol> result;
	
	// Parse the code.
	for ( int i = 0; i < code->size() ; i++ ) {
		// Get the character.
		char c = code->getAt(i);
		Symbol s = convert(new string(c));
		
		// Check for some kind of known symbol.
		if (isFinished(temp, c, s) || i == code->size() - 1) {		
			// Skip empty strings.
			if ( temp->size() > 0 ) 
				result.add(convert(temp));
			
			// Clear stuff out.
			temp->clear();
			
			// If the character is not whitespace and it's not known
			// append it. Otherwise, if it's known, add to result.
			if ( c != ' ' && s == oddsym ) temp->append(c);			
			else if ( s != oddsym ) result.add(s);
		} else {
			temp->append(c);
		}
	}
	
	return result;
}

bool isFinished(string* str, char next, Symbol nxt) {
	if ( next == ' ' ) return true;
	else if ( next == '\n' ) return true;
	else if ( nxt != oddsym ) return true;
	
	return false;
}

Symbol convert(string* str) {
		
	if ( str->equals("(") ) return lparen;
	else if ( str->equals(")") ) return rparen;
	else if ( str->equals("{") ) return lbracket;
	else if ( str->equals("}") ) return rbracket;
	else if ( str->equals(",") ) return comma;
	else if ( str->equals("=") ) return eql;
	else if ( str->equals(">") ) return gtr;
	else if ( str->equals("<") ) return lss;
	else if ( str->equals("+") ) return plus;
	else if ( str->equals("-") ) return minus;
	else if ( str->equals("*") ) return times;
	else if ( str->equals(".") ) return period;
	else if ( str->equals(";") ) return semicolon;
	else if ( str->equals("'") ) return quotesym;
	else if ( str->equals("\"") ) return quotesym;
	else if ( str->equals("if") ) return ifsym;
	else if ( str->equals("var") ) return varsym;
	else if ( str->equals("while") ) return whilesym;
	else if ( str->equals("else") ) return elsesym;
	else if ( str->equals("function") ) return functionsym;
	
	return oddsym;
}