study http://compilers.iecc.com/crenshaw/tutor7.txt

lexical scanner.cpp

#include <string>
#include <vector>
#include <iostream>
using namespace std;

char Look;

typedef string Symbol;
typedef vector<Symbol> SymTab;
typedef SymTab* TabPtr;
enum SymType {
	IfSym, ElseSym, EndifSym, EndSym, Ident, Number, Operator
};

SymType Token;
string Value;

// Definition of Keywords and Token Types
SymTab KWList;

// Table Lookup
// If the input string matches a table entry, return the entry index.
// If not, return a -1.
int Lookup(TabPtr T, string s) {
	SymTab::iterator iter = T->begin();
	for (; iter != T->end(); iter++) {
		if (*iter == s)
			return (iter-T->begin());
	}
	return -1;
}

void Expected(const string& str) {
	cout << "Expected " << str << endl;
	exit(-1);
}

void GetChar() {
	Look = getchar();
}

bool IsWhite(char ch) {
	if (ch == ' ' ||
	    //ch == '\n' ||
	    //ch == '\r' ||
	    ch == '\t') {
		return true;
	}
	else
		return false;
}

void SkipWhite() {
	while (IsWhite(Look)) {
		GetChar();
	}
}

// Skip Over a Comma
void skipComma() {
	SkipWhite();
	if (Look == ',') {
		GetChar();
		SkipWhite();
	}
}

void Fin() {
	while (Look == '\n' || Look == '\r') {
		GetChar();
	}
}

// Recognize Any Operator
bool IsOp(char c) {
	switch (c) {
		case '+':
		case '-':
		case '*':
		case '/':
		case '<':
		case '>':
		case ':':
		case '=':
			return true;
		default:
			return false;
	}
}

void Init() {
	GetChar();

	string KWarray[] = {
		"IF",
		"ELSE",
		"ENDIF",
		"END"
	};
	size_t KWarray_count = sizeof(KWarray)/sizeof(string);
	KWList.insert(KWList.begin(), KWarray, KWarray+KWarray_count);
}

// Get an Identifier
void GetName() {
	int k;
	if (!isalpha(Look))
		Expected("Name");
	Value.clear();
	while (isalnum(Look)) {
		Value.append(1, toupper(Look));
		GetChar();
	}
	k = Lookup(&KWList, Value);
	if (k == -1) Token = Ident;
	else Token = (SymType)k;
}

// Get a Number
void GetNum() {
	if (!isdigit(Look))
		Expected("Integer");
	Value.clear();
	while (isdigit(Look)) {
		Value.append(1, toupper(Look));
		GetChar();
	}
	Token = Number;
}

// Get Op
void GetOp() {
	if (!IsOp(Look))
		Expected("Op");
	else {
		Value.clear();
		Value.append(1, toupper(Look));
		GetChar();
	}
	Token = Operator;
}

// Lexical Scanner
void Scan() {
	while (Look == '\n') {
		Fin();
	}

	if (isalpha(Look)) {
		GetName();
	}
	else if (isdigit(Look)) {
		GetNum();
	}
	else if (IsOp(Look)) {
		GetOp();
	}
	else {
		Value = Look;
		Token = Operator;
		GetChar();
	}
	SkipWhite();
}

// Main Program
int main() {
	Init();
	do {
		Scan();
		switch (Token) {
			case Ident:
				cout << "Ident ";
				break;
			case Number:
				cout << "Number ";
				break;
			case Operator:
				cout << "Operator ";
				break;
			case IfSym:
			case ElseSym:
			case EndifSym:
			case EndSym:
				cout << "Keyword ";
				break;
		}
		cout << Value << endl;
	} while (Token != EndSym);
}

集中式词法解析