tokenizer-example.js

var foo = function (match, offset) {
	return { tag: 'foo', offset: offset }
}

var bar = function (match, offset) {
	return { tag: 'bar', offset: offset }
}

var baz = function (match, offset) {
	return { tag: 'baz', offset: offset }
}

var invalid = function () {
	throw new Error('Invalid character.')
}

var entries = [
	[/foo/, foo],
	[/bar/, bar],
	[/baz/, baz],
	[true,  invalid]
]

var tokenize = tokenizer(entries)

console.log(tokenize('foobarbaz'))

tokenizer.js

function tokenizer (entries) {
	var other, i, patterns, regExp
	for (i = 0; i < entries.length; i++) {
		if (entries[i][0] === true) {
			other = entries[i][1]
			break
		}
	}
	if (other === undefined) {
		throw new Error('Missing other entry.')
	}
	patterns = []
	for (i = 0; i < entries.length; i++) {
		regExp = entries[i][0]
		if (regExp !== true) {
			patterns.push('(' + regExp.source + ')')
		}
	}
	regExp = new RegExp(patterns.join('|'), 'g')
	return function tokenize (string) {
		var last = 0
		var tokens = []
		var matches
		var i
		var match
		var misc
		var entry
		while (matches = regExp.exec(string)) {
			misc = string.substring(last, matches.index)
			if (misc !== '') {
				tokens.push(other(misc), last)
			}
			for (i = 1; i < matches.length; i++) {
				match = matches[i]
				if (match !== undefined) {
					entry = entries[i - 1]
					tokens.push(entry[1](entry[0].exec(match), matches.index))
					break
				}
			}
			last = regExp.lastIndex
		}
		misc = string.substring(last)
		if (misc !== '') {
			tokens.push(other(misc, last))
		}
		return tokens
	}
}