Test.js

function getTimeMs() {
    var t = process.hrtime();
    return t[0] * 1000 + t[1] / 1000000;
}
function formatTime(tMs) {
    if (isNaN(tMs))
        return 'n/a';
    var h = Math.floor(tMs / (60 * 60 * 1000)), m = Math.floor(tMs / (60 * 1000) % 60), s = Math.floor(tMs / 1000 % 60), ms = Math.floor(tMs % 1000).toString();
    while (ms.length < 3)
        ms = "0" + ms;
    if (h > 0)
        return h + "h" + m + "m" + s + "." + ms + "s";
    if (m > 0)
        return m + "m" + s + "." + ms + "s";
    if (s > 0)
        return s + "." + ms + "s";
    return tMs.toFixed(0) + "ms";
}
function createData(t) {
    t = t % 1131;
    var len = t * t * t;
    var data = [];
    for (var i = 0; i < t; i++) {
        data[i] = (i % 6) === 0 ? ' ' : String.fromCharCode(48 + (i * i) % 74);
    }
    return data.join('');
}
function processData(tokenizer, t) {
    if (global.gc && t % 100 === 0) {
        global.gc();
    }
    var data = createData(t);
    var tokens = tokenizer(data);
    return tokens;
}
function run(size) {
    var tokenizer = process.argv[2] === 'slow' ? Slow.tokenize : Fast.tokenize;
    var start = getTimeMs();
    var counter = 0;
    for (var t = 0; t < size; t++)
        counter += processData(tokenizer, t);
    return getTimeMs() - start;
}
function suite(count) {
    var size = 10000;
    var times = [];
    for (var i = 0; i < count; i++) {
        times[i] = run(size);
    }
    times.sort(function (a, b) { return a - b; });
    var median = times[((count - 1) / 2) | 0];
    console.log(count + " runs: min " + formatTime(times[1]) + ", max " + formatTime(times[times.length - 2]) + ", median " + formatTime(median));
}
var Fast;
(function (Fast) {
    function eatValue(state) {
        while (state.position < state.length) {
            switch (state.data.charCodeAt(state.position)) {
                case 9: // \t
                case 10: // \n
                case 13: // \r
                case 32:
                    state.currentTokenEnd = state.position;
                    return;
                default:
                    ++state.position;
                    break;
            }
        }
        state.currentTokenEnd = state.position;
    }
    function skipWhitespace(state) {
        var prev = 10;
        while (state.position < state.length) {
            var c = state.data.charCodeAt(state.position);
            switch (c) {
                case 9: // '\t'
                case 32:
                    prev = c;
                    ++state.position;
                    break;
                case 10:
                    // handle \r\n
                    if (prev !== 13) {
                        ++state.currentLineNumber;
                    }
                    prev = c;
                    ++state.position;
                    break;
                case 13:
                    prev = c;
                    ++state.position;
                    ++state.currentLineNumber;
                    break;
                default:
                    return prev;
            }
        }
        return prev;
    }
    function moveNext(state) {
        var prev = skipWhitespace(state), c;
        if (state.position >= state.length) {
            state.currentTokenType = 1 /* End */;
            return;
        }
        state.currentTokenStart = state.position;
        state.currentTokenEnd = state.position;
        c = state.data.charCodeAt(state.position);
        eatValue(state);
        state.currentTokenType = 0 /* Value */;
    }
    function cifTokenizer(data) {
        return {
            data: data,
            length: data.length,
            position: 0,
            currentTokenStart: 0,
            currentTokenEnd: 0,
            currentTokenType: 1 /* End */,
            currentLineNumber: 1
        };
    }
    function tokenize(data) {
        var tok = cifTokenizer(data);
        var count = 0;
        moveNext(tok);
        while (tok.currentTokenType !== 1 /* End */) {
            count++;
            moveNext(tok);
        }
        return count;
    }
    Fast.tokenize = tokenize;
})(Fast || (Fast = {}));
var Slow;
(function (Slow) {
    var Tokenizer = (function () {
        function Tokenizer(data) {
            this.data = data;
            this.length = data.length;
            this.position = 0;
            this.currentTokenStart = 0;
            this.currentTokenEnd = 0;
            this.currentTokenType = 1 /* End */;
            this.currentLineNumber = 1;
        }
        Tokenizer.prototype.eatValue = function () {
            while (this.position < this.length) {
                switch (this.data.charCodeAt(this.position)) {
                    case 9: // \t
                    case 10: // \n
                    case 13: // \r
                    case 32:
                        this.currentTokenEnd = this.position;
                        return;
                    default:
                        ++this.position;
                        break;
                }
            }
            this.currentTokenEnd = this.position;
        };
        Tokenizer.prototype.skipWhitespace = function () {
            var prev = 10;
            while (this.position < this.length) {
                var c = this.data.charCodeAt(this.position);
                switch (c) {
                    case 9: // '\t'
                    case 32:
                        prev = c;
                        ++this.position;
                        break;
                    case 10:
                        // handle \r\n
                        if (prev !== 13) {
                            ++this.currentLineNumber;
                        }
                        prev = c;
                        ++this.position;
                        break;
                    case 13:
                        prev = c;
                        ++this.position;
                        ++this.currentLineNumber;
                        break;
                    default:
                        return prev;
                }
            }
            return prev;
        };
        Tokenizer.prototype.moveNext = function () {
            var prev = this.skipWhitespace(), c;
            if (this.position >= this.length) {
                this.currentTokenType = 1 /* End */;
                return;
            }
            this.currentTokenStart = this.position;
            this.currentTokenEnd = this.position;
            c = this.data.charCodeAt(this.position);
            this.eatValue();
            this.currentTokenType = 0 /* Value */;
        };
        return Tokenizer;
    }());
    function tokenize(data) {
        var tok = new Tokenizer(data);
        var count = 0;
        tok.moveNext();
        while (tok.currentTokenType !== 1 /* End */) {
            count++;
            tok.moveNext();
        }
        return count;
    }
    Slow.tokenize = tokenize;
})(Slow || (Slow = {}));
suite(7);