compareBenchmarks.js

#!/usr/bin/env node

/*global JSON*/
var childProcess = require('child_process');
var async = require('async');
var passError = require('passerror');
var commandLineOptions = require('optimist').demand(1).argv;
var results = [];

var refs = commandLineOptions._;
if (refs.length === 1) {
    refs.unshift('working dir');
}

function exec(commandLine, quiet, cb) {
    if (typeof quiet === 'function') {
        cb = quiet;
        quiet = false;
    }
    console.warn(commandLine);
    childProcess.exec(commandLine, function (err, stdout, stderr) {
        if (stderr.length > 0 && !quiet) {
            console.warn(stderr.toString('utf-8'));
        }
        cb(err, stdout, stderr);
    });
}

var dirtyWorkingTree = false;
exec('git diff-index --quiet HEAD', function (err, stdout, stderr) {
    if (err) {
        if (err.code > 0) {
            dirtyWorkingTree = true;
        } else {
            throw err;
        }
    }
    exec('git rev-parse --abbrev-ref HEAD', function (err, stdout, stderr) {
        if (err) {
            throw err;
        }
        var originalRef = stdout.toString('utf-8').replace(/\n/, '');
        async.eachLimit(refs, 1, function (ref, cb) {
            if (ref === 'working dir') {
                proceedToRunBenchmark();
            } else {
                if (dirtyWorkingTree) {
                    exec('git stash', passError(cb, function () {
                        exec('git checkout ' + ref.replace(/^HEAD/, originalRef), true, passError(cb, proceedToRunBenchmark));
                    }));
                } else {
                    exec('git checkout ' + ref.replace(/^HEAD/, originalRef), true, passError(cb, proceedToRunBenchmark));
                }
            }
            function proceedToRunBenchmark() {
                exec('make benchmark', passError(cb, function (stdout, stderr) {
                    var result = JSON.parse(stdout.toString('utf-8').replace(/^.*\n/, ''));
                    result.ref = ref;
                    results.push(result);
                    cb();
                }));
            }
        }, function (err) {
            if (err) {
                throw err;
            }
            var numValidResults = 0;
            var sumRatios = 0;
            console.log(results.map(function (result) {
                return result.ref;
            }).join(' vs. '));
            results[0].passes.forEach(function (result, i) {
                var otherResult = results[1].passes[i];
                if (otherResult.fullTitle === result.fullTitle) {
                    var ratio = result.metadata.operationsPrSecond / otherResult.metadata.operationsPrSecond;
                    numValidResults += 1;
                    sumRatios += ratio;
                    console.log(result.fullTitle, Math.round(result.metadata.operationsPrSecond), '/', Math.round(otherResult.metadata.operationsPrSecond), (100 * ratio).toFixed(2) + '%');
                }
            });
            var avg = sumRatios / numValidResults;
            console.log('avg', (100 * avg).toFixed(2) + '%');
            console.log(((avg > 1) ? results[0].ref : results[1].ref) + ' is the fastest');
            exec('git checkout ' + originalRef, true, function (err) {
                if (err) {
                    throw err;
                }
                if (dirtyWorkingTree) {
                    exec('git stash pop', function (err) {
                        if (err) {
                            throw err;
                        }
                    });
                }
            });
        });
    });
});