benchmark.js

var Benchmark = require('benchmark');
var child_process = require('child_process');
var fs = require('fs');
var colors = require('colors');
var path = require('path');
var Promise = require('bluebird');
var vm = require('vm');

var Immutable = require('immutable');
var immstruct = require('immstruct');
var Cursor = require('immutable/contrib/cursor');

var exec = Promise.promisify(child_process.exec);
var readdir = Promise.promisify(fs.readdir);
var readFile = Promise.promisify(fs.readFile);

var perfDir = path.resolve(__dirname, '../perf/');

Promise.all([
  readFile(path.resolve(__dirname, '../dist/omniscient.js'), { encoding: 'utf8' }),
  exec('git show master:dist/omniscient.js')
]).then(function (args) {
  var newSrc = args[0];
  var oldSrc = args[1].toString({ encoding: 'utf8' }).slice(0, -1); // wtf, comma?
  return newSrc === oldSrc ? [newSrc] : [newSrc, oldSrc];
}).then(function (sources) {
  return sources.map(function (source) {
    var sourceExports = {};
    var sourceModule = { exports: sourceExports };
    vm.runInNewContext(source, {
      require: require,
      module: sourceModule,
      exports: sourceExports
    }, 'omniscient.js');
    return sourceModule.exports;
  });
}).then(function (modules) {
  return readdir(perfDir).then(function (filepaths) {
    return Promise.all(filepaths.map(function (filepath) {
      return readFile(path.resolve(perfDir, filepath)).then(function (source) {
        return {
          path: filepath,
          source: source
        };
      });
    }))
  }).then(function (sources) {
    var tests = {};

    modules.forEach(function (omniscient, version) {
      sources.forEach(function (source) {
        var description = [];
        var beforeStack = [];
        var beforeFn;
        var prevBeforeFn;

        function describe(name, fn) {
          description.push(name);
          beforeStack.push(prevBeforeFn);
          prevBeforeFn = beforeFn;
          fn();
          beforeFn = prevBeforeFn;
          prevBeforeFn = beforeStack.pop();
          description.pop();
        }

        function beforeEach(fn) {
          beforeFn = !prevBeforeFn ? fn : function (prevBeforeFn) {
            return function () { prevBeforeFn(); fn(); };
          }(prevBeforeFn);
        }

        function it(name, test) {
          var fullName = description.join(' > ') + ' ' + name;
          (tests[fullName] || (tests[fullName] = {
            description: fullName,
            tests: []
          })).tests[version] = {
            before: beforeFn,
            test: test
          };
        }

        vm.runInNewContext(source.source, {
          describe: describe,
          it: it,
          beforeEach: beforeEach,
          console: console,
          immstruct: immstruct,
          omniscient: omniscient,
          Cursor: Cursor,
          Immutable: Immutable
        }, source.path);
      });
    });

    // Array<{
    //   description: String,
    //   tests: Array<{
    //     before: Function,
    //     test: Function
    //   }> // one per module, [new,old] or just [new]
    // }>
    return Object.keys(tests).map(function (key) { return tests[key]; });
  });
}).then(function (tests) {
  var suites = [];

  tests.forEach(function (test) {
    var suite = new Benchmark.Suite(test.description, {
      onStart: function (event) {
        console.log(event.currentTarget.name.bold);
        process.stdout.write('  ...running...  '.gray);
      },
      onComplete: function (event) {
        process.stdout.write('\r\x1B[K');
        var stats = Array.prototype.map.call(event.currentTarget, function (target) {
          return target.stats;
        });


        function pad(n, s) {
          return Array(Math.max(0, 1 + n - s.length)).join(' ') + s;
        }

        function fmt(b) {
          return Math.floor(b).toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
        }

        function pct(p) {
          return (Math.floor(p * 10000) / 100) + '%';
        }

        var dualRuns = stats.length === 2;

        if (dualRuns) {

          var prevMean = 1 / stats[1].mean;
          var prevLow = 1 / (stats[1].mean + stats[1].deviation * 2);
          var prevHigh = 1 / (stats[1].mean - stats[1].deviation * 2);

          // console.log(
          //   (dualRuns ? '  Old: '.bold.gray : '  ') +
          //   (
          //     pad(9, fmt(prevLow)) + ' ' +
          //     pad(9, fmt(prevMean)) + ' ' +
          //     pad(9, fmt(prevHigh)) + ' ops/sec'
          //   )
          // );

          var prevLowmoe = 1 / (stats[1].mean + stats[1].moe);
          var prevHighmoe = 1 / (stats[1].mean - stats[1].moe);

          console.log(
            (dualRuns ? '  Old: '.bold.gray : '  ') +
            (
              pad(9, fmt(prevLowmoe)) + ' ' +
              pad(9, fmt(prevMean)) + ' ' +
              pad(9, fmt(prevHighmoe)) + ' ops/sec'
            )
          );
        }

        var mean = 1 / stats[0].mean;
        var low = 1 / (stats[0].mean + stats[0].deviation * 2);
        var high = 1 / (stats[0].mean - stats[0].deviation * 2);

        // console.log(
        //   (dualRuns ? '  New: '.bold.gray : '  ') +
        //   (
        //     pad(9, fmt(low)) + ' ' +
        //     pad(9, fmt(mean)) + ' ' +
        //     pad(9, fmt(high)) + ' ops/sec'
        //   )
        // );

        var lowmoe = 1 / (stats[0].mean + stats[0].moe);
        var highmoe = 1 / (stats[0].mean - stats[0].moe);

        console.log(
          (dualRuns ? '  New: '.bold.gray : '  ') +
          (
            pad(9, fmt(lowmoe)) + ' ' +
            pad(9, fmt(mean)) + ' ' +
            pad(9, fmt(highmoe)) + ' ops/sec'
          )
        );

        if (dualRuns) {

          var diffMean = (mean - prevMean) / prevMean;

          var comparison = event.currentTarget[1].compare(event.currentTarget[0]);
          var comparison2 = event.currentTarget[0].compare(event.currentTarget[1]);
          console.log('  compare: ' + comparison + ' ' + comparison2);

          console.log('  diff: ' + pct(diffMean));

          function sq(p) {
            return p * p;
          }

          var rme = Math.sqrt(
            (sq(stats[0].rme / 100) + sq(stats[1].rme  / 100)) / 2
          );
          // console.log('rmeN: ' + stats[0].rme);
          // console.log('rmeO: ' + stats[1].rme);
          console.log('  rme: ' + pct(rme));
        }

        // console.log(stats);
      }
    });

    test.tests.forEach(function (run) {
      suite.add({
        fn: run.test,
        onStart: run.before,
        onCycle: run.before
      });
    });

    suites.push(suite);
  });

  var onBenchComplete;
  var promise = new Promise(function (_resolve) {
    onBenchComplete = _resolve;
  });

  Benchmark.invoke(suites, 'run', {
    onError: function () {
      console.log("args", arguments);
    },
    onAbort: function () {
      console.log("args", arguments);
    },
    onReset: function () {
      console.log("args", arguments);
    },
    onComplete: onBenchComplete
  });

  return onBenchComplete;
}).then(function () {
  console.log('all done');
}).catch(function (error) {
  console.log('ugh', error.stack);
});

tests.js

var component = omniscient;
var shouldComponentUpdate = component.shouldComponentUpdate;

describe('shouldComponentUpdate', function () {

  it('on no input', function () {

    shouldComponentUpdate.call({ }, { })
  });

  it('when immutables are same', function () {
    var data = Immutable.fromJS({ foo: 'bar', bar: [1, 2, 3] });

    shouldComponentUpdate.call({
      props: { d: data},
    }, { d: data})
  });

  it('when many immutables are passed as the same reference', function () {
    var structures = {};
    for(var i = 0; i < 100; i++) {
      structures['data' + i] = Immutable.fromJS({ foo: 'bar', bar: [1, 2, 3] });
    }

    shouldComponentUpdate.call({
      props: { d: structures},
    }, { d: structures})
  });

  it('when large object is passed', function () {
    var objects = {};
    for(var i = 0; i < 100; i++) {
      objects['data' + i] = { foo: 'bar', bar: [1, 2, 3] };
    }

    shouldComponentUpdate.call({
      props: { d: objects},
    }, { d: objects })
  });

  it('when two large object is passed', function () {
    var objects = {};
    for(var i = 0; i < 100; i++) {
      objects['data' + i] = { foo: 'bar', bar: [1, 2, 3] };
    }

    var objects2 = {};
    for(var i = 0; i < 100; i++) {
      objects2['data' + i] = { foo: 'bar', bar: [1, 2, 3] };
    }

    shouldComponentUpdate.call({
      props: { d: objects},
    }, { d: objects2 })
  });

  it('when two large object is passed but as same input', function () {
    var objects = {};
    for(var i = 0; i < 100; i++) {
      objects['data' + i] = { foo: 'bar', bar: [1, 2, 3] };
    }

    shouldComponentUpdate.call({
      props: objects,
    }, objects)
  });

  it('when immutables are different', function () {
    var data = Immutable.fromJS({ foo: 'bar', bar: [1, 2, 3] });
    var data2 = Immutable.fromJS({ foo: 'bar', bar: [1, 2, 3] });

    shouldComponentUpdate.call({
      props: { d: data},
    }, { d: data2 })
  });

  it('when long arrays (1000 elements) are the same', function () {
    var data = Immutable.Range(0, 1000).toJS();
    var data2 = Immutable.Range(0, 1000).toJS();

    shouldComponentUpdate.call({
      props: { d: data},
    }, { d: data2 })
  });

  it('when long arrays (1000 elements) are the same reference', function () {
    var data = Immutable.Range(0, 1000).toJS();

    shouldComponentUpdate.call({
      props: { d: data},
    }, { d: data })
  });

  it('when same input is passed (long array, 1000 items)', function () {
    var input = { d : Immutable.Range(0, 1000).toJS() };

    shouldComponentUpdate.call({
      props: input,
    }, input)
  });

});