script for verifying the output of running a test

flow-raw.js

// This test suite is for making sure that all the right functions are called
// in the right order. We keep track of when everything is ran and then
// output it all at the end.
//
// As such, you have to manually look at the quite cryptic output to make sure
// it is doing what you want.

if (module == require.main) {
  return require('../../lib/async_testing').run(process.ARGV, function() {/* do nothing */});
}

    // Which functions should be run async or sync. Each slot is a different flow
    // function and whether or not you want it to be sync or async.
    // this gives you fine tuned control over how to run the tests.
var runSyncAsync = ['S', 'S', 'S', 'S', 'S']
    // specify which tests to create. P means that function should pass,
    // F means it should fail
  , tests =
    { AA: 'PPP'
    , AB: 'FPP'
    , AC: 'PFP'
    , AD: 'PPF'
    , AE: 'FPF'
    , AF: 'PFF'
    , AG: 'PPPPP'
    , AH: 'FPPPP'
    , AI: 'PFPPP'
    , AJ: 'PPFPP'
    , AK: 'PPPFP'
    , AL: 'PPPPF'
    , AM: 'FPPPF'
    , AN: 'PFPFP'
    , AO: 'PFPPF'
    , AP: 'PPFFP'
    , AQ: 'PPFPF'
    , AR: 'PPPFF'
    , AS: 'EPP'
    , AT: 'PEP'
    , AU: 'PPE'
    , AV: 'EPPPP'
    , AW: 'PEPPP'
    , AX: 'PPEPP'
    , AY: 'PPPEP'
    , AZ: 'PPPPE'
    , BA: 'EEP'
    , BB: 'EPE'
    , BC: 'PEE'
    , BD: 'EFP'
    , BE: 'EPF'
    , BF: 'PEF'
    , BG: 'FEP'
    , BH: 'FPE'
    , BI: 'PFE'
    }
  ;

for (var key in tests) {
  combinations(tests[key]).forEach(function(t) {
    var k = key;
    var test = [];
    var name = '';

    t.forEach(function(f) {
      var index = test.length + 1;

      if (f == 'P') {
        name += 'pass ';
        test.push(function(t) {
          console.log(k+index);
          t.finish();
        });
      }
      else if (f == 'p') {
        name += 'apass ';
        test.push(function(t) {
          console.log(k.toLowerCase()+index);
          setTimeout(function() {
            t.finish();
          }, 10);
        });
      }
      else if (f == 'E') {
        name += 'error ';
        test.push(function(t) {
          console.log(k+index+'!');
          throw new Error('error in '+index);
        });
      }
      else if (f == 'e') {
        name += 'aerror ';
        test.push(function(t) {
          console.log(k.toLowerCase()+index+'!');
          setTimeout(function() {
            throw new Error('error in '+index);
          }, 10);
        });
      }
      else if (f == 'F') {
        name += 'fail ';
        test.push(function(t) {
          console.log(k+index+'*');
          t.ok(false, 'failure in '+index);
        });
      }
      else {
        name += 'afail ';
        test.push(function(t) {
          console.log(k.toLowerCase()+index+'*');
          setTimeout(function() {
            t.ok(false, 'failure in '+index);
          }, 10);
        });
      }
    });

    //exports[name.trim()] = test;
    exports[k] = test;
  });
}

function combinations(list, spot) {
  if (!spot) {
    spot = 0;
  }
  if (list.length > 1) {
    var right = combinations(list.slice(1), spot+1);
    var r = [];

    for (var i = 0; i < right.length; i++) {
      if (list[0] == 'E' || runSyncAsync[spot] == 'S') {
        r.push([list[0]].concat(right[i]));
      }
      else {
        r.push([list[0].toLowerCase()].concat(right[i]));
      }
    }

    return r;
  }
  else {
    var r = [];

    if (list[0] == 'E' || runSyncAsync[spot] == 'S') {
      r.push(list[0]);
    }
    else {
      r.push(list[0].toLowerCase());
    }
    return  r;
  }
}

flow-raw.json

exports.serial =
  { tests:
    { AA: false
    , AB: 'failure in 1'
    , AC: 'failure in 2'
    , AD: 'failure in 3'
    , AE: 'failure in 1'
    , AF: 'failure in 2'
    , AG: false
    , AH: 'failure in 1'
    , AI: 'failure in 2'
    , AJ: 'failure in 3'
    , AK: 'failure in 4'
    , AL: 'failure in 5'
    , AM: 'failure in 1'
    , AN: 'failure in 2'
    , AO: 'failure in 2'
    , AP: 'failure in 3'
    , AQ: 'failure in 3'
    , AR: 'failure in 4'
    , AS: 'error in 1'
    , AT: 'error in 2'
    , AU: 'error in 3'
    , AV: 'error in 1'
    , AW: 'error in 2'
    , AX: 'error in 3'
    , AY: 'error in 4'
    , AZ: 'error in 5'
    , BA: 'error in 1'
    , BB: 'error in 1'
    , BC: 'error in 2'
    , BD: 'error in 1'
    , BE: 'error in 1'
    , BF: 'error in 2'
    , BG: 'failure in 1'
    , BH: 'failure in 1'
    , BI: 'failure in 2'
    }
  , suite: 'complete'
  , stdout: 'AA1\nAA2\nAA3\nAB1*\nAB3\nAC1\nAC2*\nAC3\nAD1\nAD2\nAD3*\nAE1*\nAE3*\nAF1\nAF2*\nAF3*\nAG1\nAG2\nAG3\nAG4\nAG5\nAH1*\nAH5\nAI1\nAI2*\nAI4\nAI5\nAJ1\nAJ2\nAJ3*\nAJ4\nAJ5\nAK1\nAK2\nAK3\nAK4*\nAK5\nAL1\nAL2\nAL3\nAL4\nAL5*\nAM1*\nAM5*\nAN1\nAN2*\nAN4*\nAN5\nAO1\nAO2*\nAO4\nAO5*\nAP1\nAP2\nAP3*\nAP4*\nAP5\nAQ1\nAQ2\nAQ3*\nAQ4\nAQ5*\nAR1\nAR2\nAR3\nAR4*\nAR5*\nAS1!\nAS3\nAT1\nAT2!\nAT3\nAU1\nAU2\nAU3!\nAV1!\nAV5\nAW1\nAW2!\nAW4\nAW5\nAX1\nAX2\nAX3!\nAX4\nAX5\nAY1\nAY2\nAY3\nAY4!\nAY5\nAZ1\nAZ2\nAZ3\nAZ4\nAZ5!\nBA1!\nBA3\nBB1!\nBB3!\nBC1\nBC2!\nBC3!\nBD1!\nBD3\nBE1!\nBE3*\nBF1\nBF2!\nBF3*\nBG1*\nBG3\nBH1*\nBH3!\nBI1\nBI2*\nBI3!\n'
  }

verify.js

var testing = require('../../lib/testing')
  , assert = require('assert')
  ;

var files = process.ARGV.slice(2);

var index = 0
  , suites = []
  ;

files.forEach(function(file) {
  file = file.replace(/\.js$/, '');
  if (file.substr(0,1) !== '/') {
    file = process.cwd() + '/' + file;
  }

  var results =  require(file+'.json');

  suites.push([file, results.serial, false]);
  suites.push([file, results.parallel || results.serial, true]);
})

verifyNextSuite();

function verifyNextSuite() {
  if (index >= suites.length) {
    // no more suites to verify
    return;
  }

  var file = suites[index][0]
    , results = suites[index][1]
    , parallel = suites[index][2]
    , stdout = ''
    ;

  index++;

  var opts =
    { parallel: parallel
    , onTestDone: function(status, r) {
        //TODO
        // right now we can handle success and failure, we need to handle
        // not run, never finished and apart of a 'multi-error'
        if (status == 'success') {
          assert.strictEqual(false, results.tests[r.name]);
        }
        else {
          assert.strictEqual(r.failure.message, results.tests[r.name]);
        }
      }
    , onSuiteDone: function(status, r) {
        assert.strictEqual(status, results.suite);

        /*
        var expected = results.stdout.split('\n');
        var actual = stdout.split('\n');
        for (var i = 0; i < Math.min(expected.length, actual.length); i++) {
          console.log(expected[i] + ' ' + actual[i]);
        }
        */
        assert.equal(stdout, results.stdout);

        //TODO handle other statuses here
        verifyNextSuite();
      }
    , stdout: function(str) {
        //console.log(str);
        stdout += str;
      }
    }

  testing.runFile(file, opts);
}