Request for Comments on new API for Step

twostep.js

module.exports = TwoStep;

var slice = Array.prototype.slice;

function Group(callback) {
  this.args = [null];
  this.left = 0;
  this.callback = callback;
  this.isDone = false;
}

Group.prototype.done = function done() {
  if (this.isDone) return;
  this.isDone = true;
  this.callback.apply(null, this.args);
};

Group.prototype.error = function error(err) {
  if (this.isDone) return;
  this.isDone = true;
  var callback = this.callback;
  callback(err);
};

// Simple utility for passing a sync value to the next step.
Group.prototype.pass = function pass() {
  var values = slice.call(arguments);
  for (var i = 0, l = values.length; i < l; i++) {
    this.args.push(values[i]);
  }
};

// Register a slot in the next step and return a callback
Group.prototype.slot = function slot() {
  var group = this;
  var index = group.args.length;
  group.args.length++;
  group.left++;
  return function (err, data) {
    if (err) return group.error(err);
    group.args[index] = data;
    if (--group.left === 0) group.done();
  };
}

// Creates a nested group where several callbacks go into a single array.
Group.prototype.makeGroup = function makeGroup() {
  var group = this;
  var index = this.args.length;
  this.args.length++;
  group.left++;
  return new Group(function (err) {
    if (err) return group.error(err);
    var data = slice.call(arguments, 1);
    group.args[index] = data;
    if (--group.left === 0) group.done();
  });
};

// Expose just for fun and extensibility
TwoStep.Group = Group;

// Stepper function
function exec(steps, args, callback) {
  var pos = 0;
  next.apply(null, args);
  function next() {
    var step = steps[pos++];
    if (!step) {
      callback && callback.apply(null, arguments);
      return;
    }
    var group = new Group(next);
    step.apply(group, arguments);
    if (group.left === 0) group.done();
  }
}

// Execute steps immedietly
function TwoStep() {
  exec(slice.call(arguments), []);
}


// Create a composite function with steps built-in
TwoStep.fn = function () {
  var steps = slice.call(arguments);
  return function () {
    var args = slice.call(arguments);
    var callback = args.pop();
    exec(steps, args, callback);
  };
}

usage-fn.js

var TwoStep = require('./twostep');
var FS = require('fs');
var Path = require('path');

// Create a composite function using TwoStep.fn
var statdir = TwoStep.fn(
  function (directory) {
    this.pass(directory);
    FS.readdir(directory, this.slot());
  },
  function (err, directory, fileNames) {
    if (err) return this.error(err);
    this.pass(directory, fileNames);
    var group = this.makeGroup();
    fileNames.forEach(function (name) {
      FS.stat(name, group.slot());
    });
  },
  function (err, directory, filenames, stats) {
    if (err) return this.error(err);
    var output = {};
    filenames.forEach(function (name, i) {
      var path = Path.join(directory, name);
      output[path] = stats[i];
    });
    this.pass(output);
  }
);

statdir(__dirname, function (err, stats) {
  if (err) throw err;
  console.log("Stats", stats);
})

usage.js

var TwoStep = require('./twostep');
var FS = require('fs');

TwoStep(
  function one() {
    this.pass(__filename + ".bak");
    FS.readFile(__filename, 'utf8', this.slot());
  },
  function two(err, target, contents) {
    if (err) throw err;
    this.pass(target);
    FS.writeFile(target, contents, this.slot())
  },
  function three(err, target) {
    if (err) throw err;
    console.log("%s written to successfully", target);
    FS.readdir(__dirname, this.slot());
  },
  function four(err, fileNames) {
    if (err) throw err;
    this.pass(fileNames);
    var group = this.makeGroup();
    fileNames.forEach(function (filename) {
      FS.stat(filename, group.slot());
    });
  },
  function five(err, fileNames, stats) {
    if (err) throw err;
    this.pass(fileNames.filter(function (name, i) {
      return stats[i].isFile();
    }));
    var group = this.makeGroup();
    stats.forEach(function (stat, i) {
      if (stat.isFile()) FS.readFile(fileNames[i], 'utf8', group.slot());
    });
  },
  function six(err, fileNames, contents) {
    if (err) throw err;
    var merged = {};
    fileNames.forEach(function (name, i) {
      merged[name] = contents[i].substr(0, 80);
    });
    console.log(merged);
  }
);

Thanks for good work. We used the first step, but we didn't have this.pass function. We used your gist as base and add some features - simple wrappers, try catch to steps and tests. Source can be found here https://github.com/Nordberg/node-twostep and in npm twostep.

Great work @nordberg! I mentioned your project on twitter. https://twitter.com/creationix/status/215446113422032896

I think the new API comes directly out of everyone's demand. I wish I had noticed this effort earlier.

While I'm using Step I always do this.pass() in this way:

this.parallel()(undefined, results);

Now there is an API to do it explicitly. Other changes in the new API are just name changes. They are also good because they look more intuitive than before.

And I don't return values synchronously either. If I do that my syntax checker always warns me that some branches return without values while others do return values. Therefore currently I stick with this.parallel()(undefined, results); or callback(undefined, results);, which looks pretty stupid. I should consider migrating to the new API :)

So far I'm happy with Step except for two messes. One is, as I reported and fixed, caused by the process.nextTick() check. Another is caused by try ... catch in Step. I should have reported an issue, anyway, here it is:

    try {
      lock = true;
      var result = fn.apply(next, arguments);
    } catch (e) {
      // Pass any exceptions on through the next callback
      next(e);
    }

Suppose in certain step of fn.apply(next, arguments);, fn wanna finish prematurely and call the final callback, and the final callback throws an exception, then it gets catched by catch(e){next(e);} -- and we can't finish prematurely, have to go on to next step!

With nested callbacks and exceptions things get more complicated, so I try not to throw or incur an exception in my asynchronous code.

I hope these two messes has been taken care of. Anyway I'll have a look into the code of TwoStep and try it soon:)

My attemp (I ported test from Step, but it doesn't pass all yet). Step is anyway much simpler and elegant then async. It covers 80% of use cases. For thoes 20% we can expand API if it suits.

We (@GameClosure) loved the ideas behind this gist, so we made our own "fork" of it called FF. We expanded on TwoStep by adding immediate failure and success calls (skipping the rest of the function chain), and we made the return object promise compatible. We are actively maintaining it currently, so feedback is highly welcomed.