Streaming CSV Parsing in D3

chunkParser.js

var d3 = require("../d3"), // Make sure this has this patch: https://github.com/mbostock/d3/pull/783
    fs = require("fs");

var bufferSize = 1 << 20;

streamCSV("test.csv", row, done);

function row(d) {
  console.log("this is a row", d);
}

function done() {
  console.log("all done!");
}

function streamCSV(filename, row, callback) {
  var stream = fs.createReadStream(filename, {encoding: "utf8", bufferSize: bufferSize}),
      parse = chunkParser(d3.csv.parseRows, row);
  stream.on("data", parse);
  stream.on("end", function() {
    parse();
    callback(null);
  });
};

function chunkParser(parse, callback) {
  var n = 0,
      remainder = "";
  return function(text) {
    text = remainder + (text || "");
    var offset0 = 0,
        offset1 = 0,
        previous = null;

    parse(text, buffer);
    if (arguments.length) remainder = text.substring(offset1);
    else buffer();

    function buffer(d, i, j) {
      if (previous != null) callback(previous, n++);
      previous = d;
      offset1 = offset0;
      offset0 = j;
    }
  };
};

test.csv