Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save bouriate/f4285d07d1eb2169da296e8ddb0dae5b to your computer and use it in GitHub Desktop.
Save bouriate/f4285d07d1eb2169da296e8ddb0dae5b to your computer and use it in GitHub Desktop.
// Parsing huge logfiles in Node.js - read in line-by-line
//
// http://stackoverflow.com/a/23695940/4989460
//
// npm install event-stream --save
//
var fs = require('fs')
, util = require('util')
, stream = require('stream')
, es = require('event-stream');
var lineNr = 1;
var s = fs.createReadStream('very-large-file.csv')
.pipe(es.split())
.pipe(es.mapSync(function(line){
// pause the readstream
s.pause();
lineNr += 1;
// process line here and call s.resume() when rdy
logMemoryUsage(lineNr);
// resume the readstream
s.resume();
})
.on('error', function(){
console.log('Error while reading file.');
})
.on('end', function(){
console.log('Read entirefile.')
})
);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment