Read and transform big file line by line with Node.js

index.js

var fs = require('fs'), es = require('event-stream')

// Define your input and output files

var inputFile = 'Path:/to/file'
var outputFile = 'Path:/to/file'

var lineNr = 0
var writeStream = fs.createWriteStream(outputFile)
var readStream = fs.createReadStream(inputFile)

readStream
  .pipe(es.split())
  .pipe(es.mapSync(function (line) {
    
    // We will pause the reading to process our data
    // The current processed line is in the 'line' variable
    // We will resume processing after transformation
    readStream.pause();
    lineNr++;
    transformAndWriteData(line, s.resume);
  })
    .on('error', function (err) {
      console.log('Error while reading file.', err);
    })
    .on('end', function () {
      console.log('Read entire file.')
    })
  );

function transformAndWriteData(data, callback) {
  // You can do here what ever you want.
  // It is possiblem to write the transformed line to a new file:
  // Here I read a tab-splitted file, concat it with ';' and transform some json data
  let n = data.split('\t')[0].concat(';', JSON.stringify(data.split('\t')[1]), '\n')
  stream.write(n, function () {
    callback()
  })
}