JSON stream to multiple files

json-to-files.js

'use strict';

const fs = require('fs');
const path = require('path');
const { Writable } = require('stream');

class JSONToFiles extends Writable {
  constructor (options, params) {
    super(options);

    this.params = params;
    this.outputs = new Map();
  }

  _write(chunk, encoding, next) {
    const outStream = this._resolveOutStream(chunk);
    const { mapFn } = this.params;

    if (mapFn) {
      chunk = mapFn(chunk);
    }

    if (typeof chunk !== 'string') {
      chunk = JSON.stringify(chunk);
    }

    outStream.write(chunk);
    next();
  }

  _resolveOutStream(chunk) {
    const key = this.params.fileName(chunk);

    let outStream = this.outputs.get(key);
    if (!outStream) {
      outStream = this._newOutStream(chunk);
    }

    return outStream;
  }

  _newOutStream(chunk) {
    const key = this.params.fileName(chunk);

    const dir = this.params.dir;
    const extension = this.params.extension;
    const fullFileName = `${key}.${extension}`;

    const outStream = fs.createWriteStream(path.join(dir, fullFileName));
    outStream.write('{\n');
    this.outputs.set(key, outStream);

    return outStream;
  }

  _final() {
    for (const o of this.outputs.values()) {
      o.end('}');
    }
  }
}

module.exports = JSONToFiles;

main.js

const miss = require('mississippi');

const write = new JSONToFiles(
  {
    objectMode: true,
  },
  {
    dir: __dirname + '/out',
    fileName: (item) => {
      return item.language;
    },
    extension: 'json',
    mapFn: (item) => {
      let { key, translation } = item;
      return `"${key}": ${JSON.stringify(translation)}`;
    }
  }
);

miss.pipe(
  // Reader & transform streams ommited
  write,
  function (err) {
    if (err) return console.error('Error!', err)
    console.log('🎉!')
  }
);