index-elastic-dump.js

const schema = require('./schema')
const db = require('./connection')
const fileName = './elastic-dump-jsonlines.txt'
const readJson = require('./read-json')

db.once('open', function () {
  console.log('Connected to DB.')
  readJsonLines()
})

function readJsonLines () {
  const rl = readJson(fileName)

  const Model = db.model('Model', schema)
  const updateOpts = { upsert: true, new: true }

  let lines = 0

  rl.on('close', function () {
    console.log('END. (%s lines)', lines)
  })

  rl.on('line', function (line) {
    try {
      let obj = JSON.parse(line)
      Clip.findByIdAndUpdate(obj._id, obj, updateOpts).exec()
      .then((clip) => { rl.resume() })
      .catch((err) => {
        console.log('Error writing obj to DB at line %s: %s', lines, err.toString())
      })
    } catch (e) {
      console.log('Error parsing JSON at line %s: %s', lines, e.toString())
    }
    lines++
    console.log('Read %s lines.', lines)
  })
  
}

read-json.js

/**
 * Read a file line by line
 * (used for files that contain JSON-Objects separated by newline)
 */

var readline = require('readline')
const Stream = require('stream')
const fs = require('fs')

/**
 * Pass a fileName, get a readline-Stream.
 *
 * @param  {[type]} fileName [description]
 * @return {[type]}          [description]
 */
function readLines (fileName) {
  const stream = fs.createReadStream(fileName, {
    flags: 'r',
    encoding: 'utf-8',
    fd: null,
    bufferSize: 1
  })
  // let lines = 0
  const outstream = new Stream()
  const rl = readline.createInterface(stream, outstream)

  stream.on('error', (e) => {
    console.log('Error in read-stream: %s', e.toString())
  })

  stream.on('end', () => {
    console.log('Read-Stream ended.')
    rl.close()
  })

  return rl
}

module.exports = readLines