Skip to content

Instantly share code, notes, and snippets.

@bastibeckr
Created March 12, 2017 15:27
Show Gist options
  • Save bastibeckr/e1e4c035dbeb5fb3ff031d0f269cc971 to your computer and use it in GitHub Desktop.
Save bastibeckr/e1e4c035dbeb5fb3ff031d0f269cc971 to your computer and use it in GitHub Desktop.
const schema = require('./schema')
const db = require('./connection')
const fileName = './elastic-dump-jsonlines.txt'
const readJson = require('./read-json')
db.once('open', function () {
console.log('Connected to DB.')
readJsonLines()
})
function readJsonLines () {
const rl = readJson(fileName)
const Model = db.model('Model', schema)
const updateOpts = { upsert: true, new: true }
let lines = 0
rl.on('close', function () {
console.log('END. (%s lines)', lines)
})
rl.on('line', function (line) {
try {
let obj = JSON.parse(line)
Clip.findByIdAndUpdate(obj._id, obj, updateOpts).exec()
.then((clip) => { rl.resume() })
.catch((err) => {
console.log('Error writing obj to DB at line %s: %s', lines, err.toString())
})
} catch (e) {
console.log('Error parsing JSON at line %s: %s', lines, e.toString())
}
lines++
console.log('Read %s lines.', lines)
})
}
/**
* Read a file line by line
* (used for files that contain JSON-Objects separated by newline)
*/
var readline = require('readline')
const Stream = require('stream')
const fs = require('fs')
/**
* Pass a fileName, get a readline-Stream.
*
* @param {[type]} fileName [description]
* @return {[type]} [description]
*/
function readLines (fileName) {
const stream = fs.createReadStream(fileName, {
flags: 'r',
encoding: 'utf-8',
fd: null,
bufferSize: 1
})
// let lines = 0
const outstream = new Stream()
const rl = readline.createInterface(stream, outstream)
stream.on('error', (e) => {
console.log('Error in read-stream: %s', e.toString())
})
stream.on('end', () => {
console.log('Read-Stream ended.')
rl.close()
})
return rl
}
module.exports = readLines
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment