Skip to content

Instantly share code, notes, and snippets.

@midnightcodr
Last active March 24, 2017 01:20
Show Gist options
  • Save midnightcodr/077cae7d2168c520a971de8f42607774 to your computer and use it in GitHub Desktop.
Save midnightcodr/077cae7d2168c520a971de8f42607774 to your computer and use it in GitHub Desktop.
mongodb bulk insert with bluebird disposer
module.exports = function(col, limit=5000) {
this.bulk = col.initializeUnorderedBulkOp()
let cnt = 0
this.add = record => {
cnt++
this.bulk.insert(record)
if(cnt >= limit) {
this.flush()
}
return this.bulk
}
this.flush = () => {
if(cnt > 0) {
this.bulk.execute()
this.bulk = col.initializeUnorderedBulkOp()
cnt=0
}
return this.bulk
}
}
const MongoClient=require('mongodb').MongoClient
const Promise = require('bluebird')
module.exports = (connOpts) => {
return MongoClient
.connect(connOpts, {promiseLibrary: Promise})
.then(db => db)
.disposer(db => {
db.close()
})
}
const N=2000000
const Promise = require('bluebird')
const conn = require('./connect')
const Bulker = require('./bulk')
Promise.using(conn('mongodb://localhost:27017/test'), db => {
const col = db.collection('colTest')
const bulker = new Bulker(col)
console.time('prepare')
let task = []
for(let k=0;k<N;k++) {
task.push(k)
}
console.timeEnd('prepare')
console.time('task')
return Promise.each(task, t=> {
return bulker.add({v: t})
}).then(() => {
console.timeEnd('task')
return bulker.flush()
})
})
// npm install mongodb bluebird
// node main.js
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment