Skip to content

Instantly share code, notes, and snippets.

@soldair
Created March 22, 2013 23:42
Show Gist options
  • Save soldair/5225603 to your computer and use it in GitHub Desktop.
Save soldair/5225603 to your computer and use it in GitHub Desktop.
levelup writeStream doesn't pause when behind. this leads to huge memory leaks importing lots of data especially from stdin.
var level = require('levelup')
, db = level('./streamtest.db')
var dbws = db.createWriteStream()
, c = 0
, start = Date.now()
, intervalCount = 0
, bytes = 0
, paused = false
;
dbws.on('pause',function(){
console.log('paused.');
paused = true;
}).on('drain',function(){
console.log('drained.');
paused = false;
});
// generate 10000 key values per ms
(function fn(){
if(paused) return setImmediate(fn);
var s = Date.now(), values = 0,key,value;
while(s == Date.now() || values > 10000){
values++;
key = 'a_key_'+(++c);
value = '{"time":'+s+',"updated":'+s+',"id":'+c+',"number":1,"string":"some important string value"}';
dbws.write({key:key,value:value});
bytes += key.length+value.length;
}
intervalCount += values;
setImmediate(fn);
})()
interval = setInterval(function(){
console.log(c/(Date.now()-start),'values per ms. ',intervalCount,' since last interval ',bytes,' bytes and ',c,' rows written so far.');
intervalCount = 0;
},1000);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment