Created
March 22, 2013 23:42
-
-
Save soldair/5225603 to your computer and use it in GitHub Desktop.
levelup writeStream doesn't pause when behind. this leads to huge memory leaks importing lots of data especially from stdin.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var level = require('levelup') | |
, db = level('./streamtest.db') | |
var dbws = db.createWriteStream() | |
, c = 0 | |
, start = Date.now() | |
, intervalCount = 0 | |
, bytes = 0 | |
, paused = false | |
; | |
dbws.on('pause',function(){ | |
console.log('paused.'); | |
paused = true; | |
}).on('drain',function(){ | |
console.log('drained.'); | |
paused = false; | |
}); | |
// generate 10000 key values per ms | |
(function fn(){ | |
if(paused) return setImmediate(fn); | |
var s = Date.now(), values = 0,key,value; | |
while(s == Date.now() || values > 10000){ | |
values++; | |
key = 'a_key_'+(++c); | |
value = '{"time":'+s+',"updated":'+s+',"id":'+c+',"number":1,"string":"some important string value"}'; | |
dbws.write({key:key,value:value}); | |
bytes += key.length+value.length; | |
} | |
intervalCount += values; | |
setImmediate(fn); | |
})() | |
interval = setInterval(function(){ | |
console.log(c/(Date.now()-start),'values per ms. ',intervalCount,' since last interval ',bytes,' bytes and ',c,' rows written so far.'); | |
intervalCount = 0; | |
},1000); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment