Skip to content

Instantly share code, notes, and snippets.

@fergiemcdowall
Last active December 21, 2015 03:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save fergiemcdowall/6239924 to your computer and use it in GitHub Desktop.
Save fergiemcdowall/6239924 to your computer and use it in GitHub Desktop.
A gist that stress tests levelUP by inserting lots of batch files containing JSON. This can be configured to create a "memory leak" in levelUP.
//fergusmcdowall@gmail.com
//A very simple stress test for levelUP
//make lots of LorumIpsum json files, put them into batches, and then
//cram as many of them as possible into LevelUP
//At the time of writing (15 aug 2013) this will create a memory leak
//adjust size of batches here
var totalBatchFiles = 300;
var totalDocsPerBatchFile = 100;
var totalFieldsPerDoc = 10;
var totalSentencesPerField = 50;
var loremIpsum = require('lorem-ipsum');
var levelup = require('levelup');
var db = levelup('./db',
{valueEncoding: 'json'})
for (var i = 0; i < totalBatchFiles; i++) {
var batch = [];
for (var j = 0; j < totalDocsPerBatchFile; j++) {
var value = {};
var key = i + '~' + j;
for (var k = 0; k < totalFieldsPerDoc; k++) {
var fieldText = loremIpsum({
count: totalSentencesPerField,
units: 'sentences',
sentenceLowerBound: 5,
sentenceUpperBound: 15,
paragraphLowerBound: 3,
paragraphUpperBound: 7,
format: 'plain',
random: Math.random
});
var fieldIndex = loremIpsum({
count: 1,
units: 'words',
format: 'plain',
random: Math.random
});
value[fieldIndex] = fieldText;
}
batch.push({
type: 'put',
key: key,
value: value});
console.log(key);
}
insertBatch(i, batch);
}
function insertBatch(id, thisBatch) {
console.log('queueing batch ' + id);
db.batch(thisBatch, function (err) {
console.log('inserted batch ' + id);
if (err) return console.log('Ooops!', err);
return;
});
}
{
"name": "LevelUpStressTest",
"description": "a module that creates pretent JSON files an inserts them into levelUP",
"version": "0.1.1",
"engines": {
"node": ">=0.4.10"
},
"dependencies": {
"lorem-ipsum": "*",
"levelup": "*",
"leveldown": "*"
},
"author": "Fergus McDowall <fergusmcdowall@gmail.com>"
}
@rvagg
Copy link

rvagg commented Aug 15, 2013

my version with some breathing space for V8, in the above version there is absolutely no room for the callbacks to get back in to the event loop.

//fergusmcdowall@gmail.com
//A very simple stress test for levelUP
//make lots of LorumIpsum json files, put them into batches, and then
//cram as many of them as possible into LevelUP

//At the time of writing (15 aug 2013) this will create a memory leak

//adjust size of batches here
var totalBatchFiles = 30000;
var totalDocsPerBatchFile = 100;
var totalFieldsPerDoc = 10;
var totalSentencesPerField = 50;

var loremIpsum = require('lorem-ipsum');
var levelup = require('levelup');
var db = levelup('./db',
                 {valueEncoding: 'json'})

var batches = 0
var inserted = 0
var queued = 0


function doBatch () {
  var batch = [];
  for (var j = 0; j < totalDocsPerBatchFile; j++) {
    var value = {};
    var key = batches + '~' + j;
    for (var k = 0; k < totalFieldsPerDoc; k++) {
      var fieldText = loremIpsum({
        count: totalSentencesPerField, 
        units: 'sentences',
        sentenceLowerBound: 5,
        sentenceUpperBound: 15,
        paragraphLowerBound: 3,
        paragraphUpperBound: 7,
        format: 'plain',
        random: Math.random   
      });
      var fieldIndex = loremIpsum({
        count: 1, 
        units: 'words',
        format: 'plain',
        random: Math.random   
      });
      value[fieldIndex] = fieldText;
    }
    batch.push({
      type: 'put',
      key: key,
      value: value});
    //console.log(key);
  }
  insertBatch(batches, batch);
  if (++batches < totalBatchFiles)
    setImmediate(doBatch)
}
db.on('ready', doBatch)

setInterval(function () {
  console.log(queued, inserted, process.memoryUsage())
}, 5000)

function insertBatch(id, thisBatch) {
  queued++
  //console.log('queueing batch ' + id);
  db.batch(thisBatch, function (err) {
    queued--
    inserted++
    //console.log('inserted batch ' + id);
    if (err) return console.log('Ooops!', err);
    return;
  });
}

@fergiemcdowall
Copy link
Author

for convenience: link back to issue Level/levelup#171

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment