set/append/get performance test script of kyoto-client

kc_test.js

// test for kyoto-client.js

var crypto = require('crypto');

var randomSelect = function(array){
  return Math.floor(Math.random() * array.length);
};

var makedata = function(times){
  var seed = 'abcdefghijklmnopqrstuvwxyz012345'; //32bytes
  var val = '';
  for (var i = 0; i < times; i++) {
    val += seed;
  }
  return val;
};
var smalldata = makedata(10);    //        320 bytes ; query-key-list, info
var largedata = makedata(10000); //    320,000 bytes ; normal result
var bigdata = makedata(1000000); // 32,000,000 bytes ; big result

var hexkeys = (function(){
  var array = [];
  for(var i = 0; i < 1000; i++) {
    array.push(crypto.createHash('md5').update(i.toString()).digest('hex'));
  }
  return array;
})();
var keyA = [];
var keyB = ['aa1', 'bx2', 'bx3', 'caa0', 'cax10', 'czzxx9', 'dx', 'd9', 'ddd7', 'e', 'eeeax', 'ef9'];
var generateKeySets = function(year){
  var array = [];
  ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'].forEach(function(m){
    keyA.push(year+m);
    keyB.forEach(function(k){
      array.push({a:year+m, b:k, c:hexkeys.pop()});
    });
  });
  var ret = [];
  while (array.length > 0) {
    ret.push(array.splice(randomSelect(array),1)[0]);
  }
  return ret;
};

var kt = require('kyoto-client');
var db = new kt.Db();
db.open();

var clear_database = function(){
  db.clear(function(){
    db.clear('shibresults.kcd', function(){
      process_next_test();
    });
  });
};

var keysets1 = generateKeySets('2011').slice(0,50);
var serial_put_large = function(){
  var serial_put_large_start = (new Date()).getTime();
  var pushOne = function(num){
    if (num >= keysets1.length) {
      console.log("serial_put_large:", (new Date()).getTime() - serial_put_large_start);
      process_next_test();
      return;
    }
    var k = keysets1[num];
    db.append('history:' + k.a, ',' + k.c, function(err,output){
      db.append('services:' + k.b, ',' + k.c, function(err,output){
        db.set('queries:' + k.c, smalldata, function(err,output){
          db.set(k.c, largedata, 'shibresults.kcd', function(err,output){
            process.nextTick(function(){pushOne(num+1);});
          });
        });
      });
    });
  };
  pushOne(0);
};

var keysets2 = generateKeySets('2010').slice(0,2);
var serial_put_big = function(){
  var serial_put_big_start = (new Date()).getTime();
  var pushBigOne = function(num){
    if (num >= keysets2.length) {
      console.log("serial_put_big:", (new Date()).getTime() - serial_put_big_start);
      process_next_test();
      return;
    }
    var k = keysets2[num];
    db.append('history:' + k.a, ',' + k.c, function(err, output){
      db.append('services:' + k.b, ',' + k.c, function(err, output){
        db.set('queries:' + k.c, smalldata, function(err, output){
          db.set(k.c, bigdata, 'shibresults.kcd', function(err, output){
            process.nextTick(function(){pushBigOne(num+1);});
          });
        });
      });
    });
  };
  pushBigOne(0);
};

var fetch_large = [];
var get_random = function(){
  var get_random_start = (new Date()).getTime();
  var getOne = function(num){
    if (num >= keysets1.length * 100){
      console.log("get_random:", (new Date()).getTime() - get_random_start);
      process_next_test();
      return;
    }
    var k = keysets1[randomSelect(keysets1)];
    db.get("history:" + k.a, function(err, value){
      var qs = value.toString('utf8').split(',').slice(1);
      var q = qs[randomSelect(qs)];
      db.get("queries:" + q, function(err, value){
        fetch_large.push(q);
        process.nextTick(function(){getOne(num+1);});
      });
    });
  };
  getOne(0);
};
var get_random_large = function(){
  var get_random_large_start = (new Date()).getTime();
  var getLargeOne = function(num){
    if (num >= keysets1.length){
      console.log("get_random_large:", (new Date()).getTime() - get_random_large_start);
      process_next_test();
      return;
    }
    db.get(fetch_large[num], 'shibresults.kcd', function(err, value){
      process.nextTick(function(){getLargeOne(num+1);});
    });
  };
  getLargeOne(0);
};

var get_random_big = function(){
  var get_random_big_start = (new Date()).getTime();
  var getBigOne = function(num){
    if (num >= 1){
      console.log("get_random_big:", (new Date()).getTime() - get_random_big_start);
      process_next_test();
      return;
    }
    db.get(keysets2[randomSelect(keysets2)].c, 'shibresults.kcd', function(err, value){
      process.nextTick(function(){getBigOne(num+1);});
    });
  };
  getBigOne(0);
};

var tests = [clear_database, serial_put_large, serial_put_big, get_random, get_random_large, get_random_big];
var process_next_test = function(){
  if (tests.length < 1) {
    db.close(function(){
      process.exit(0);
    });
  }
  setTimeout(tests.shift(), 2000);
};

(tests.shift())();

kt_test.js

// test for kyoto-tycoon.js

var crypto = require('crypto');

var randomSelect = function(array){
  return Math.floor(Math.random() * array.length);
};

var makedata = function(times){
  var seed = 'abcdefghijklmnopqrstuvwxyz012345'; //32bytes
  var val = '';
  for (var i = 0; i < times; i++) {
    val += seed;
  }
  return val;
};
var smalldata = makedata(10);    //        320 bytes ; query-key-list, info
var largedata = makedata(10000); //    320,000 bytes ; normal result
var bigdata = makedata(1000000); // 32,000,000 bytes ; big result

var hexkeys = (function(){
  var array = [];
  for(var i = 0; i < 1000; i++) {
    array.push(crypto.createHash('md5').update(i.toString()).digest('hex'));
  }
  return array;
})();
var keyA = [];
var keyB = ['aa1', 'bx2', 'bx3', 'caa0', 'cax10', 'czzxx9', 'dx', 'd9', 'ddd7', 'e', 'eeeax', 'ef9'];
var generateKeySets = function(year){
  var array = [];
  ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'].forEach(function(m){
    keyA.push(year+m);
    keyB.forEach(function(k){
      array.push({a:year+m, b:k, c:hexkeys.pop()});
    });
  });
  var ret = [];
  while (array.length > 0) {
    ret.push(array.splice(randomSelect(array),1)[0]);
  }
  return ret;
};

var KyotoTycoon = require('kyoto-tycoon').KyotoTycoon;
var db = new KyotoTycoon();

var clear_database = function(){
  db.clear(function(){
    db.clear('shibresults.kcd', function(){
      process_next_test();
    });
  });
};

var keysets1 = generateKeySets('2011').slice(0,50);
var serial_put_large = function(){
  var serial_put_large_start = (new Date()).getTime();
  var pushOne = function(num){
    if (num >= keysets1.length) {
      console.log("serial_put_large:", (new Date()).getTime() - serial_put_large_start);
      process_next_test();
      return;
    }
    var k = keysets1[num];
    db.append('history:' + k.a, ',' + k.c, function(err,output){
      db.append('services:' + k.b, ',' + k.c, function(err,output){
        db.set('queries:' + k.c, smalldata, function(err,output){
          db.set(k.c, largedata, {DB:'shibresults.kcd'}, function(err,output){
            process.nextTick(function(){pushOne(num+1);});
          });
        });
      });
    });
  };
  pushOne(0);
};

var keysets2 = generateKeySets('2010').slice(0,2);
var serial_put_big = function(){
  var serial_put_big_start = (new Date()).getTime();
  var pushBigOne = function(num){
    if (num >= keysets2.length) {
      console.log("serial_put_big:", (new Date()).getTime() - serial_put_big_start);
      process_next_test();
      return;
    }
    var k = keysets2[num];
    db.append('history:' + k.a, ',' + k.c, function(err, output){
      db.append('services:' + k.b, ',' + k.c, function(err, output){
        db.set('queries:' + k.c, smalldata, function(err, output){
          db.set(k.c, bigdata, {DB:'shibresults.kcd'}, function(err, output){
            process.nextTick(function(){pushBigOne(num+1);});
          });
        });
      });
    });
  };
  pushBigOne(0);
};

var fetch_large = [];
var get_random = function(){
  var get_random_start = (new Date()).getTime();
  var getOne = function(num){
    if (num >= keysets1.length * 100){
      console.log("get_random:", (new Date()).getTime() - get_random_start);
      process_next_test();
      return;
    }
    var k = keysets1[randomSelect(keysets1)];
    db.get("history:" + k.a, function(err, value){
      var qs = value.toString('utf8').split(',').slice(1);
      var q = qs[randomSelect(qs)];
      db.get("queries:" + q, function(err, value){
        fetch_large.push(q);
        process.nextTick(function(){getOne(num+1);});
      });
    });
  };
  getOne(0);
};
var get_random_large = function(){
  var get_random_large_start = (new Date()).getTime();
  var getLargeOne = function(num){
    if (num >= keysets1.length){
      console.log("get_random_large:", (new Date()).getTime() - get_random_large_start);
      process_next_test();
      return;
    }
    db.get(fetch_large[num], {DB:'shibresults.kcd'}, function(err, value){
      process.nextTick(function(){getLargeOne(num+1);});
    });
  };
  getLargeOne(0);
};

var get_random_big = function(){
  var get_random_big_start = (new Date()).getTime();
  var getBigOne = function(num){
    if (num >= 1){
      console.log("get_random_big:", (new Date()).getTime() - get_random_big_start);
      process_next_test();
      return;
    }
    db.get(keysets2[randomSelect(keysets2)].c, {DB:'shibresults.kcd'}, function(err, value){
      process.nextTick(function(){getBigOne(num+1);});
    });
  };
  getBigOne(0);
};

var tests = [clear_database, serial_put_large, serial_put_big, get_random, get_random_large, get_random_big];
var process_next_test = function(){
  if (tests.length < 1) {
    db.end();
    process.exit(0);
  }
  setTimeout(tests.shift(), 2000);
};

(tests.shift())();