Data import script for performance test with ParseServer.

importDataToParseServer.js

/**
 * Requirement:
 *   sudo npm install -g babel-cli
 *   sudo npm install -g babel-preset-es2015
 *   sudo npm install -g babel-preset-es2017
 *   sudo npm install -g parse
 *   sudo npm install -g randomstring
 *   sudo npm install -g shuffle-seed
 *
 * To execute:
 *   1. Create .babelrc at the same directory.
 *   2. In the .babelrc, put the following text.
 *      { "presets": ["es2015", "es2017"] }
 *   3. Execute commands in bash terminal like the following.
 *      $ babel-node --expose_gc <this-js-file-name>
 *      > let Feeder = require('./this-js-file-name');
 *      > let feeder = new Feeder(120, 100000, 'test1');
 *      > feeder.importDataAsync();
 *
 */

let Parse = require('parse/node');
Parse.initialize('an-app-id-for-running-parse-server');
Parse.serverURL = 'http://localhost:28080/api/v1';

let randomstring = require('randomstring');
let shuffleSeed = require('shuffle-seed');

let seed = 'seed';
let columnCount = 120;
let rowCount = 10;
let maxImportUnit = 10000;
let className = "perf1";

let dataFeeder = function(columns, rows, name){
  this.columns = columns ? columns : columnCount;
  this.rows = rows ? rows : rowCount;
  this.name = name ? name : className;

  this.PerfObject = Parse.Object.extend(this.name);
};

dataFeeder.prototype.generateRowSeed = function(){
  let record = [];
  let cntType = {};

  cntType.String =  this.columns * 0.50;
  cntType.Number =  this.columns * 0.20;
  cntType.Boolean = this.columns * 0.10;
  cntType.Date =    this.columns * 0.05;
  cntType.Array =   this.columns * 0.15;

  // Generate random string whose length is between 3 and 24.
  for(let i=0;i<cntType.String;i++){
    record.push({
      colName: "String" + i,
      value: randomstring.generate((parseInt(Math.random()*21) + 3))
    });
  }
  // Generate random number.
  for(let i=0;i<cntType.Number;i++){
    record.push({
      colName: "Number" + i,
      value: Math.random()
    });
  }
  // Generate random boolean.
  for(let i=0;i<cntType.Boolean;i++){
    record.push({
      colName: "Boolean" + i,
      value: Math.random() >= 0.5
    });
  }
  // Generate random date between 2000/01/01 00:00:00 and 2017/12/31 23:59:59
  for(let i=0;i<cntType.Date;i++){
    record.push({
      colName: "Date" + i,
      value: new Date(946652400000 + Math.random()* 568079999000)
    });
  }
  // Generate random length of array whose values is a set of string with random length(3-15)
  for(let i=0;i<cntType.Array;i++){
    let arr = new Array((parseInt(Math.random()*6) + 1)).fill(0);
    record.push({
      colName: "Array" + i,
      value: arr.map(()=>randomstring.generate((parseInt(Math.random()*12) + 3)))
    });
  }

  return shuffleSeed.shuffle(record, seed);
};

dataFeeder.prototype.generateParseObjects = function(rows=null){
  console.log("Started generating ParseObjects.");
  rows = rows ? rows : this.rows;

  let startTime = Date.now();

  let perfObjects = new Array(rows).fill(0).map(function(v1,i1){
    let parseObj = new this.PerfObject();
    this.generateRowSeed().map(function(v2){
      parseObj.set(v2.colName, v2.value)
    });

    return parseObj;
  }.bind(this));

  let endTime = Date.now();

  console.log("Generating ParseObject finished! Elapsed time: " + (endTime - startTime) + " msec");

  return perfObjects;
};

dataFeeder.prototype.generateParseObjectsAsync = function(rows=null){
  return new Promise(function(resolve, reject){
    resolve(this.generateParseObjects(rows));
  }.bind(this));
};

dataFeeder.prototype.importDataUnitAsync = function(perfObjects){
  if(!perfObjects){
    console.err("Error. Please generate parse objects first");
    throw new Error();
  }

  console.log("Started importing ParseObject data.");
  let startTime = Date.now();
  return Parse.Object.saveAll(perfObjects, {
    success: function(list){
      let endTime = Date.now();
      console.log("Finished importing ParseObject data. Elapsed time: "+ (endTime - startTime) + " msec");
    },
    error: function(err){
      let endTime = Date.now();
      console.error(err);
      console.log("Error while importing ParseObject data. Elapsed time: "+ (endTime - startTime) + " msec");
    }
  });
};

dataFeeder.prototype.importDataAsync = async function(rows=null){
  rows = rows ? rows : this.rows;

  console.log("Started importing Parse Objects.");
  console.log("Cols:" + this.columns + " Rows:" + rows);

  if(rows > maxImportUnit && !global.gc){
    console.error("Please run babel-node with '--expose_gc' optin");
    throw new Error();
  }

  let startTime = Date.now();
  let repeatCnt = Math.floor(rows / maxImportUnit);
  let remainder = rows % maxImportUnit;

  for(let i=0;i<repeatCnt;i++){
    let startTimeLocal;
    await this.generateParseObjectsAsync(maxImportUnit).then(
      function(result){
        return this.importDataUnitAsync(result);
      }.bind(this)
    ).then(function(result){
      console.log("Progress:" + (i+1)+"/" + (repeatCnt + (remainder>0?1:0)) + " Total ElapsedTime: " + (Date.now()-startTime) + " msec");
    });

    console.log("GC started");
    console.log(process.memoryUsage());
    global.gc();
    console.log(process.memoryUsage());
    console.log("GC finished");
  }
  if(remainder > 0){
    let startTimeLocal;
    await this.generateParseObjectsAsync(remainder).then(
      function(result){
        return this.importDataUnitAsync(result);
      }.bind(this)
    ).then(function(result){
      console.log("Progress:" + (repeatCnt+1)+"/" + (repeatCnt+1) + " Total ElapsedTime: " + (Date.now()-startTime) + " msec");
    });
  }

  console.log("Import data done!");
  console.log("Total elapsed time: " + (Date.now() - startTime) + " msec");
};

exports = module.exports = dataFeeder;