Mongodb BenchMarking

BulkOpToDuplicate.md

import { Promise } from "bluebird";
// import fs from "fs";
import { ParameterizedContext } from "koa";
import Router from "koa-router";
import { performance, PerformanceObserver } from "perf_hooks";
const perQuery = 100;
const MongoClient = Promise.promisifyAll(require("mongodb").MongoClient);

const url = process.env.MONGODB_URI || "mongodb://127.0.0.1/OrderStats";
let db: any;
const obs = new PerformanceObserver(items => {
    console.log(
        items.getEntries()[0]?.name,
        " _:_ ",
        items.getEntries()[0].duration,
    );
    // performance.clearMarks();
});
obs.observe({ entryTypes: ["measure"] });
const connectToDB = async () => {
    db = await MongoClient.connect(url);
    db = db.db(url.split("/").splice(-1)[0]);
};
export const initScript = async (
    ctx: ParameterizedContext<any, Router.IRouterParamContext<any, {}>>,
) => {
    await connectToDB();
    performance.mark("FetchQueryStarts");
    console.log(db.collection);
    const result: any = await db
        .collection("oldorders")
        .find({})
        .sort({ _id: -1 })
        .skip(0 * perQuery)
        .limit(1)
        .toArray();

    performance.mark("FetchQueryEnds");
    performance.measure("FetchFromDB", "FetchQueryStarts", "FetchQueryEnds");
    delete result[0]._id;
    console.log("result", result[0]._id);
    await createHalfMillionRecords({ ...result[0] } as any);
    performance.measure("TotalTimeTaken", "FetchQueryStarts", "InsertArrayEnd");
    ctx.body = result;
};
export const createHalfMillionRecords = async (toDuplicateOrder: any) => {
    performance.mark("CreateArrayStarts");
    const orderChunk = new Array(5000).fill(
        JSON.parse(JSON.stringify(toDuplicateOrder)),
    );
    const chunkCollection = new Array(100).fill([...orderChunk]);
    performance.mark("CreateArrayEnds");
    performance.measure("CreateArray", "CreateArrayStarts", "CreateArrayEnds");
    performance.mark("InsertArrayStarts");
    const result = await Promise.map(
        chunkCollection,
        async (i, index) => {
            try {
                const bulk = db
                    .collection("oldorders")
                    .initializeUnorderedBulkOp();

                performance.mark(`InsertArrayStarts${index}`);
                i.map((order: any) => {
                    bulk.insert({ ...order });
                });
                await bulk.execute(); // insertMany is quite faster compared to create already compared
                performance.mark(`InsertArrayEnds${index}`);
                performance.measure(
                    `InsertArray in DB ${index}`,
                    `InsertArrayStarts${index}`,
                    `InsertArrayEnds${index}`,
                );
                return `${index}`;
            } catch (e) {
                console.log(e);
                return `failed ${index}`;
            }
        },
        { concurrency: 1 },
    );

    performance.mark("InsertArrayEnd");
    performance.measure(
        "Inserting in db",
        "InsertArrayStarts",
        "InsertArrayEnd",
    );
    console.log("result", result);
};

This has been super fast compared to the above mongoose version.

InsertFromArray.md

import { ParameterizedContext } from "koa";
import Router from "koa-router";
import mongoose, { Schema } from "mongoose";
import { performance, PerformanceObserver } from "perf_hooks";
const perQuery = 100;
const emptySchema = new Schema({}, { strict: false });
const orderModel = mongoose.model("oldorders", emptySchema);

const obs = new PerformanceObserver(items => {
    console.log(
        items.getEntries()[0]?.name,
        " _:_ ",
        items.getEntries()[0].duration,
    );
    // performance.clearMarks();
});
obs.observe({ entryTypes: ["measure"] });

export const initScript = async (
    ctx: ParameterizedContext<any, Router.IRouterParamContext<any, {}>>,
) => {
    performance.mark("FetchQueryStarts");
    const result = await orderModel
        .find({}, { _id: 0 })
        .sort({ _id: -1 })
        .skip(0 * perQuery)
        .limit(100)
        .exec();
    performance.mark("FetchQueryEnds");
    performance.measure("FetchFromDB", "FetchQueryStarts", "FetchQueryEnds");
    await createHalfMillionRecords(result);
    performance.measure("TotalTimeTaken", "FetchQueryStarts", "InsertArrayEnd");
    ctx.body = result;
};
export const createHalfMillionRecords = async (result: any) => {
    performance.mark("CreateArrayStarts");
    const toInsertOrders = new Array(500).fill(result);
    performance.mark("CreateArrayEnds");
    performance.measure("CreateArray", "CreateArrayStarts", "CreateArrayEnds");
    performance.mark("InsertArrayStarts");
    await orderModel.create(toInsertOrders);
    performance.mark("InsertArrayEnd");
    performance.measure(
        "Inserting in db",
        "InsertArrayStarts",
        "InsertArrayEnd",
    );
};