Sorting data with lmdb-store

bench.js

console.time(`lmdb-init`)
const { open } = require("lmdb-store")

const store = open({
  name: `store`,
  path: process.cwd() + `/test`,
})
const db = store.openDB({
  name: `db`,
})
const tmp = store.openDB({
  name: `tmp`,
  noSync: true,
  useWritemap: true,
  dupSort: true,
  encoding: "ordered-binary",
})
console.timeEnd(`lmdb-init`)

const ptop = require("process-top")()

async function generateData() {
  db.transactionSync(() => {
    db.clear()
  })
  const faker = require('faker');

  console.time(`generateData`)
  let promise
  for (let i = 0; i < 1000000; i++) {
    promise = db.put(i, faker.helpers.userCard())
    if (i % 10000 === 0) {
      console.log(`Created ${i} items`)
      await new Promise(resolve => setTimeout(resolve, 10))
    }
  }
  await promise
  console.timeEnd(`generateData`)
}

function mem() {
  if (global.gc) {
    global.gc()
  }
  console.log(ptop.toString())
}

async function benchInMemory() {
  // mem()
  console.time(`load`)
  const t = Array.from(db.getRange({ }))
  console.timeEnd(`load`)

  console.time(`sort in memory`)
  t.sort(compare)
  console.timeEnd(`sort in memory`)

  console.time(`iterate`)
  let l
  let count = 0
  for (const item of t) {
    l = item
    count++
    if (count === 500000) {
      // mem()
    }
  }
  console.timeEnd(`iterate`)
  mem()
  console.log(``)
  console.log(`length: `, t.length, `last: `, l)
}

async function benchLmdb() {
  // mem()
  console.time(`load+sort`)
  let promise
  for (const { value, key } of db.getRange()) {
    promise = tmp.put(`1`, [value.name, key])
  }
  await promise
  console.timeEnd(`load+sort`)
  console.time(`iterate`)
  let l
  let count = 0
  for (const item of tmp.getValues(`1`)) {
    // l = db.get(item[1])
    l = item
    count++
    if (count === 500000) {
      // mem()
    }
  }
  tmp.remove(`1`)
  console.timeEnd(`iterate`)
  mem()
  console.log(``)
  console.log(`length: `, count, `last: `, l)
}

function compare(a, b) {
  if (a.value.name === b.value.name) return 0
  return a.value.name > b.value.name ? 1 : -1
}

// 1st run: uncomment to generate the data
// generateData().catch(console.error)

// Uncomment to bench different scenarios
benchLmdb().catch(console.error)
// benchInMemory().catch(console.error)