Skip to content

Instantly share code, notes, and snippets.

@vladar
Created Jul 7, 2021
Embed
What would you like to do?
Sorting data with lmdb-store
console.time(`lmdb-init`)
const { open } = require("lmdb-store")
const store = open({
name: `store`,
path: process.cwd() + `/test`,
})
const db = store.openDB({
name: `db`,
})
const tmp = store.openDB({
name: `tmp`,
noSync: true,
useWritemap: true,
dupSort: true,
encoding: "ordered-binary",
})
console.timeEnd(`lmdb-init`)
const ptop = require("process-top")()
async function generateData() {
db.transactionSync(() => {
db.clear()
})
const faker = require('faker');
console.time(`generateData`)
let promise
for (let i = 0; i < 1000000; i++) {
promise = db.put(i, faker.helpers.userCard())
if (i % 10000 === 0) {
console.log(`Created ${i} items`)
await new Promise(resolve => setTimeout(resolve, 10))
}
}
await promise
console.timeEnd(`generateData`)
}
function mem() {
if (global.gc) {
global.gc()
}
console.log(ptop.toString())
}
async function benchInMemory() {
// mem()
console.time(`load`)
const t = Array.from(db.getRange({ }))
console.timeEnd(`load`)
console.time(`sort in memory`)
t.sort(compare)
console.timeEnd(`sort in memory`)
console.time(`iterate`)
let l
let count = 0
for (const item of t) {
l = item
count++
if (count === 500000) {
// mem()
}
}
console.timeEnd(`iterate`)
mem()
console.log(``)
console.log(`length: `, t.length, `last: `, l)
}
async function benchLmdb() {
// mem()
console.time(`load+sort`)
let promise
for (const { value, key } of db.getRange()) {
promise = tmp.put(`1`, [value.name, key])
}
await promise
console.timeEnd(`load+sort`)
console.time(`iterate`)
let l
let count = 0
for (const item of tmp.getValues(`1`)) {
// l = db.get(item[1])
l = item
count++
if (count === 500000) {
// mem()
}
}
tmp.remove(`1`)
console.timeEnd(`iterate`)
mem()
console.log(``)
console.log(`length: `, count, `last: `, l)
}
function compare(a, b) {
if (a.value.name === b.value.name) return 0
return a.value.name > b.value.name ? 1 : -1
}
// 1st run: uncomment to generate the data
// generateData().catch(console.error)
// Uncomment to bench different scenarios
benchLmdb().catch(console.error)
// benchInMemory().catch(console.error)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment