Lunr Test

main.mjs

import { readFileSync, writeFileSync } from 'fs'
import { brotliCompressSync, gzipSync } from 'zlib'
import hf from 'human-format'

import lunr from 'lunr'

// https://gutenberg.org/ebooks/6130
const file = readFileSync('./6130-0.txt', 'utf8')
const opts = {
  scale: 'binary',
  unit: 'B',
}

const data = [['Pieces', 'Size', 'Index', 'GZip', 'BR', 'HSize', 'HIndex', 'HGZip', 'HBr']]

for (const pieces of [1, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000]) {
  // for (const pieces of [1, 5, 10, 20, 50, 100, 200, 500, 2000, 3000, 5000]) {
  // const size = Math.floor(file.length / pieces)
  const size = 10000
  const hSize = hf(Buffer.from(file.slice(0, size)).length)
  console.log('Pieces', pieces)
  console.log('Piece Size', hSize)
  const idx = lunr(function () {
    this.ref('id')
    this.field('text')

    for (let i = 0; i < pieces; i++) {
      const position = (i * size) % file.length
      this.add({
        id: Math.random().toString(36),
        text: file.slice(position, position + size),
      })
    }
  })

  const prebuilt = JSON.stringify(idx)
  const bytes = Buffer.from(prebuilt)
  const gz = gzipSync(bytes, { level: 9 })
  const br = brotliCompressSync(bytes)

  const hIndex = hf(bytes.length, opts)
  const hGz = hf(gz.length, opts)
  const hBr = hf(br.length, opts)
  console.log(`Normal:\t${hIndex}`)
  console.log(`GZip:\t${hGz}`)
  console.log(`Brotli:\t${hBr}`)

  data.push([pieces, size, bytes.length, gz.length, br.length, hSize, hIndex, hGz, hBr])
}

writeFileSync('./data.csv', data.map((row) => row.join(',')).join('\n'))

results.csv