benchmark code from blog post

bench-automerge.js

// Run with node --expose-gc bench.js ../automerge-paper.json.gz
// automerge-paper.json.gz from https://github.com/josephg/crdt-benchmarks

// Read in a patch file and check that the patches all apply correctly.
const fs = require('fs')
const assert = require('assert')
const zlib = require('zlib')
const automerge = require('automerge')
const v8 = require('v8')

// For automerge-rs
// const backend = require('automerge-backend-wasm')
// automerge.setDefaultBackend(backend)

const filename = process.argv[2]

if (filename == null) {
  console.error(`Usage: $ ${process.argv.join(' ')} file.json[.gz]`)
  process.exit(1)
}

//console.log('snapshot', v8.writeHeapSnapshot())
gc()
console.log(v8.getHeapStatistics())
const startMemory = v8.getHeapStatistics().used_heap_size

console.log('Processing with automerge version', require('./node_modules/automerge/package.json').version)

const {
  startContent,
  endContent,
  txns
} = JSON.parse(
  filename.endsWith('.gz')
  ? zlib.gunzipSync(fs.readFileSync(filename))
  : fs.readFileSync(filename, 'utf-8')
)

console.log('applying', txns.length, 'txns...')

console.time('apply')

let state = automerge.from({text: new automerge.Text("hi")})
for (let i = 0; i < txns.length; i++) {
  //if (i > 20000) break
  if (i % 10000 == 0) console.log(i)
  const {patches} = txns[i]

  for (const [pos, delHere, insContent] of patches) {
    // console.log(pos, delHere, insContent)
    state = automerge.change(state, doc => {
      if (delHere > 0) doc.text.deleteAt(pos, delHere)
      if (insContent !== '') doc.text.insertAt(pos, insContent)
    })
  }
}
console.timeEnd('apply')
gc()
console.log(v8.getHeapStatistics())
//console.log('snapshot', v8.writeHeapSnapshot())
// assert.strictEqual(state.text.toSpans().join(''), endContent)
console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory)
console.log(state.text.toSpans().length, 'spans')

bench-diamond.js

// Read in a patch file and check that the patches all apply correctly.
import fs from 'fs'
import assert from 'assert'
import zlib from 'zlib'
import v8 from 'v8'
import {Doc} from 'diamond-js'

const filename = process.argv[2]

if (filename == null) {
  console.error(`Usage: $ ${process.argv.join(' ')} file.json[.gz]`)
  process.exit(1)
}

// console.log(v8.getHeapStatistics())
// console.log('snapshot', v8.writeHeapSnapshot())
// gc()
// let startHeap = v8.getHeapStatistics().used_heap_size

const perf = []
let start, lastTime

const mark = (i) => {
  let now = performance.now()
  perf.push({time: now - start, thisTime: now - lastTime, count: i})
  lastTime = now
}

const {
  startContent,
  endContent,
  txns
} = JSON.parse(
  filename.endsWith('.gz')
  ? zlib.gunzipSync(fs.readFileSync(filename))
  : fs.readFileSync(filename, 'utf-8')
)

console.log('applying', txns.length, 'txns...')

const run = () => {
  perf.length = 0
  start = lastTime = performance.now()

  const state = new Doc()

  for (let i = 0; i < txns.length; i++) {
    // if (i > 20000) break
    // if (i % 10000 == 0) console.log(i)
    if (i % 1000 == 0) mark(i)
    const {patches} = txns[i]

    for (const [pos, delHere, insContent] of patches) {
      // console.log(pos, delHere, insContent)
      if (delHere > 0) state.del(pos, delHere)
      if (insContent !== '') state.ins(pos, insContent)
    }
  }

  mark(txns.length)
  // assert.strictEqual(state.get(), endContent)

  // return state.len()

  // console.log(JSON.stringify(state.get_txn_since([])).length)
}


const run2 = () => {
  const state = new Doc()

  for (let i = 0; i < 5000000; i++) {
    state.ins(0, 'x')
  }
  assert.strictEqual(state.len(), 5000000)
}
// warm up
// for (let i = 0; i < 10; i++) run()

run()
run()
run()
run()
run()
run()
console.time('apply')
// const len = run2()
const len = run()
console.timeEnd('apply')
// gc()
// console.log(v8.getHeapStatistics())
// console.log('heap', v8.getHeapStatistics().used_heap_size - startHeap)
// console.log('snapshot', v8.writeHeapSnapshot())
// assert.strictEqual(state.text.toSpans().join(''), endContent)

// assert.strictEqual(doc.get(), endContent)
// console.log(len)

fs.writeFileSync('perf.json', JSON.stringify(perf))

bench-yjs.js

// Run with node --expose-gc bench.js ../automerge-paper.json.gz
// automerge-paper.json.gz from https://github.com/josephg/crdt-benchmarks

// Read in a patch file and check that the patches all apply correctly.
const fs = require('fs')
const assert = require('assert')
const zlib = require('zlib')
const Y = require('yjs')
const v8 = require('v8')

const filename = process.argv[2]

if (filename == null) {
  console.error(`Usage: $ ${process.argv.join(' ')} file.json[.gz]`)
  process.exit(1)
}

// console.log('snapshot', v8.writeHeapSnapshot())
console.log(v8.getHeapStatistics())
console.log('heap', process.memoryUsage().heapUsed)

const {
  startContent,
  endContent,
  txns
} = JSON.parse(
  filename.endsWith('.gz')
  ? zlib.gunzipSync(fs.readFileSync(filename))
  : fs.readFileSync(filename, 'utf-8')
)


const run = () => {
  gc()
  const startMemory = v8.getHeapStatistics().used_heap_size
  const state = new Y.Doc()

  for (let i = 0; i < txns.length; i++) {
    // if (i > 20000) break
    // if (i % 10000 == 0) console.log(i)
    const {patches} = txns[i]

    state.transact(txn => {
      const text = txn.doc.getText()
      for (const [pos, delHere, insContent] of patches) {
        // console.log(pos, delHere, insContent)
        if (delHere > 0) text.delete(pos, delHere)
        if (insContent !== '') text.insert(pos, insContent)
        // state = automerge.change(state, doc => {
          //   if (delHere > 0) doc.text.deleteAt(pos, delHere)
          //   if (insContent !== '') doc.text.insertAt(pos, insContent)
          // })
      }
    })
  }
  gc()
  console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory)
  console.log(state.getText().length)
  console.log(txns.length)
}

const run2 = () => {
  // const state = new Doc()
  const state = new Y.Doc()
  const text = state.getText()

  for (let i = 0; i < 5000000; i++) {
    text.insert(0, 'x')
  }
  assert.strictEqual(text.length, 5000000)
}

console.log('applying', txns.length, 'txns...')

run()
run()
run()
console.time('apply')
run()
console.timeEnd('apply')
// gc()
console.log(v8.getHeapStatistics())
console.log('heap', process.memoryUsage().heapUsed)
// console.log('snapshot', v8.writeHeapSnapshot())
// assert.strictEqual(state.text.toSpans().join(''), endContent)

// assert.strictEqual(state.getText().toJSON(), endContent)

js_baseline.js

// Again, run against automerge-paper.json.gz. Sorry about this code. No, I'm not sorry. Welcome to research.

const fs = require('fs')
const assert = require('assert')
const zlib = require('zlib')
const v8 = require('v8')

const filename = process.argv[2]

if (filename == null) {
  console.error(`Usage: $ node check.js file.json[.gz]`)
  process.exit(1)
}

const {
  startContent,
  endContent,
  txns
} = JSON.parse(
  filename.endsWith('.gz')
  ? zlib.gunzipSync(fs.readFileSync(filename))
  : fs.readFileSync(filename, 'utf-8')
)


console.log('applying', txns.length, 'txns...')
const x = () => {
  let content = startContent
  gc()
  const startMemory = v8.getHeapStatistics().used_heap_size

  console.time('apply')
  let lastTime = 0
  for (let i = 0; i < txns.length; i++) {
    const {time, patches} = txns[i]

    for (const [pos, delHere, insContent] of patches) {
      const before = content.slice(0, pos)
      const after = content.slice(pos + delHere)
      content = before + insContent + after
    }
  }
  gc()
  console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory)
  console.timeEnd('apply')
  console.log(txns.length)
  assert.strictEqual(content, endContent)
}

x()

console.log(`Looking good - ${txns.length} apply cleanly.`)

ref-crdt-bench.ts

// Copy into the reference-crdts repository and run with node --loader ts-node/esm --expose-gc ref-crdt-bench.ts

import zlib from 'zlib'
import fs from 'fs'
import {Algorithm, newDoc, localDelete, yjsMod, automerge, getArray, sync9} from './crdts'
import assert from 'assert'
import v8 from 'v8'

const bench = (algName: string, alg: Algorithm) => {
  // const filename = 'sveltecomponent'
  const filename = 'automerge-paper'
  const {
    startContent,
    endContent,
    txns
  } = JSON.parse(zlib.gunzipSync(fs.readFileSync(`../crdt-benchmarks/${filename}.json.gz`)).toString())

  console.time(`${algName} ${filename}`)

  ;(globalThis as any).gc()
  const startMemory = v8.getHeapStatistics().used_heap_size

  const doc = newDoc()

  let i = 0
  for (const txn of txns) {
    if (++i % 10000 === 0) console.log(i)
    for (const patch of txn.patches) {
      // Ignoring any deletes for now.
      const [pos, delCount, inserted] = patch as [number, number, string]
      if (inserted.length) {
        alg.localInsert(doc, 'A', pos, inserted)
      } else if (delCount) {
        localDelete(doc, 'A', pos)
      }
    }
  }
  console.timeEnd(`${algName} ${filename}`)

  ;(globalThis as any).gc()
  console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory)
  assert.strictEqual(getArray(doc).join(''), endContent)
  console.log(txns.length)
}

// bench('yjs mod', yjsMod)
bench('automerge', automerge)
// bench('sync9', sync9)