Last active
March 4, 2024 19:03
-
-
Save josephg/13efc1444660c07870fcbd0b3e917638 to your computer and use it in GitHub Desktop.
benchmark code from blog post
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Run with node --expose-gc bench.js ../automerge-paper.json.gz | |
// automerge-paper.json.gz from https://github.com/josephg/crdt-benchmarks | |
// Read in a patch file and check that the patches all apply correctly. | |
const fs = require('fs') | |
const assert = require('assert') | |
const zlib = require('zlib') | |
const automerge = require('automerge') | |
const v8 = require('v8') | |
// For automerge-rs | |
// const backend = require('automerge-backend-wasm') | |
// automerge.setDefaultBackend(backend) | |
const filename = process.argv[2] | |
if (filename == null) { | |
console.error(`Usage: $ ${process.argv.join(' ')} file.json[.gz]`) | |
process.exit(1) | |
} | |
//console.log('snapshot', v8.writeHeapSnapshot()) | |
gc() | |
console.log(v8.getHeapStatistics()) | |
const startMemory = v8.getHeapStatistics().used_heap_size | |
console.log('Processing with automerge version', require('./node_modules/automerge/package.json').version) | |
const { | |
startContent, | |
endContent, | |
txns | |
} = JSON.parse( | |
filename.endsWith('.gz') | |
? zlib.gunzipSync(fs.readFileSync(filename)) | |
: fs.readFileSync(filename, 'utf-8') | |
) | |
console.log('applying', txns.length, 'txns...') | |
console.time('apply') | |
let state = automerge.from({text: new automerge.Text("hi")}) | |
for (let i = 0; i < txns.length; i++) { | |
//if (i > 20000) break | |
if (i % 10000 == 0) console.log(i) | |
const {patches} = txns[i] | |
for (const [pos, delHere, insContent] of patches) { | |
// console.log(pos, delHere, insContent) | |
state = automerge.change(state, doc => { | |
if (delHere > 0) doc.text.deleteAt(pos, delHere) | |
if (insContent !== '') doc.text.insertAt(pos, insContent) | |
}) | |
} | |
} | |
console.timeEnd('apply') | |
gc() | |
console.log(v8.getHeapStatistics()) | |
//console.log('snapshot', v8.writeHeapSnapshot()) | |
// assert.strictEqual(state.text.toSpans().join(''), endContent) | |
console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory) | |
console.log(state.text.toSpans().length, 'spans') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Read in a patch file and check that the patches all apply correctly. | |
import fs from 'fs' | |
import assert from 'assert' | |
import zlib from 'zlib' | |
import v8 from 'v8' | |
import {Doc} from 'diamond-js' | |
const filename = process.argv[2] | |
if (filename == null) { | |
console.error(`Usage: $ ${process.argv.join(' ')} file.json[.gz]`) | |
process.exit(1) | |
} | |
// console.log(v8.getHeapStatistics()) | |
// console.log('snapshot', v8.writeHeapSnapshot()) | |
// gc() | |
// let startHeap = v8.getHeapStatistics().used_heap_size | |
const perf = [] | |
let start, lastTime | |
const mark = (i) => { | |
let now = performance.now() | |
perf.push({time: now - start, thisTime: now - lastTime, count: i}) | |
lastTime = now | |
} | |
const { | |
startContent, | |
endContent, | |
txns | |
} = JSON.parse( | |
filename.endsWith('.gz') | |
? zlib.gunzipSync(fs.readFileSync(filename)) | |
: fs.readFileSync(filename, 'utf-8') | |
) | |
console.log('applying', txns.length, 'txns...') | |
const run = () => { | |
perf.length = 0 | |
start = lastTime = performance.now() | |
const state = new Doc() | |
for (let i = 0; i < txns.length; i++) { | |
// if (i > 20000) break | |
// if (i % 10000 == 0) console.log(i) | |
if (i % 1000 == 0) mark(i) | |
const {patches} = txns[i] | |
for (const [pos, delHere, insContent] of patches) { | |
// console.log(pos, delHere, insContent) | |
if (delHere > 0) state.del(pos, delHere) | |
if (insContent !== '') state.ins(pos, insContent) | |
} | |
} | |
mark(txns.length) | |
// assert.strictEqual(state.get(), endContent) | |
// return state.len() | |
// console.log(JSON.stringify(state.get_txn_since([])).length) | |
} | |
const run2 = () => { | |
const state = new Doc() | |
for (let i = 0; i < 5000000; i++) { | |
state.ins(0, 'x') | |
} | |
assert.strictEqual(state.len(), 5000000) | |
} | |
// warm up | |
// for (let i = 0; i < 10; i++) run() | |
run() | |
run() | |
run() | |
run() | |
run() | |
run() | |
console.time('apply') | |
// const len = run2() | |
const len = run() | |
console.timeEnd('apply') | |
// gc() | |
// console.log(v8.getHeapStatistics()) | |
// console.log('heap', v8.getHeapStatistics().used_heap_size - startHeap) | |
// console.log('snapshot', v8.writeHeapSnapshot()) | |
// assert.strictEqual(state.text.toSpans().join(''), endContent) | |
// assert.strictEqual(doc.get(), endContent) | |
// console.log(len) | |
fs.writeFileSync('perf.json', JSON.stringify(perf)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Run with node --expose-gc bench.js ../automerge-paper.json.gz | |
// automerge-paper.json.gz from https://github.com/josephg/crdt-benchmarks | |
// Read in a patch file and check that the patches all apply correctly. | |
const fs = require('fs') | |
const assert = require('assert') | |
const zlib = require('zlib') | |
const Y = require('yjs') | |
const v8 = require('v8') | |
const filename = process.argv[2] | |
if (filename == null) { | |
console.error(`Usage: $ ${process.argv.join(' ')} file.json[.gz]`) | |
process.exit(1) | |
} | |
// console.log('snapshot', v8.writeHeapSnapshot()) | |
console.log(v8.getHeapStatistics()) | |
console.log('heap', process.memoryUsage().heapUsed) | |
const { | |
startContent, | |
endContent, | |
txns | |
} = JSON.parse( | |
filename.endsWith('.gz') | |
? zlib.gunzipSync(fs.readFileSync(filename)) | |
: fs.readFileSync(filename, 'utf-8') | |
) | |
const run = () => { | |
gc() | |
const startMemory = v8.getHeapStatistics().used_heap_size | |
const state = new Y.Doc() | |
for (let i = 0; i < txns.length; i++) { | |
// if (i > 20000) break | |
// if (i % 10000 == 0) console.log(i) | |
const {patches} = txns[i] | |
state.transact(txn => { | |
const text = txn.doc.getText() | |
for (const [pos, delHere, insContent] of patches) { | |
// console.log(pos, delHere, insContent) | |
if (delHere > 0) text.delete(pos, delHere) | |
if (insContent !== '') text.insert(pos, insContent) | |
// state = automerge.change(state, doc => { | |
// if (delHere > 0) doc.text.deleteAt(pos, delHere) | |
// if (insContent !== '') doc.text.insertAt(pos, insContent) | |
// }) | |
} | |
}) | |
} | |
gc() | |
console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory) | |
console.log(state.getText().length) | |
console.log(txns.length) | |
} | |
const run2 = () => { | |
// const state = new Doc() | |
const state = new Y.Doc() | |
const text = state.getText() | |
for (let i = 0; i < 5000000; i++) { | |
text.insert(0, 'x') | |
} | |
assert.strictEqual(text.length, 5000000) | |
} | |
console.log('applying', txns.length, 'txns...') | |
run() | |
run() | |
run() | |
console.time('apply') | |
run() | |
console.timeEnd('apply') | |
// gc() | |
console.log(v8.getHeapStatistics()) | |
console.log('heap', process.memoryUsage().heapUsed) | |
// console.log('snapshot', v8.writeHeapSnapshot()) | |
// assert.strictEqual(state.text.toSpans().join(''), endContent) | |
// assert.strictEqual(state.getText().toJSON(), endContent) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Again, run against automerge-paper.json.gz. Sorry about this code. No, I'm not sorry. Welcome to research. | |
const fs = require('fs') | |
const assert = require('assert') | |
const zlib = require('zlib') | |
const v8 = require('v8') | |
const filename = process.argv[2] | |
if (filename == null) { | |
console.error(`Usage: $ node check.js file.json[.gz]`) | |
process.exit(1) | |
} | |
const { | |
startContent, | |
endContent, | |
txns | |
} = JSON.parse( | |
filename.endsWith('.gz') | |
? zlib.gunzipSync(fs.readFileSync(filename)) | |
: fs.readFileSync(filename, 'utf-8') | |
) | |
console.log('applying', txns.length, 'txns...') | |
const x = () => { | |
let content = startContent | |
gc() | |
const startMemory = v8.getHeapStatistics().used_heap_size | |
console.time('apply') | |
let lastTime = 0 | |
for (let i = 0; i < txns.length; i++) { | |
const {time, patches} = txns[i] | |
for (const [pos, delHere, insContent] of patches) { | |
const before = content.slice(0, pos) | |
const after = content.slice(pos + delHere) | |
content = before + insContent + after | |
} | |
} | |
gc() | |
console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory) | |
console.timeEnd('apply') | |
console.log(txns.length) | |
assert.strictEqual(content, endContent) | |
} | |
x() | |
console.log(`Looking good - ${txns.length} apply cleanly.`) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Copy into the reference-crdts repository and run with node --loader ts-node/esm --expose-gc ref-crdt-bench.ts | |
import zlib from 'zlib' | |
import fs from 'fs' | |
import {Algorithm, newDoc, localDelete, yjsMod, automerge, getArray, sync9} from './crdts' | |
import assert from 'assert' | |
import v8 from 'v8' | |
const bench = (algName: string, alg: Algorithm) => { | |
// const filename = 'sveltecomponent' | |
const filename = 'automerge-paper' | |
const { | |
startContent, | |
endContent, | |
txns | |
} = JSON.parse(zlib.gunzipSync(fs.readFileSync(`../crdt-benchmarks/${filename}.json.gz`)).toString()) | |
console.time(`${algName} ${filename}`) | |
;(globalThis as any).gc() | |
const startMemory = v8.getHeapStatistics().used_heap_size | |
const doc = newDoc() | |
let i = 0 | |
for (const txn of txns) { | |
if (++i % 10000 === 0) console.log(i) | |
for (const patch of txn.patches) { | |
// Ignoring any deletes for now. | |
const [pos, delCount, inserted] = patch as [number, number, string] | |
if (inserted.length) { | |
alg.localInsert(doc, 'A', pos, inserted) | |
} else if (delCount) { | |
localDelete(doc, 'A', pos) | |
} | |
} | |
} | |
console.timeEnd(`${algName} ${filename}`) | |
;(globalThis as any).gc() | |
console.log('RAM used:', v8.getHeapStatistics().used_heap_size - startMemory) | |
assert.strictEqual(getArray(doc).join(''), endContent) | |
console.log(txns.length) | |
} | |
// bench('yjs mod', yjsMod) | |
bench('automerge', automerge) | |
// bench('sync9', sync9) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment