Automerge Performance Branch

benchmarks/b1.js

@@ -1,9 +1,10 @@
 
 import * as Y from 'yjs'
-import { setBenchmarkResult, gen, N, benchmarkTime, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed } from './utils.js'
+import { setBenchmarkResult, gen, N, benchmarkTime, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed, computeAutomergeUpdateSize } from './utils.js'
 import * as prng from 'lib0/prng.js'
 import * as math from 'lib0/math.js'
 import * as t from 'lib0/testing.js'
+// @ts-ignore
 import Automerge from 'automerge'
 
 const benchmarkYjs = (id, inputData, changeFunction, check) => {
@@ -22,7 +23,13 @@ const benchmarkYjs = (id, inputData, changeFunction, check) => {
   })
   check(doc1, doc2)
   setBenchmarkResult('yjs', `${id} (avgUpdateSize)`, `${math.round(updateSize / inputData.length)} bytes`)
-  const encodedState = Y.encodeStateAsUpdateV2(doc1)
+  /**
+   * @type {any}
+   */
+  let encodedState
+  benchmarkTime('yjs', `${id} (encodeTime)`, () => {
+    encodedState = Y.encodeStateAsUpdateV2(doc1)
+  })
   const documentSize = encodedState.byteLength
   setBenchmarkResult('yjs', `${id} (docSize)`, `${documentSize} bytes`)
   benchmarkTime('yjs', `${id} (parseTime)`, () => {
@@ -40,23 +47,29 @@ const benchmarkAutomerge = (id, init, inputData, changeFunction, check) => {
   }
   const emptyDoc = Automerge.init()
   let doc1 = Automerge.change(emptyDoc, init)
-  let doc2 = Automerge.applyChanges(Automerge.init(), Automerge.getChanges(emptyDoc, doc1))
+  let doc2 = Automerge.applyChanges(Automerge.init(), Automerge.getAllChanges(doc1))
   let updateSize = 0
   benchmarkTime('automerge', `${id} (time)`, () => {
     for (let i = 0; i < inputData.length; i++) {
       const updatedDoc = Automerge.change(doc1, doc => {
         changeFunction(doc, inputData[i], i)
       })
-      const update = JSON.stringify(Automerge.getChanges(doc1, updatedDoc))
-      updateSize += update.length
-      doc2 = Automerge.applyChanges(doc2, JSON.parse(update))
+      const update = Automerge.getChanges(doc1, updatedDoc)
+      updateSize += computeAutomergeUpdateSize(update)
+      doc2 = Automerge.applyChanges(doc2, update)
       doc1 = updatedDoc
     }
   })
   check(doc1, doc2)
   setBenchmarkResult('automerge', `${id} (avgUpdateSize)`, `${math.round(updateSize / inputData.length)} bytes`)
-  const encodedState = Automerge.save(doc1)
-  const documentSize = encodedState.length
+  /**
+   * @type {any}
+   */
+  let encodedState
+  benchmarkTime('automerge', `${id} (encodeTime)`, () => {
+    encodedState = Automerge.save(doc1)
+  })
+  const documentSize = encodedState.byteLength
   setBenchmarkResult('automerge', `${id} (docSize)`, `${documentSize} bytes`)
   benchmarkTime('automerge', `${id} (parseTime)`, () => {
     Automerge.load(encodedState)

benchmarks/b2.js

@@ -1,9 +1,10 @@
 
 import * as Y from 'yjs'
-import { setBenchmarkResult, gen, N, benchmarkTime, cpy, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed } from './utils.js'
+import { setBenchmarkResult, gen, N, benchmarkTime, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed, computeAutomergeUpdateSize } from './utils.js'
 import * as prng from 'lib0/prng.js'
 import * as math from 'lib0/math.js'
 import * as t from 'lib0/testing.js'
+// @ts-ignore
 import Automerge from 'automerge'
 
 const initText = prng.word(gen, 100, 100)
@@ -44,7 +45,13 @@ const benchmarkYjs = (id, changeDoc1, changeDoc2, check) => {
   })
   check(doc1, doc2)
   setBenchmarkResult('yjs', `${id} (updateSize)`, `${math.round(update1To2.byteLength + update2To1.byteLength)} bytes`)
-  const encodedState = Y.encodeStateAsUpdateV2(doc1)
+  /**
+   * @type {any}
+   */
+  let encodedState
+  benchmarkTime('yjs', `${id} (encodeTime)`, () => {
+    encodedState = Y.encodeStateAsUpdateV2(doc1)
+  })
   const documentSize = encodedState.byteLength
   setBenchmarkResult('yjs', `${id} (docSize)`, `${documentSize} bytes`)
   benchmarkTime('yjs', `${id} (parseTime)`, () => {
@@ -65,20 +72,26 @@ const benchmarkAutomerge = (id, changeDoc1, changeDoc2, check) => {
     doc.text = new Automerge.Text()
     doc.text.insertAt(0, ...initText)
   })
-  let doc2 = Automerge.applyChanges(Automerge.init(), cpy(Automerge.getChanges(emptyDoc, doc1)))
+  let doc2 = Automerge.applyChanges(Automerge.init(), Automerge.getAllChanges(doc1))
   let updateSize = 0
   benchmarkTime('automerge', `${id} (time)`, () => {
     const updatedDoc1 = Automerge.change(doc1, changeDoc1)
     const updatedDoc2 = Automerge.change(doc2, changeDoc2)
-    const update2 = JSON.stringify(Automerge.getChanges(doc1, updatedDoc1))
-    const update1 = JSON.stringify(Automerge.getChanges(doc2, updatedDoc2))
-    updateSize += update1.length + update2.length
-    doc2 = Automerge.applyChanges(updatedDoc2, JSON.parse(update2))
-    doc1 = Automerge.applyChanges(updatedDoc1, JSON.parse(update1))
+    const update2 = Automerge.getChanges(doc1, updatedDoc1)
+    const update1 = Automerge.getChanges(doc2, updatedDoc2)
+    updateSize += computeAutomergeUpdateSize(update1) + computeAutomergeUpdateSize(update2)
+    doc2 = Automerge.applyChanges(updatedDoc2, update2)
+    doc1 = Automerge.applyChanges(updatedDoc1, update1)
   })
   check(doc1, doc2)
   setBenchmarkResult('automerge', `${id} (updateSize)`, `${math.round(updateSize)} bytes`)
-  const encodedState = Automerge.save(doc1)
+  /**
+   * @type {any}
+   */
+  let encodedState
+  benchmarkTime('automerge', `${id} (encodeTime)`, () => {
+    encodedState = Automerge.save(doc1)
+  })
   const documentSize = encodedState.length
   setBenchmarkResult('automerge', `${id} (docSize)`, `${documentSize} bytes`)
   benchmarkTime('automerge', `${id} (parseTime)`, () => {

benchmarks/b3.js

@@ -1,8 +1,9 @@
 
 import * as Y from 'yjs'
-import { setBenchmarkResult, benchmarkTime, N, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed } from './utils.js'
+import { setBenchmarkResult, benchmarkTime, N, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed, computeAutomergeUpdateSize } from './utils.js'
 import * as t from 'lib0/testing.js'
 import * as math from 'lib0/math.js'
+// @ts-ignore
 import Automerge from 'automerge'
 
 const sqrtN = math.floor(Math.sqrt(N))
@@ -35,7 +36,13 @@ const benchmarkYjs = (id, changeDoc, check) => {
   t.assert(updates.length === sqrtN)
   check(docs.slice(0, 2))
   setBenchmarkResult('yjs', `${id} (updateSize)`, `${updates.reduce((len, update) => len + update.byteLength, 0)} bytes`)
-  const encodedState = Y.encodeStateAsUpdateV2(docs[0])
+  /**
+   * @type {any}
+   */
+  let encodedState
+  benchmarkTime('yjs', `${id} (encodeTime)`, () => {
+    encodedState = Y.encodeStateAsUpdateV2(docs[0])
+  })
   const documentSize = encodedState.byteLength
   setBenchmarkResult('yjs', `${id} (docSize)`, `${documentSize} bytes`)
   benchmarkTime('yjs', `${id} (parseTime)`, () => {
@@ -55,31 +62,37 @@ const benchmarkAutomerge = (id, init, changeDoc, check) => {
   for (let i = 0; i < sqrtN; i++) {
     docs.push(Automerge.init())
   }
-  const initDoc = Automerge.change(docs[0], init)
-  const initUpdate = JSON.stringify(Automerge.getChanges(docs[0], initDoc))
-  for (let i = 0; i < docs.length; i++) {
-    docs[i] = Automerge.applyChanges(docs[i], JSON.parse(initUpdate))
+  docs[0] = Automerge.change(docs[0], init)
+  const initUpdate = Automerge.getAllChanges(docs[0])
+  for (let i = 1; i < docs.length; i++) {
+    docs[i] = Automerge.applyChanges(docs[i], initUpdate)
   }
   const updates = []
   for (let i = 0; i < docs.length; i++) {
     const doc = docs[i]
     const updatedDoc = Automerge.change(doc, d => { changeDoc(d, i) })
-    const update = JSON.stringify(Automerge.getChanges(doc, updatedDoc))
+    const update = Automerge.getChanges(doc, updatedDoc)
     updates.push(update)
     docs[i] = updatedDoc
   }
   for (let i = 0; i < updates.length; i++) {
-    docs[0] = Automerge.applyChanges(docs[0], JSON.parse(updates[i]))
+    docs[0] = Automerge.applyChanges(docs[0], updates[i])
   }
   benchmarkTime('automerge', `${id} (time)`, () => {
     for (let i = 0; i < updates.length; i++) {
-      docs[1] = Automerge.applyChanges(docs[1], JSON.parse(updates[i]))
+      docs[1] = Automerge.applyChanges(docs[1], updates[i])
     }
   })
   check(docs.slice(0, 2))
-  setBenchmarkResult('automerge', `${id} (updateSize)`, `${updates.reduce((len, update) => len + update.length, 0)} bytes`)
-  const encodedState = Automerge.save(docs[0])
-  const documentSize = encodedState.length
+  setBenchmarkResult('automerge', `${id} (updateSize)`, `${updates.reduce((len, update) => len + computeAutomergeUpdateSize(update), 0)} bytes`)
+  /**
+   * @type {any}
+   */
+  let encodedState
+  benchmarkTime('automerge', `${id} (encodeTime)`, () => {
+    encodedState = Automerge.save(docs[0])
+  })
+  const documentSize = encodedState.byteLength
   setBenchmarkResult('automerge', `${id} (docSize)`, `${documentSize} bytes`)
   benchmarkTime('automerge', `${id} (parseTime)`, () => {
     Automerge.load(encodedState)

benchmarks/b4.js

@@ -2,6 +2,8 @@ import * as Y from 'yjs'
 import { setBenchmarkResult, N, benchmarkTime, disableAutomergeBenchmarks, logMemoryUsed, getMemUsed, tryGc } from './utils.js'
 import * as math from 'lib0/math.js'
 import * as t from 'lib0/testing.js'
+import * as time from 'lib0/time.js'
+// @ts-ignore
 import Automerge from 'automerge'
 // @ts-ignore
 import { edits, finalText } from './b4-editing-trace.js'
@@ -21,7 +23,7 @@ const benchmarkYjs = (id, inputData, changeFunction, check) => {
       }
     })
     check(doc1)
-    setBenchmarkResult('yjs', `${id} (updateSize)`, `${math.round(updateSize)} bytes`)
+    setBenchmarkResult('yjs', `${id} (avgUpdateSize)`, `${math.round(updateSize / inputData.length)} bytes`)
     /**
      * @type {any}
      */
@@ -59,24 +61,33 @@ const benchmarkAutomerge = (id, init, inputData, changeFunction, check) => {
     // containing all the edits from b4.
     const emptyDoc = Automerge.init()
     let doc1 = Automerge.change(emptyDoc, init)
-    let updateSize = 0
+    let lastUpdate = time.getUnixTime()
     benchmarkTime('automerge', `${id} (time)`, () => {
+      const percentageSteps = math.floor(inputData.length / 20)
       for (let i = 0; i < inputData.length; i++) {
-        const updatedDoc = Automerge.change(doc1, doc => {
+        if (i % percentageSteps === 0) {
+          const nextTime = time.getUnixTime()
+          console.log(`progress: ${math.round(100 * i / inputData.length)}%  - last update ${nextTime - lastUpdate} ms ago`)
+          lastUpdate = nextTime
+        }
+        doc1 = Automerge.change(doc1, doc => {
           changeFunction(doc, inputData[i], i)
         })
-        const update = JSON.stringify(Automerge.getChanges(doc1, updatedDoc))
-        updateSize += update.length
-        doc1 = updatedDoc
+        // const update = Automerge.getChanges(doc1, updatedDoc)
+        // updateSize += computeAutomergeUpdateSize(update)
+        // doc1 = updatedDoc
       }
+      console.log('Almost done.. computing the updated automerge document could take a while..')
     })
     check(doc1)
-    setBenchmarkResult('automerge', `${id} (updateSize)`, `${math.round(updateSize)} bytes`)
+    // We perform all changes in a single Automerge transaction because
+    // currently the time to compute updates grows exponentially with each change.
+    // setBenchmarkResult('automerge', `${id} (avgUpdateSize)`, `${math.ceil(updateSize / inputData.length)} bytes`)
+    setBenchmarkResult('automerge', `${id} (avgUpdateSize)`, `-`)
     benchmarkTime('automerge', `${id} (encodeTime)`, () => {
       encodedState = Automerge.save(doc1)
     })
-    const documentSize = encodedState.length
-    setBenchmarkResult('automerge', `${id} (docSize)`, `${documentSize} bytes`)
+    setBenchmarkResult('automerge', `${id} (docSize)`, `${encodedState.byteLength} bytes`)
   })()
   ;(() => {
     const startHeapUsed = getMemUsed()

benchmarks/run.js

@@ -1,7 +1,5 @@
 import './bundle.js'
-import './b1.js'
-import './b2.js'
-import './b3.js'
+
 import './b4.js'
 import { benchmarkResults, N } from './utils.js'
 

benchmarks/utils.js

@@ -28,6 +28,8 @@ export const benchmarkTime = (libname, id, f) => {
  */
 export const gen = prng.create(42)
 
+export const computeAutomergeUpdateSize = updates => updates.reduce((len, update) => update.byteLength, 0)
+
 export const cpy = o => JSON.parse(JSON.stringify(o))
 
 export const getMemUsed = () => {

package.json

@@ -29,7 +29,7 @@
   },
   "homepage": "https://github.com/dmonad/crdt-benchmarks#readme",
   "dependencies": {
-    "automerge": "^0.14.1",
+    "automerge": "github:automerge/automerge#edc859c0970a57adcebb7dc6009adb1e03a3ab00",
     "lib0": "^0.2.32",
     "rollup": "^1.32.1",
     "rollup-plugin-commonjs": "^8.3.4",