implement diffUpdates with tests - #263

src/index.js

@@ -57,14 +57,12 @@ export {
   encodeStateAsUpdate,
   encodeStateAsUpdateV2,
   encodeStateVector,
-  encodeStateVectorV2,
   UndoManager,
   decodeSnapshot,
   encodeSnapshot,
   decodeSnapshotV2,
   encodeSnapshotV2,
   decodeStateVector,
-  decodeStateVectorV2,
   logUpdate,
   logUpdateV2,
   isDeleted,
@@ -82,5 +80,7 @@ export {
   encodeStateVectorFromUpdate,
   encodeStateVectorFromUpdateV2,
   encodeRelativePosition,
-  decodeRelativePosition
+  decodeRelativePosition,
+  diffUpdate,
+  diffUpdateV2
 } from './internals.js'

src/structs/Skip.js

@@ -4,6 +4,7 @@ import {
   UpdateEncoderV1, UpdateEncoderV2, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 import * as error from 'lib0/error.js'
+import * as encoding from 'lib0/encoding.js'
 
 export const structSkipRefNumber = 10
 
@@ -44,7 +45,8 @@ export class Skip extends AbstractStruct {
    */
   write (encoder, offset) {
     encoder.writeInfo(structSkipRefNumber)
-    encoder.writeLen(this.length - offset)
+    // write as VarUint because Skips can't make use of predictable length-encoding
+    encoding.writeVarUint(encoder.restEncoder, this.length - offset)
   }
 
   /**

src/utils/encoding.js

@@ -29,14 +29,13 @@ import {
   UpdateDecoderV2,
   UpdateEncoderV1,
   UpdateEncoderV2,
-  DSDecoderV2,
   DSEncoderV2,
   DSDecoderV1,
   DSEncoderV1,
   mergeUpdatesV2,
   Skip,
-  diffUpdate,
+  diffUpdateV2,
-  Doc, Transaction, GC, Item, StructStore // eslint-disable-line
+  DSDecoderV2, Doc, Transaction, GC, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -136,7 +135,7 @@ export const readClientsStructRefs = (decoder, doc) => {
         }
         case 10: { // Skip Struct (nothing to apply)
           // @todo we could reduce the amount of checks by adding Skip struct to clientRefs so we know that something is missing.
-          const len = decoder.readLen()
+          const len = decoding.readVarUint(decoder.restDecoder)
           refs[i] = new Skip(createID(client, clock), len)
           clock += len
           break
@@ -517,8 +516,8 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map())
  *
  * @function
  */
-export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => {
+export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector = new Uint8Array([0]), encoder = new UpdateEncoderV2()) => {
-  const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector)
+  const targetStateVector = decodeStateVector(encodedTargetStateVector)
   writeStateAsUpdate(encoder, doc, targetStateVector)
   const updates = [encoder.toUint8Array()]
   // also add the pending updates (if there are any)
@@ -528,7 +527,7 @@ export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = n
       updates.push(doc.store.pendingDs)
     }
     if (doc.store.pendingStructs) {
-      updates.push(diffUpdate(doc.store.pendingStructs.update, encodedTargetStateVector))
+      updates.push(diffUpdateV2(doc.store.pendingStructs.update, encodedTargetStateVector))
     }
     if (updates.length > 1) {
       return mergeUpdatesV2(updates)
@@ -578,7 +577,7 @@ export const readStateVector = decoder => {
  *
  * @function
  */
-export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
+// export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
 
 /**
  * Read decodedState and return State as Map.
@@ -615,21 +614,25 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod
 /**
  * Encode State as Uint8Array.
  *
- * @param {Doc} doc
+ * @param {Doc|Map<number,number>} doc
  * @param {DSEncoderV1 | DSEncoderV2} [encoder]
  * @return {Uint8Array}
  *
  * @function
  */
 export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => {
-  writeDocumentStateVector(encoder, doc)
+  if (doc instanceof Map) {
+    writeStateVector(encoder, doc)
+  } else {
+    writeDocumentStateVector(encoder, doc)
+  }
   return encoder.toUint8Array()
 }
 
 /**
  * Encode State as Uint8Array.
  *
- * @param {Doc} doc
+ * @param {Doc|Map<number,number>} doc
  * @return {Uint8Array}
  *
  * @function

src/utils/updates.js

@@ -3,6 +3,7 @@ import * as binary from 'lib0/binary.js'
 import * as decoding from 'lib0/decoding.js'
 import * as encoding from 'lib0/encoding.js'
 import * as logging from 'lib0/logging.js'
+import * as math from 'lib0/math.js'
 import {
   createID,
   readItemContent,
@@ -12,6 +13,7 @@ import {
   mergeDeleteSets,
   DSEncoderV1,
   DSEncoderV2,
+  decodeStateVector,
   Item, GC, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2 // eslint-disable-line
 } from '../internals.js'
 
@@ -28,7 +30,7 @@ function * lazyStructReaderGenerator (decoder) {
       const info = decoder.readInfo()
       // @todo use switch instead of ifs
       if (info === 10) {
-        const len = decoder.readLen()
+        const len = decoding.readVarUint(decoder.restDecoder)
         yield new Skip(createID(client, clock), len)
         clock += len
       } else if ((binary.BITS5 & info) !== 0) {
@@ -62,25 +64,27 @@ function * lazyStructReaderGenerator (decoder) {
 export class LazyStructReader {
   /**
    * @param {UpdateDecoderV1 | UpdateDecoderV2} decoder
+   * @param {boolean} filterSkips
    */
-  constructor (decoder) {
+  constructor (decoder, filterSkips) {
     this.gen = lazyStructReaderGenerator(decoder)
     /**
-     * @type {null | Item | GC}
+     * @type {null | Item | Skip | GC}
      */
     this.curr = null
     this.done = false
+    this.filterSkips = filterSkips
     this.next()
   }
 
   /**
-   * @return {Item | GC | null}
+   * @return {Item | GC | Skip |null}
    */
   next () {
     // ignore "Skip" structs
     do {
       this.curr = this.gen.next().value || null
-    } while (this.curr !== null && this.curr.constructor === Skip)
+    } while (this.filterSkips && this.curr !== null && this.curr.constructor === Skip)
     return this.curr
   }
 }
@@ -99,7 +103,7 @@ export const logUpdate = update => logUpdateV2(update, UpdateDecoderV1)
 export const logUpdateV2 = (update, YDecoder = UpdateDecoderV2) => {
   const structs = []
   const updateDecoder = new YDecoder(decoding.createDecoder(update))
-  const lazyDecoder = new LazyStructReader(updateDecoder)
+  const lazyDecoder = new LazyStructReader(updateDecoder, false)
   for (let curr = lazyDecoder.curr; curr !== null; curr = lazyDecoder.next()) {
     structs.push(curr)
   }
@@ -145,7 +149,7 @@ export const mergeUpdates = updates => mergeUpdatesV2(updates, UpdateDecoderV1,
  */
 export const encodeStateVectorFromUpdateV2 = (update, YEncoder = DSEncoderV2, YDecoder = UpdateDecoderV2) => {
   const encoder = new YEncoder()
-  const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)))
+  const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), true)
   let curr = updateDecoder.curr
   if (curr !== null) {
     let size = 1
@@ -204,7 +208,7 @@ export const parseUpdateMetaV2 = (update, YDecoder = UpdateDecoderV2) => {
    * @type {Map<number, number>}
    */
   const to = new Map()
-  const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)))
+  const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), false)
   let curr = updateDecoder.curr
   if (curr !== null) {
     let currClient = curr.id.client
@@ -277,7 +281,7 @@ const sliceStruct = (left, diff) => {
  */
 export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => {
   const updateDecoders = updates.map(update => new YDecoder(decoding.createDecoder(update)))
-  let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder))
+  let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder, true))
 
   /**
    * @todo we don't need offset because we always slice before
@@ -395,13 +399,52 @@ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = U
 
 /**
  * @param {Uint8Array} update
- * @param {Uint8Array} [sv]
+ * @param {Uint8Array} sv
+ * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder]
+ * @param {typeof UpdateEncoderV1 | typeof UpdateEncoderV2} [YEncoder]
  */
-export const diffUpdate = (update, sv = new Uint8Array([0])) => {
+export const diffUpdateV2 = (update, sv, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => {
-  // @todo!!!
+  const state = decodeStateVector(sv)
-  return update
+  const encoder = new YEncoder()
+  const lazyStructWriter = new LazyStructWriter(encoder)
+  const decoder = new YDecoder(decoding.createDecoder(update))
+  const reader = new LazyStructReader(decoder, false)
+  while (reader.curr) {
+    const curr = reader.curr
+    const currClient = curr.id.client
+    const svClock = state.get(currClient) || 0
+    if (reader.curr.constructor === Skip) {
+      // the first written struct shouldn't be a skip
+      reader.next()
+      continue
+    }
+    if (curr.id.clock + curr.length > svClock) {
+      writeStructToLazyStructWriter(lazyStructWriter, curr, math.max(svClock - curr.id.clock, 0))
+      reader.next()
+      while (reader.curr && reader.curr.id.client === currClient) {
+        writeStructToLazyStructWriter(lazyStructWriter, reader.curr, 0)
+        reader.next()
+      }
+    } else {
+      // read until something new comes up
+      while (reader.curr && reader.curr.id.client === currClient && reader.curr.id.clock + reader.curr.length <= svClock) {
+        reader.next()
+      }
+    }
+  }
+  finishLazyStructWriting(lazyStructWriter)
+  // write ds
+  const ds = readDeleteSet(decoder)
+  writeDeleteSet(encoder, ds)
+  return encoder.toUint8Array()
 }
 
+/**
+ * @param {Uint8Array} update
+ * @param {Uint8Array} sv
+ */
+export const diffUpdate = (update, sv) => diffUpdateV2(update, sv, UpdateDecoderV1, UpdateEncoderV1)
+
 /**
  * @param {LazyStructWriter} lazyWriter
  */
@@ -428,7 +471,7 @@ const writeStructToLazyStructWriter = (lazyWriter, struct, offset) => {
     // write next client
     lazyWriter.encoder.writeClient(struct.id.client)
     // write startClock
-    encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock)
+    encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock + offset)
   }
   struct.write(lazyWriter.encoder, offset)
   lazyWriter.written++

tests/testHelper.js

@@ -34,7 +34,8 @@ export const encV1 = {
   mergeUpdates: Y.mergeUpdates,
   applyUpdate: Y.applyUpdate,
   logUpdate: Y.logUpdate,
-  updateEventName: 'update'
+  updateEventName: 'update',
+  diffUpdate: Y.diffUpdate
 }
 
 export const encV2 = {
@@ -42,7 +43,8 @@ export const encV2 = {
   mergeUpdates: Y.mergeUpdatesV2,
   applyUpdate: Y.applyUpdateV2,
   logUpdate: Y.logUpdateV2,
-  updateEventName: 'updateV2'
+  updateEventName: 'updateV2',
+  diffUpdate: Y.diffUpdateV2
 }
 
 export let enc = encV1

tests/updates.tests.js

@@ -1,6 +1,9 @@
 import * as t from 'lib0/testing.js'
 import { init, compare } from './testHelper.js' // eslint-disable-line
 import * as Y from '../src/index.js'
+import { readClientsStructRefs, readDeleteSet, UpdateDecoderV2, UpdateEncoderV2, writeDeleteSet } from '../src/internals.js'
+import * as encoding from 'lib0/encoding.js'
+import * as decoding from 'lib0/decoding.js'
 
 /**
  * @typedef {Object} Enc
@@ -13,6 +16,7 @@ import * as Y from '../src/index.js'
  * @property {function(Uint8Array):Uint8Array} Enc.encodeStateVectorFromUpdate
  * @property {string} Enc.updateEventName
  * @property {string} Enc.description
+ * @property {function(Uint8Array, Uint8Array):Uint8Array} Enc.diffUpdate
  */
 
 /**
@@ -27,7 +31,8 @@ const encV1 = {
   encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdate,
   encodeStateVector: Y.encodeStateVector,
   updateEventName: 'update',
-  description: 'V1'
+  description: 'V1',
+  diffUpdate: Y.diffUpdate
 }
 
 /**
@@ -40,9 +45,10 @@ const encV2 = {
   logUpdate: Y.logUpdateV2,
   parseUpdateMeta: Y.parseUpdateMetaV2,
   encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
-  encodeStateVector: Y.encodeStateVectorV2,
+  encodeStateVector: Y.encodeStateVector,
   updateEventName: 'updateV2',
-  description: 'V2'
+  description: 'V2',
+  diffUpdate: Y.diffUpdateV2
 }
 
 /**
@@ -50,7 +56,7 @@ const encV2 = {
  */
 const encDoc = {
   mergeUpdates: (updates) => {
-    const ydoc = new Y.Doc()
+    const ydoc = new Y.Doc({ gc: false })
     updates.forEach(update => {
       Y.applyUpdateV2(ydoc, update)
     })
@@ -61,9 +67,18 @@ const encDoc = {
   logUpdate: Y.logUpdateV2,
   parseUpdateMeta: Y.parseUpdateMetaV2,
   encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
-  encodeStateVector: Y.encodeStateVectorV2,
+  encodeStateVector: Y.encodeStateVector,
   updateEventName: 'updateV2',
-  description: 'Merge via Y.Doc'
+  description: 'Merge via Y.Doc',
+  /**
+   * @param {Uint8Array} update
+   * @param {Uint8Array} sv
+   */
+  diffUpdate: (update, sv) => {
+    const ydoc = new Y.Doc({ gc: false })
+    Y.applyUpdateV2(ydoc, update)
+    return Y.encodeStateAsUpdateV2(ydoc, sv)
+  }
 }
 
 const encoders = [encV1, encV2, encDoc]
@@ -101,8 +116,9 @@ export const testMergeUpdates = tc => {
  * @param {Y.Doc} ydoc
  * @param {Array<Uint8Array>} updates - expecting at least 4 updates
  * @param {Enc} enc
+ * @param {boolean} hasDeletes
  */
-const checkUpdateCases = (ydoc, updates, enc) => {
+const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => {
   const cases = []
 
   // Case 1: Simple case, simply merge everything
@@ -135,14 +151,47 @@ const checkUpdateCases = (ydoc, updates, enc) => {
   // t.info('Target State: ')
   // enc.logUpdate(targetState)
 
-  cases.forEach((updates, i) => {
+  cases.forEach((mergedUpdates, i) => {
     // t.info('State Case $' + i + ':')
     // enc.logUpdate(updates)
-    const merged = new Y.Doc()
+    const merged = new Y.Doc({ gc: false })
-    enc.applyUpdate(merged, updates)
+    enc.applyUpdate(merged, mergedUpdates)
     t.compareArrays(merged.getArray().toArray(), ydoc.getArray().toArray())
-    t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(updates))
+    t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(mergedUpdates))
-    const meta = enc.parseUpdateMeta(updates)
+
+    if (enc.updateEventName !== 'update') { // @todo should this also work on legacy updates?
+      for (let j = 1; j < updates.length; j++) {
+        const partMerged = enc.mergeUpdates(updates.slice(j))
+        const partMeta = enc.parseUpdateMeta(partMerged)
+        const targetSV = Y.encodeStateVectorFromUpdateV2(Y.mergeUpdatesV2(updates.slice(0, j)))
+        const diffed = enc.diffUpdate(mergedUpdates, targetSV)
+        const diffedMeta = enc.parseUpdateMeta(diffed)
+        const decDiffedSV = Y.decodeStateVector(enc.encodeStateVectorFromUpdate(diffed))
+        t.compare(partMeta, diffedMeta)
+        t.compare(decDiffedSV, partMeta.to)
+        {
+          // We can'd do the following
+          //  - t.compare(diffed, mergedDeletes)
+          // because diffed contains the set of all deletes.
+          // So we add all deletes from `diffed` to `partDeletes` and compare then
+          const decoder = decoding.createDecoder(diffed)
+          const updateDecoder = new UpdateDecoderV2(decoder)
+          readClientsStructRefs(updateDecoder, new Y.Doc())
+          const ds = readDeleteSet(updateDecoder)
+          const updateEncoder = new UpdateEncoderV2()
+          encoding.writeVarUint(updateEncoder.restEncoder, 0) // 0 structs
+          writeDeleteSet(updateEncoder, ds)
+          const deletesUpdate = updateEncoder.toUint8Array()
+          const mergedDeletes = Y.mergeUpdatesV2([deletesUpdate, partMerged])
+          if (!hasDeletes || enc !== encDoc) {
+            // deletes will almost definitely lead to different encoders because of the mergeStruct feature that is present in encDoc
+            t.compare(diffed, mergedDeletes)
+          }
+        }
+      }
+    }
+
+    const meta = enc.parseUpdateMeta(mergedUpdates)
     meta.from.forEach((clock, client) => t.assert(clock === 0))
     meta.to.forEach((clock, client) => {
       const structs = /** @type {Array<Y.Item>} */ (merged.store.clients.get(client))
@@ -168,7 +217,7 @@ export const testMergeUpdates1 = tc => {
     array.insert(0, [3])
     array.insert(0, [4])
 
-    checkUpdateCases(ydoc, updates, enc)
+    checkUpdateCases(ydoc, updates, enc, false)
   })
 }
 
@@ -188,7 +237,7 @@ export const testMergeUpdates2 = tc => {
     array.insert(0, [3, 4])
     array.delete(1, 2)
 
-    checkUpdateCases(ydoc, updates, enc)
+    checkUpdateCases(ydoc, updates, enc, true)
   })
 }