From 275d52b19d5dd665937fa5b337cb246fdf996cd0 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 29 Jan 2021 18:18:09 +0100 Subject: [PATCH] implement diffUpdates with tests - #263 --- src/index.js | 6 ++-- src/structs/Skip.js | 4 ++- src/utils/encoding.js | 25 ++++++++------ src/utils/updates.js | 71 ++++++++++++++++++++++++++++++-------- tests/testHelper.js | 6 ++-- tests/updates.tests.js | 77 ++++++++++++++++++++++++++++++++++-------- 6 files changed, 144 insertions(+), 45 deletions(-) diff --git a/src/index.js b/src/index.js index ec2fe7f9..53eb066d 100644 --- a/src/index.js +++ b/src/index.js @@ -57,14 +57,12 @@ export { encodeStateAsUpdate, encodeStateAsUpdateV2, encodeStateVector, - encodeStateVectorV2, UndoManager, decodeSnapshot, encodeSnapshot, decodeSnapshotV2, encodeSnapshotV2, decodeStateVector, - decodeStateVectorV2, logUpdate, logUpdateV2, isDeleted, @@ -82,5 +80,7 @@ export { encodeStateVectorFromUpdate, encodeStateVectorFromUpdateV2, encodeRelativePosition, - decodeRelativePosition + decodeRelativePosition, + diffUpdate, + diffUpdateV2 } from './internals.js' diff --git a/src/structs/Skip.js b/src/structs/Skip.js index eaec5c06..c763f853 100644 --- a/src/structs/Skip.js +++ b/src/structs/Skip.js @@ -4,6 +4,7 @@ import { UpdateEncoderV1, UpdateEncoderV2, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' import * as error from 'lib0/error.js' +import * as encoding from 'lib0/encoding.js' export const structSkipRefNumber = 10 @@ -44,7 +45,8 @@ export class Skip extends AbstractStruct { */ write (encoder, offset) { encoder.writeInfo(structSkipRefNumber) - encoder.writeLen(this.length - offset) + // write as VarUint because Skips can't make use of predictable length-encoding + encoding.writeVarUint(encoder.restEncoder, this.length - offset) } /** diff --git a/src/utils/encoding.js b/src/utils/encoding.js index 2f0b3b40..c81bdab8 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -29,14 +29,13 @@ import { UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, - DSDecoderV2, DSEncoderV2, DSDecoderV1, DSEncoderV1, mergeUpdatesV2, Skip, - diffUpdate, - Doc, Transaction, GC, Item, StructStore // eslint-disable-line + diffUpdateV2, + DSDecoderV2, Doc, Transaction, GC, Item, StructStore // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -136,7 +135,7 @@ export const readClientsStructRefs = (decoder, doc) => { } case 10: { // Skip Struct (nothing to apply) // @todo we could reduce the amount of checks by adding Skip struct to clientRefs so we know that something is missing. - const len = decoder.readLen() + const len = decoding.readVarUint(decoder.restDecoder) refs[i] = new Skip(createID(client, clock), len) clock += len break @@ -517,8 +516,8 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map()) * * @function */ -export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => { - const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector) +export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector = new Uint8Array([0]), encoder = new UpdateEncoderV2()) => { + const targetStateVector = decodeStateVector(encodedTargetStateVector) writeStateAsUpdate(encoder, doc, targetStateVector) const updates = [encoder.toUint8Array()] // also add the pending updates (if there are any) @@ -528,7 +527,7 @@ export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = n updates.push(doc.store.pendingDs) } if (doc.store.pendingStructs) { - updates.push(diffUpdate(doc.store.pendingStructs.update, encodedTargetStateVector)) + updates.push(diffUpdateV2(doc.store.pendingStructs.update, encodedTargetStateVector)) } if (updates.length > 1) { return mergeUpdatesV2(updates) @@ -578,7 +577,7 @@ export const readStateVector = decoder => { * * @function */ -export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState))) +// export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState))) /** * Read decodedState and return State as Map. @@ -615,21 +614,25 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod /** * Encode State as Uint8Array. * - * @param {Doc} doc + * @param {Doc|Map} doc * @param {DSEncoderV1 | DSEncoderV2} [encoder] * @return {Uint8Array} * * @function */ export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => { - writeDocumentStateVector(encoder, doc) + if (doc instanceof Map) { + writeStateVector(encoder, doc) + } else { + writeDocumentStateVector(encoder, doc) + } return encoder.toUint8Array() } /** * Encode State as Uint8Array. * - * @param {Doc} doc + * @param {Doc|Map} doc * @return {Uint8Array} * * @function diff --git a/src/utils/updates.js b/src/utils/updates.js index 74acdb10..4bac25bc 100644 --- a/src/utils/updates.js +++ b/src/utils/updates.js @@ -3,6 +3,7 @@ import * as binary from 'lib0/binary.js' import * as decoding from 'lib0/decoding.js' import * as encoding from 'lib0/encoding.js' import * as logging from 'lib0/logging.js' +import * as math from 'lib0/math.js' import { createID, readItemContent, @@ -12,6 +13,7 @@ import { mergeDeleteSets, DSEncoderV1, DSEncoderV2, + decodeStateVector, Item, GC, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2 // eslint-disable-line } from '../internals.js' @@ -28,7 +30,7 @@ function * lazyStructReaderGenerator (decoder) { const info = decoder.readInfo() // @todo use switch instead of ifs if (info === 10) { - const len = decoder.readLen() + const len = decoding.readVarUint(decoder.restDecoder) yield new Skip(createID(client, clock), len) clock += len } else if ((binary.BITS5 & info) !== 0) { @@ -62,25 +64,27 @@ function * lazyStructReaderGenerator (decoder) { export class LazyStructReader { /** * @param {UpdateDecoderV1 | UpdateDecoderV2} decoder + * @param {boolean} filterSkips */ - constructor (decoder) { + constructor (decoder, filterSkips) { this.gen = lazyStructReaderGenerator(decoder) /** - * @type {null | Item | GC} + * @type {null | Item | Skip | GC} */ this.curr = null this.done = false + this.filterSkips = filterSkips this.next() } /** - * @return {Item | GC | null} + * @return {Item | GC | Skip |null} */ next () { // ignore "Skip" structs do { this.curr = this.gen.next().value || null - } while (this.curr !== null && this.curr.constructor === Skip) + } while (this.filterSkips && this.curr !== null && this.curr.constructor === Skip) return this.curr } } @@ -99,7 +103,7 @@ export const logUpdate = update => logUpdateV2(update, UpdateDecoderV1) export const logUpdateV2 = (update, YDecoder = UpdateDecoderV2) => { const structs = [] const updateDecoder = new YDecoder(decoding.createDecoder(update)) - const lazyDecoder = new LazyStructReader(updateDecoder) + const lazyDecoder = new LazyStructReader(updateDecoder, false) for (let curr = lazyDecoder.curr; curr !== null; curr = lazyDecoder.next()) { structs.push(curr) } @@ -145,7 +149,7 @@ export const mergeUpdates = updates => mergeUpdatesV2(updates, UpdateDecoderV1, */ export const encodeStateVectorFromUpdateV2 = (update, YEncoder = DSEncoderV2, YDecoder = UpdateDecoderV2) => { const encoder = new YEncoder() - const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update))) + const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), true) let curr = updateDecoder.curr if (curr !== null) { let size = 1 @@ -204,7 +208,7 @@ export const parseUpdateMetaV2 = (update, YDecoder = UpdateDecoderV2) => { * @type {Map} */ const to = new Map() - const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update))) + const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), false) let curr = updateDecoder.curr if (curr !== null) { let currClient = curr.id.client @@ -277,7 +281,7 @@ const sliceStruct = (left, diff) => { */ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => { const updateDecoders = updates.map(update => new YDecoder(decoding.createDecoder(update))) - let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder)) + let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder, true)) /** * @todo we don't need offset because we always slice before @@ -395,13 +399,52 @@ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = U /** * @param {Uint8Array} update - * @param {Uint8Array} [sv] + * @param {Uint8Array} sv + * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder] + * @param {typeof UpdateEncoderV1 | typeof UpdateEncoderV2} [YEncoder] */ -export const diffUpdate = (update, sv = new Uint8Array([0])) => { - // @todo!!! - return update +export const diffUpdateV2 = (update, sv, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => { + const state = decodeStateVector(sv) + const encoder = new YEncoder() + const lazyStructWriter = new LazyStructWriter(encoder) + const decoder = new YDecoder(decoding.createDecoder(update)) + const reader = new LazyStructReader(decoder, false) + while (reader.curr) { + const curr = reader.curr + const currClient = curr.id.client + const svClock = state.get(currClient) || 0 + if (reader.curr.constructor === Skip) { + // the first written struct shouldn't be a skip + reader.next() + continue + } + if (curr.id.clock + curr.length > svClock) { + writeStructToLazyStructWriter(lazyStructWriter, curr, math.max(svClock - curr.id.clock, 0)) + reader.next() + while (reader.curr && reader.curr.id.client === currClient) { + writeStructToLazyStructWriter(lazyStructWriter, reader.curr, 0) + reader.next() + } + } else { + // read until something new comes up + while (reader.curr && reader.curr.id.client === currClient && reader.curr.id.clock + reader.curr.length <= svClock) { + reader.next() + } + } + } + finishLazyStructWriting(lazyStructWriter) + // write ds + const ds = readDeleteSet(decoder) + writeDeleteSet(encoder, ds) + return encoder.toUint8Array() } +/** + * @param {Uint8Array} update + * @param {Uint8Array} sv + */ +export const diffUpdate = (update, sv) => diffUpdateV2(update, sv, UpdateDecoderV1, UpdateEncoderV1) + /** * @param {LazyStructWriter} lazyWriter */ @@ -428,7 +471,7 @@ const writeStructToLazyStructWriter = (lazyWriter, struct, offset) => { // write next client lazyWriter.encoder.writeClient(struct.id.client) // write startClock - encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock) + encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock + offset) } struct.write(lazyWriter.encoder, offset) lazyWriter.written++ diff --git a/tests/testHelper.js b/tests/testHelper.js index a438a31e..5ebb7836 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -34,7 +34,8 @@ export const encV1 = { mergeUpdates: Y.mergeUpdates, applyUpdate: Y.applyUpdate, logUpdate: Y.logUpdate, - updateEventName: 'update' + updateEventName: 'update', + diffUpdate: Y.diffUpdate } export const encV2 = { @@ -42,7 +43,8 @@ export const encV2 = { mergeUpdates: Y.mergeUpdatesV2, applyUpdate: Y.applyUpdateV2, logUpdate: Y.logUpdateV2, - updateEventName: 'updateV2' + updateEventName: 'updateV2', + diffUpdate: Y.diffUpdateV2 } export let enc = encV1 diff --git a/tests/updates.tests.js b/tests/updates.tests.js index 0fd831ef..9c006642 100644 --- a/tests/updates.tests.js +++ b/tests/updates.tests.js @@ -1,6 +1,9 @@ import * as t from 'lib0/testing.js' import { init, compare } from './testHelper.js' // eslint-disable-line import * as Y from '../src/index.js' +import { readClientsStructRefs, readDeleteSet, UpdateDecoderV2, UpdateEncoderV2, writeDeleteSet } from '../src/internals.js' +import * as encoding from 'lib0/encoding.js' +import * as decoding from 'lib0/decoding.js' /** * @typedef {Object} Enc @@ -13,6 +16,7 @@ import * as Y from '../src/index.js' * @property {function(Uint8Array):Uint8Array} Enc.encodeStateVectorFromUpdate * @property {string} Enc.updateEventName * @property {string} Enc.description + * @property {function(Uint8Array, Uint8Array):Uint8Array} Enc.diffUpdate */ /** @@ -27,7 +31,8 @@ const encV1 = { encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdate, encodeStateVector: Y.encodeStateVector, updateEventName: 'update', - description: 'V1' + description: 'V1', + diffUpdate: Y.diffUpdate } /** @@ -40,9 +45,10 @@ const encV2 = { logUpdate: Y.logUpdateV2, parseUpdateMeta: Y.parseUpdateMetaV2, encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2, - encodeStateVector: Y.encodeStateVectorV2, + encodeStateVector: Y.encodeStateVector, updateEventName: 'updateV2', - description: 'V2' + description: 'V2', + diffUpdate: Y.diffUpdateV2 } /** @@ -50,7 +56,7 @@ const encV2 = { */ const encDoc = { mergeUpdates: (updates) => { - const ydoc = new Y.Doc() + const ydoc = new Y.Doc({ gc: false }) updates.forEach(update => { Y.applyUpdateV2(ydoc, update) }) @@ -61,9 +67,18 @@ const encDoc = { logUpdate: Y.logUpdateV2, parseUpdateMeta: Y.parseUpdateMetaV2, encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2, - encodeStateVector: Y.encodeStateVectorV2, + encodeStateVector: Y.encodeStateVector, updateEventName: 'updateV2', - description: 'Merge via Y.Doc' + description: 'Merge via Y.Doc', + /** + * @param {Uint8Array} update + * @param {Uint8Array} sv + */ + diffUpdate: (update, sv) => { + const ydoc = new Y.Doc({ gc: false }) + Y.applyUpdateV2(ydoc, update) + return Y.encodeStateAsUpdateV2(ydoc, sv) + } } const encoders = [encV1, encV2, encDoc] @@ -101,8 +116,9 @@ export const testMergeUpdates = tc => { * @param {Y.Doc} ydoc * @param {Array} updates - expecting at least 4 updates * @param {Enc} enc + * @param {boolean} hasDeletes */ -const checkUpdateCases = (ydoc, updates, enc) => { +const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => { const cases = [] // Case 1: Simple case, simply merge everything @@ -135,14 +151,47 @@ const checkUpdateCases = (ydoc, updates, enc) => { // t.info('Target State: ') // enc.logUpdate(targetState) - cases.forEach((updates, i) => { + cases.forEach((mergedUpdates, i) => { // t.info('State Case $' + i + ':') // enc.logUpdate(updates) - const merged = new Y.Doc() - enc.applyUpdate(merged, updates) + const merged = new Y.Doc({ gc: false }) + enc.applyUpdate(merged, mergedUpdates) t.compareArrays(merged.getArray().toArray(), ydoc.getArray().toArray()) - t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(updates)) - const meta = enc.parseUpdateMeta(updates) + t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(mergedUpdates)) + + if (enc.updateEventName !== 'update') { // @todo should this also work on legacy updates? + for (let j = 1; j < updates.length; j++) { + const partMerged = enc.mergeUpdates(updates.slice(j)) + const partMeta = enc.parseUpdateMeta(partMerged) + const targetSV = Y.encodeStateVectorFromUpdateV2(Y.mergeUpdatesV2(updates.slice(0, j))) + const diffed = enc.diffUpdate(mergedUpdates, targetSV) + const diffedMeta = enc.parseUpdateMeta(diffed) + const decDiffedSV = Y.decodeStateVector(enc.encodeStateVectorFromUpdate(diffed)) + t.compare(partMeta, diffedMeta) + t.compare(decDiffedSV, partMeta.to) + { + // We can'd do the following + // - t.compare(diffed, mergedDeletes) + // because diffed contains the set of all deletes. + // So we add all deletes from `diffed` to `partDeletes` and compare then + const decoder = decoding.createDecoder(diffed) + const updateDecoder = new UpdateDecoderV2(decoder) + readClientsStructRefs(updateDecoder, new Y.Doc()) + const ds = readDeleteSet(updateDecoder) + const updateEncoder = new UpdateEncoderV2() + encoding.writeVarUint(updateEncoder.restEncoder, 0) // 0 structs + writeDeleteSet(updateEncoder, ds) + const deletesUpdate = updateEncoder.toUint8Array() + const mergedDeletes = Y.mergeUpdatesV2([deletesUpdate, partMerged]) + if (!hasDeletes || enc !== encDoc) { + // deletes will almost definitely lead to different encoders because of the mergeStruct feature that is present in encDoc + t.compare(diffed, mergedDeletes) + } + } + } + } + + const meta = enc.parseUpdateMeta(mergedUpdates) meta.from.forEach((clock, client) => t.assert(clock === 0)) meta.to.forEach((clock, client) => { const structs = /** @type {Array} */ (merged.store.clients.get(client)) @@ -168,7 +217,7 @@ export const testMergeUpdates1 = tc => { array.insert(0, [3]) array.insert(0, [4]) - checkUpdateCases(ydoc, updates, enc) + checkUpdateCases(ydoc, updates, enc, false) }) } @@ -188,7 +237,7 @@ export const testMergeUpdates2 = tc => { array.insert(0, [3, 4]) array.delete(1, 2) - checkUpdateCases(ydoc, updates, enc) + checkUpdateCases(ydoc, updates, enc, true) }) }