yonote/yjs
23
0
Fork 0
Code Pull Requests 1 Releases Activity

implement diffUpdates with tests - #263

Browse Source
This commit is contained in:
Kevin Jahns 2021-01-29 18:18:09 +01:00
parent 7edbb2485f
commit 275d52b19d
6 changed files with 144 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/index.js
View File

@ -57,14 +57,12 @@ export {
encodeStateAsUpdate,
encodeStateAsUpdateV2,
encodeStateVector,
encodeStateVectorV2,
UndoManager,
decodeSnapshot,
encodeSnapshot,
decodeSnapshotV2,
encodeSnapshotV2,
decodeStateVector,
decodeStateVectorV2,
logUpdate,
logUpdateV2,
isDeleted,
@ -82,5 +80,7 @@ export {
encodeStateVectorFromUpdate,
encodeStateVectorFromUpdateV2,
encodeRelativePosition,
decodeRelativePosition
decodeRelativePosition,
diffUpdate,
diffUpdateV2
} from './internals.js'

4
src/structs/Skip.js
View File

@ -4,6 +4,7 @@ import {
UpdateEncoderV1, UpdateEncoderV2, StructStore, Transaction, ID // eslint-disable-line
} from '../internals.js'
import * as error from 'lib0/error.js'
import * as encoding from 'lib0/encoding.js'
export const structSkipRefNumber = 10
@ -44,7 +45,8 @@ export class Skip extends AbstractStruct {
*/
write (encoder, offset) {
encoder.writeInfo(structSkipRefNumber)
encoder.writeLen(this.length - offset)
// write as VarUint because Skips can't make use of predictable length-encoding
encoding.writeVarUint(encoder.restEncoder, this.length - offset)
}
/**

25
src/utils/encoding.js
View File

@ -29,14 +29,13 @@ import {
UpdateDecoderV2,
UpdateEncoderV1,
UpdateEncoderV2,
DSDecoderV2,
DSEncoderV2,
DSDecoderV1,
DSEncoderV1,
mergeUpdatesV2,
Skip,
diffUpdate,
Doc, Transaction, GC, Item, StructStore // eslint-disable-line
diffUpdateV2,
DSDecoderV2, Doc, Transaction, GC, Item, StructStore // eslint-disable-line
} from '../internals.js'
import * as encoding from 'lib0/encoding.js'
@ -136,7 +135,7 @@ export const readClientsStructRefs = (decoder, doc) => {
}
case 10: { // Skip Struct (nothing to apply)
// @todo we could reduce the amount of checks by adding Skip struct to clientRefs so we know that something is missing.
const len = decoder.readLen()
const len = decoding.readVarUint(decoder.restDecoder)
refs[i] = new Skip(createID(client, clock), len)
clock += len
break
@ -517,8 +516,8 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map())
*
* @function
*/
export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => {
const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector)
export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector = new Uint8Array([0]), encoder = new UpdateEncoderV2()) => {
const targetStateVector = decodeStateVector(encodedTargetStateVector)
writeStateAsUpdate(encoder, doc, targetStateVector)
const updates = [encoder.toUint8Array()]
// also add the pending updates (if there are any)
@ -528,7 +527,7 @@ export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = n
updates.push(doc.store.pendingDs)
}
if (doc.store.pendingStructs) {
updates.push(diffUpdate(doc.store.pendingStructs.update, encodedTargetStateVector))
updates.push(diffUpdateV2(doc.store.pendingStructs.update, encodedTargetStateVector))
}
if (updates.length > 1) {
return mergeUpdatesV2(updates)
@ -578,7 +577,7 @@ export const readStateVector = decoder => {
*
* @function
*/
export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
// export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
/**
* Read decodedState and return State as Map.
@ -615,21 +614,25 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod
/**
* Encode State as Uint8Array.
*
* @param {Doc} doc
* @param {Doc|Map<number,number>} doc
* @param {DSEncoderV1 | DSEncoderV2} [encoder]
* @return {Uint8Array}
*
* @function
*/
export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => {
writeDocumentStateVector(encoder, doc)
if (doc instanceof Map) {
writeStateVector(encoder, doc)
} else {
writeDocumentStateVector(encoder, doc)
}
return encoder.toUint8Array()
}
/**
* Encode State as Uint8Array.
*
* @param {Doc} doc
* @param {Doc|Map<number,number>} doc
* @return {Uint8Array}
*
* @function

71
src/utils/updates.js
View File

@ -3,6 +3,7 @@ import * as binary from 'lib0/binary.js'
import * as decoding from 'lib0/decoding.js'
import * as encoding from 'lib0/encoding.js'
import * as logging from 'lib0/logging.js'
import * as math from 'lib0/math.js'
import {
createID,
readItemContent,
@ -12,6 +13,7 @@ import {
mergeDeleteSets,
DSEncoderV1,
DSEncoderV2,
decodeStateVector,
Item, GC, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2 // eslint-disable-line
} from '../internals.js'
@ -28,7 +30,7 @@ function * lazyStructReaderGenerator (decoder) {
const info = decoder.readInfo()
// @todo use switch instead of ifs
if (info === 10) {
const len = decoder.readLen()
const len = decoding.readVarUint(decoder.restDecoder)
yield new Skip(createID(client, clock), len)
clock += len
} else if ((binary.BITS5 & info) !== 0) {
@ -62,25 +64,27 @@ function * lazyStructReaderGenerator (decoder) {
export class LazyStructReader {
/**
* @param {UpdateDecoderV1 | UpdateDecoderV2} decoder
* @param {boolean} filterSkips
*/
constructor (decoder) {
constructor (decoder, filterSkips) {
this.gen = lazyStructReaderGenerator(decoder)
/**
* @type {null | Item | GC}
* @type {null | Item | Skip | GC}
*/
this.curr = null
this.done = false
this.filterSkips = filterSkips
this.next()
}
/**
* @return {Item | GC | null}
* @return {Item | GC | Skip |null}
*/
next () {
// ignore "Skip" structs
do {
this.curr = this.gen.next().value || null
} while (this.curr !== null && this.curr.constructor === Skip)
} while (this.filterSkips && this.curr !== null && this.curr.constructor === Skip)
return this.curr
}
}
@ -99,7 +103,7 @@ export const logUpdate = update => logUpdateV2(update, UpdateDecoderV1)
export const logUpdateV2 = (update, YDecoder = UpdateDecoderV2) => {
const structs = []
const updateDecoder = new YDecoder(decoding.createDecoder(update))
const lazyDecoder = new LazyStructReader(updateDecoder)
const lazyDecoder = new LazyStructReader(updateDecoder, false)
for (let curr = lazyDecoder.curr; curr !== null; curr = lazyDecoder.next()) {
structs.push(curr)
}
@ -145,7 +149,7 @@ export const mergeUpdates = updates => mergeUpdatesV2(updates, UpdateDecoderV1,
*/
export const encodeStateVectorFromUpdateV2 = (update, YEncoder = DSEncoderV2, YDecoder = UpdateDecoderV2) => {
const encoder = new YEncoder()
const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)))
const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), true)
let curr = updateDecoder.curr
if (curr !== null) {
let size = 1
@ -204,7 +208,7 @@ export const parseUpdateMetaV2 = (update, YDecoder = UpdateDecoderV2) => {
* @type {Map<number, number>}
*/
const to = new Map()
const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)))
const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), false)
let curr = updateDecoder.curr
if (curr !== null) {
let currClient = curr.id.client
@ -277,7 +281,7 @@ const sliceStruct = (left, diff) => {
*/
export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => {
const updateDecoders = updates.map(update => new YDecoder(decoding.createDecoder(update)))
let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder))
let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder, true))
/**
* @todo we don't need offset because we always slice before
@ -395,13 +399,52 @@ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = U
/**
* @param {Uint8Array} update
* @param {Uint8Array} [sv]
* @param {Uint8Array} sv
* @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder]
* @param {typeof UpdateEncoderV1 | typeof UpdateEncoderV2} [YEncoder]
*/
export const diffUpdate = (update, sv = new Uint8Array([0])) => {
// @todo!!!
return update
export const diffUpdateV2 = (update, sv, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => {
const state = decodeStateVector(sv)
const encoder = new YEncoder()
const lazyStructWriter = new LazyStructWriter(encoder)
const decoder = new YDecoder(decoding.createDecoder(update))
const reader = new LazyStructReader(decoder, false)
while (reader.curr) {
const curr = reader.curr
const currClient = curr.id.client
const svClock = state.get(currClient) || 0
if (reader.curr.constructor === Skip) {
// the first written struct shouldn't be a skip
reader.next()
continue
}
if (curr.id.clock + curr.length > svClock) {
writeStructToLazyStructWriter(lazyStructWriter, curr, math.max(svClock - curr.id.clock, 0))
reader.next()
while (reader.curr && reader.curr.id.client === currClient) {
writeStructToLazyStructWriter(lazyStructWriter, reader.curr, 0)
reader.next()
}
} else {
// read until something new comes up
while (reader.curr && reader.curr.id.client === currClient && reader.curr.id.clock + reader.curr.length <= svClock) {
reader.next()
}
}
}
finishLazyStructWriting(lazyStructWriter)
// write ds
const ds = readDeleteSet(decoder)
writeDeleteSet(encoder, ds)
return encoder.toUint8Array()
}
/**
* @param {Uint8Array} update
* @param {Uint8Array} sv
*/
export const diffUpdate = (update, sv) => diffUpdateV2(update, sv, UpdateDecoderV1, UpdateEncoderV1)
/**
* @param {LazyStructWriter} lazyWriter
*/
@ -428,7 +471,7 @@ const writeStructToLazyStructWriter = (lazyWriter, struct, offset) => {
// write next client
lazyWriter.encoder.writeClient(struct.id.client)
// write startClock
encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock)
encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock + offset)
}
struct.write(lazyWriter.encoder, offset)
lazyWriter.written++

6
tests/testHelper.js
View File

@ -34,7 +34,8 @@ export const encV1 = {
mergeUpdates: Y.mergeUpdates,
applyUpdate: Y.applyUpdate,
logUpdate: Y.logUpdate,
updateEventName: 'update'
updateEventName: 'update',
diffUpdate: Y.diffUpdate
}
export const encV2 = {
@ -42,7 +43,8 @@ export const encV2 = {
mergeUpdates: Y.mergeUpdatesV2,
applyUpdate: Y.applyUpdateV2,
logUpdate: Y.logUpdateV2,
updateEventName: 'updateV2'
updateEventName: 'updateV2',
diffUpdate: Y.diffUpdateV2
}
export let enc = encV1

77
tests/updates.tests.js
View File

@ -1,6 +1,9 @@
import * as t from 'lib0/testing.js'
import { init, compare } from './testHelper.js' // eslint-disable-line
import * as Y from '../src/index.js'
import { readClientsStructRefs, readDeleteSet, UpdateDecoderV2, UpdateEncoderV2, writeDeleteSet } from '../src/internals.js'
import * as encoding from 'lib0/encoding.js'
import * as decoding from 'lib0/decoding.js'
/**
* @typedef {Object} Enc
@ -13,6 +16,7 @@ import * as Y from '../src/index.js'
* @property {function(Uint8Array):Uint8Array} Enc.encodeStateVectorFromUpdate
* @property {string} Enc.updateEventName
* @property {string} Enc.description
* @property {function(Uint8Array, Uint8Array):Uint8Array} Enc.diffUpdate
*/
/**
@ -27,7 +31,8 @@ const encV1 = {
encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdate,
encodeStateVector: Y.encodeStateVector,
updateEventName: 'update',
description: 'V1'
description: 'V1',
diffUpdate: Y.diffUpdate
}
/**
@ -40,9 +45,10 @@ const encV2 = {
logUpdate: Y.logUpdateV2,
parseUpdateMeta: Y.parseUpdateMetaV2,
encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
encodeStateVector: Y.encodeStateVectorV2,
encodeStateVector: Y.encodeStateVector,
updateEventName: 'updateV2',
description: 'V2'
description: 'V2',
diffUpdate: Y.diffUpdateV2
}
/**
@ -50,7 +56,7 @@ const encV2 = {
*/
const encDoc = {
mergeUpdates: (updates) => {
const ydoc = new Y.Doc()
const ydoc = new Y.Doc({ gc: false })
updates.forEach(update => {
Y.applyUpdateV2(ydoc, update)
})
@ -61,9 +67,18 @@ const encDoc = {
logUpdate: Y.logUpdateV2,
parseUpdateMeta: Y.parseUpdateMetaV2,
encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
encodeStateVector: Y.encodeStateVectorV2,
encodeStateVector: Y.encodeStateVector,
updateEventName: 'updateV2',
description: 'Merge via Y.Doc'
description: 'Merge via Y.Doc',
/**
* @param {Uint8Array} update
* @param {Uint8Array} sv
*/
diffUpdate: (update, sv) => {
const ydoc = new Y.Doc({ gc: false })
Y.applyUpdateV2(ydoc, update)
return Y.encodeStateAsUpdateV2(ydoc, sv)
}
}
const encoders = [encV1, encV2, encDoc]
@ -101,8 +116,9 @@ export const testMergeUpdates = tc => {
* @param {Y.Doc} ydoc
* @param {Array<Uint8Array>} updates - expecting at least 4 updates
* @param {Enc} enc
* @param {boolean} hasDeletes
*/
const checkUpdateCases = (ydoc, updates, enc) => {
const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => {
const cases = []
// Case 1: Simple case, simply merge everything
@ -135,14 +151,47 @@ const checkUpdateCases = (ydoc, updates, enc) => {
// t.info('Target State: ')
// enc.logUpdate(targetState)
cases.forEach((updates, i) => {
cases.forEach((mergedUpdates, i) => {
// t.info('State Case $' + i + ':')
// enc.logUpdate(updates)
const merged = new Y.Doc()
enc.applyUpdate(merged, updates)
const merged = new Y.Doc({ gc: false })
enc.applyUpdate(merged, mergedUpdates)
t.compareArrays(merged.getArray().toArray(), ydoc.getArray().toArray())
t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(updates))
const meta = enc.parseUpdateMeta(updates)
t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(mergedUpdates))
if (enc.updateEventName !== 'update') { // @todo should this also work on legacy updates?
for (let j = 1; j < updates.length; j++) {
const partMerged = enc.mergeUpdates(updates.slice(j))
const partMeta = enc.parseUpdateMeta(partMerged)
const targetSV = Y.encodeStateVectorFromUpdateV2(Y.mergeUpdatesV2(updates.slice(0, j)))
const diffed = enc.diffUpdate(mergedUpdates, targetSV)
const diffedMeta = enc.parseUpdateMeta(diffed)
const decDiffedSV = Y.decodeStateVector(enc.encodeStateVectorFromUpdate(diffed))
t.compare(partMeta, diffedMeta)
t.compare(decDiffedSV, partMeta.to)
{
// We can'd do the following
// - t.compare(diffed, mergedDeletes)
// because diffed contains the set of all deletes.
// So we add all deletes from `diffed` to `partDeletes` and compare then
const decoder = decoding.createDecoder(diffed)
const updateDecoder = new UpdateDecoderV2(decoder)
readClientsStructRefs(updateDecoder, new Y.Doc())
const ds = readDeleteSet(updateDecoder)
const updateEncoder = new UpdateEncoderV2()
encoding.writeVarUint(updateEncoder.restEncoder, 0) // 0 structs
writeDeleteSet(updateEncoder, ds)
const deletesUpdate = updateEncoder.toUint8Array()
const mergedDeletes = Y.mergeUpdatesV2([deletesUpdate, partMerged])
if (!hasDeletes || enc !== encDoc) {
// deletes will almost definitely lead to different encoders because of the mergeStruct feature that is present in encDoc
t.compare(diffed, mergedDeletes)
}
}
}
}
const meta = enc.parseUpdateMeta(mergedUpdates)
meta.from.forEach((clock, client) => t.assert(clock === 0))
meta.to.forEach((clock, client) => {
const structs = /** @type {Array<Y.Item>} */ (merged.store.clients.get(client))
@ -168,7 +217,7 @@ export const testMergeUpdates1 = tc => {
array.insert(0, [3])
array.insert(0, [4])
checkUpdateCases(ydoc, updates, enc)
checkUpdateCases(ydoc, updates, enc, false)
})
}
@ -188,7 +237,7 @@ export const testMergeUpdates2 = tc => {
array.insert(0, [3, 4])
array.delete(1, 2)
checkUpdateCases(ydoc, updates, enc)
checkUpdateCases(ydoc, updates, enc, true)
})
}