yonote/yjs
14
0
Fork 0
Code Pull Requests 1 Releases Activity

Merge pull request #274 from yjs/differential-updates-263

Browse Source 
Differential updates
This commit is contained in:
Kevin Jahns
2021-02-07 23:58:59 +01:00
committed by GitHub
parent ce80cb4a0d 2199ac3e4e
commit fdf632f03e
37 changed files with 1327 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/encoding.tests.js
View File

@@ -22,7 +22,7 @@ import {
* @param {t.TestCase} tc
*/
export const testStructReferences = tc => {
t.assert(contentRefs.length === 10)
t.assert(contentRefs.length === 11)
t.assert(contentRefs[1] === readContentDeleted)
t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json?
t.assert(contentRefs[3] === readContentBinary)
@@ -32,6 +32,7 @@ export const testStructReferences = tc => {
t.assert(contentRefs[7] === readContentType)
t.assert(contentRefs[8] === readContentAny)
t.assert(contentRefs[9] === readContentDoc)
// contentRefs[10] is reserved for Skip structs
}
/**

3
tests/index.js
View File

@@ -8,6 +8,7 @@ import * as undoredo from './undo-redo.tests.js'
import * as compatibility from './compatibility.tests.js'
import * as doc from './doc.tests.js'
import * as snapshot from './snapshot.tests.js'
import * as updates from './updates.tests.js'
import * as relativePositions from './relativePositions.tests.js'
import { runTests } from 'lib0/testing.js'
@@ -18,7 +19,7 @@ if (isBrowser) {
log.createVConsole(document.body)
}
runTests({
doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, relativePositions
doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions
}).then(success => {
/* istanbul ignore next */
if (isNode) {

72
tests/testHelper.js
View File

@@ -27,6 +27,39 @@ const broadcastMessage = (y, m) => {
}
}
export let useV2 = false
export const encV1 = {
encodeStateAsUpdate: Y.encodeStateAsUpdate,
mergeUpdates: Y.mergeUpdates,
applyUpdate: Y.applyUpdate,
logUpdate: Y.logUpdate,
updateEventName: 'update',
diffUpdate: Y.diffUpdate
}
export const encV2 = {
encodeStateAsUpdate: Y.encodeStateAsUpdateV2,
mergeUpdates: Y.mergeUpdatesV2,
applyUpdate: Y.applyUpdateV2,
logUpdate: Y.logUpdateV2,
updateEventName: 'updateV2',
diffUpdate: Y.diffUpdateV2
}
export let enc = encV1
const useV1Encoding = () => {
useV2 = false
enc = encV1
}
const useV2Encoding = () => {
console.error('sync protocol doesnt support v2 protocol yet, fallback to v1 encoding') // @Todo
useV2 = false
enc = encV1
}
export class TestYInstance extends Y.Doc {
/**
* @param {TestConnector} testConnector
@@ -44,12 +77,19 @@ export class TestYInstance extends Y.Doc {
*/
this.receiving = new Map()
testConnector.allConns.add(this)
/**
* The list of received updates.
* We are going to merge them later using Y.mergeUpdates and check if the resulting document is correct.
* @type {Array<Uint8Array>}
*/
this.updates = []
// set up observe on local model
this.on('update', /** @param {Uint8Array} update @param {any} origin */ (update, origin) => {
this.on(enc.updateEventName, /** @param {Uint8Array} update @param {any} origin */ (update, origin) => {
if (origin !== testConnector) {
const encoder = encoding.createEncoder()
syncProtocol.writeUpdate(encoder, update)
broadcastMessage(this, encoding.toUint8Array(encoder))
this.updates.push(update)
}
})
this.connect()
@@ -162,6 +202,17 @@ export class TestConnector {
// send reply message
sender._receive(encoding.toUint8Array(encoder), receiver)
}
{
// If update message, add the received message to the list of received messages
const decoder = decoding.createDecoder(m)
const messageType = decoding.readVarUint(decoder)
switch (messageType) {
case syncProtocol.messageYjsUpdate:
case syncProtocol.messageYjsSyncStep2:
receiver.updates.push(decoding.readVarUint8Array(decoder))
break
}
}
return true
}
return false
@@ -240,9 +291,9 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => {
const gen = tc.prng
// choose an encoding approach at random
if (prng.bool(gen)) {
Y.useV2Encoding()
useV2Encoding()
} else {
Y.useV1Encoding()
useV1Encoding()
}
const testConnector = new TestConnector(gen)
@@ -258,7 +309,7 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => {
}
testConnector.syncAll()
result.testObjects = result.users.map(initTestObject || (() => null))
Y.useV1Encoding()
useV1Encoding()
return /** @type {any} */ (result)
}
@@ -274,14 +325,21 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => {
export const compare = users => {
users.forEach(u => u.connect())
while (users[0].tc.flushAllMessages()) {}
// For each document, merge all received document updates with Y.mergeUpdates and create a new document which will be added to the list of "users"
// This ensures that mergeUpdates works correctly
const mergedDocs = users.map(user => {
const ydoc = new Y.Doc()
enc.applyUpdate(ydoc, enc.mergeUpdates(user.updates))
return ydoc
})
users.push(.../** @type {any} */(mergedDocs))
const userArrayValues = users.map(u => u.getArray('array').toJSON())
const userMapValues = users.map(u => u.getMap('map').toJSON())
const userXmlValues = users.map(u => u.get('xml', Y.YXmlElement).toString())
const userTextValues = users.map(u => u.getText('text').toDelta())
for (const u of users) {
t.assert(u.store.pendingDeleteReaders.length === 0)
t.assert(u.store.pendingStack.length === 0)
t.assert(u.store.pendingClientsStructRefs.size === 0)
t.assert(u.store.pendingDs === null)
t.assert(u.store.pendingStructs === null)
}
// Test Array iterator
t.compare(users[0].getArray('array').toArray(), Array.from(users[0].getArray('array')))

246
tests/updates.tests.js Normal file
View File

@@ -0,0 +1,246 @@
import * as t from 'lib0/testing.js'
import { init, compare } from './testHelper.js' // eslint-disable-line
import * as Y from '../src/index.js'
import { readClientsStructRefs, readDeleteSet, UpdateDecoderV2, UpdateEncoderV2, writeDeleteSet } from '../src/internals.js'
import * as encoding from 'lib0/encoding.js'
import * as decoding from 'lib0/decoding.js'
/**
* @typedef {Object} Enc
* @property {function(Array<Uint8Array>):Uint8Array} Enc.mergeUpdates
* @property {function(Y.Doc):Uint8Array} Enc.encodeStateAsUpdate
* @property {function(Y.Doc, Uint8Array):void} Enc.applyUpdate
* @property {function(Uint8Array):void} Enc.logUpdate
* @property {function(Uint8Array):{from:Map<number,number>,to:Map<number,number>}} Enc.parseUpdateMeta
* @property {function(Y.Doc):Uint8Array} Enc.encodeStateVector
* @property {function(Uint8Array):Uint8Array} Enc.encodeStateVectorFromUpdate
* @property {string} Enc.updateEventName
* @property {string} Enc.description
* @property {function(Uint8Array, Uint8Array):Uint8Array} Enc.diffUpdate
*/
/**
* @type {Enc}
*/
const encV1 = {
mergeUpdates: Y.mergeUpdates,
encodeStateAsUpdate: Y.encodeStateAsUpdate,
applyUpdate: Y.applyUpdate,
logUpdate: Y.logUpdate,
parseUpdateMeta: Y.parseUpdateMeta,
encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdate,
encodeStateVector: Y.encodeStateVector,
updateEventName: 'update',
description: 'V1',
diffUpdate: Y.diffUpdate
}
/**
* @type {Enc}
*/
const encV2 = {
mergeUpdates: Y.mergeUpdatesV2,
encodeStateAsUpdate: Y.encodeStateAsUpdateV2,
applyUpdate: Y.applyUpdateV2,
logUpdate: Y.logUpdateV2,
parseUpdateMeta: Y.parseUpdateMetaV2,
encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
encodeStateVector: Y.encodeStateVector,
updateEventName: 'updateV2',
description: 'V2',
diffUpdate: Y.diffUpdateV2
}
/**
* @type {Enc}
*/
const encDoc = {
mergeUpdates: (updates) => {
const ydoc = new Y.Doc({ gc: false })
updates.forEach(update => {
Y.applyUpdateV2(ydoc, update)
})
return Y.encodeStateAsUpdateV2(ydoc)
},
encodeStateAsUpdate: Y.encodeStateAsUpdateV2,
applyUpdate: Y.applyUpdateV2,
logUpdate: Y.logUpdateV2,
parseUpdateMeta: Y.parseUpdateMetaV2,
encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
encodeStateVector: Y.encodeStateVector,
updateEventName: 'updateV2',
description: 'Merge via Y.Doc',
/**
* @param {Uint8Array} update
* @param {Uint8Array} sv
*/
diffUpdate: (update, sv) => {
const ydoc = new Y.Doc({ gc: false })
Y.applyUpdateV2(ydoc, update)
return Y.encodeStateAsUpdateV2(ydoc, sv)
}
}
const encoders = [encV1, encV2, encDoc]
/**
* @param {Array<Y.Doc>} users
* @param {Enc} enc
*/
const fromUpdates = (users, enc) => {
const updates = users.map(user =>
enc.encodeStateAsUpdate(user)
)
const ydoc = new Y.Doc()
enc.applyUpdate(ydoc, enc.mergeUpdates(updates))
return ydoc
}
/**
* @param {t.TestCase} tc
*/
export const testMergeUpdates = tc => {
const { users, array0, array1 } = init(tc, { users: 3 })
array0.insert(0, [1])
array1.insert(0, [2])
compare(users)
encoders.forEach(enc => {
const merged = fromUpdates(users, enc)
t.compareArrays(array0.toArray(), merged.getArray('array').toArray())
})
}
/**
* @param {Y.Doc} ydoc
* @param {Array<Uint8Array>} updates - expecting at least 4 updates
* @param {Enc} enc
* @param {boolean} hasDeletes
*/
const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => {
const cases = []
// Case 1: Simple case, simply merge everything
cases.push(enc.mergeUpdates(updates))
// Case 2: Overlapping updates
cases.push(enc.mergeUpdates([
enc.mergeUpdates(updates.slice(2)),
enc.mergeUpdates(updates.slice(0, 2))
]))
// Case 3: Overlapping updates
cases.push(enc.mergeUpdates([
enc.mergeUpdates(updates.slice(2)),
enc.mergeUpdates(updates.slice(1, 3)),
updates[0]
]))
// Case 4: Separated updates (containing skips)
cases.push(enc.mergeUpdates([
enc.mergeUpdates([updates[0], updates[2]]),
enc.mergeUpdates([updates[1], updates[3]]),
enc.mergeUpdates(updates.slice(4))
]))
// Case 5: overlapping with many duplicates
cases.push(enc.mergeUpdates(cases))
// const targetState = enc.encodeStateAsUpdate(ydoc)
// t.info('Target State: ')
// enc.logUpdate(targetState)
cases.forEach((mergedUpdates, i) => {
// t.info('State Case $' + i + ':')
// enc.logUpdate(updates)
const merged = new Y.Doc({ gc: false })
enc.applyUpdate(merged, mergedUpdates)
t.compareArrays(merged.getArray().toArray(), ydoc.getArray().toArray())
t.compare(enc.encodeStateVector(merged), enc.encodeStateVectorFromUpdate(mergedUpdates))
if (enc.updateEventName !== 'update') { // @todo should this also work on legacy updates?
for (let j = 1; j < updates.length; j++) {
const partMerged = enc.mergeUpdates(updates.slice(j))
const partMeta = enc.parseUpdateMeta(partMerged)
const targetSV = Y.encodeStateVectorFromUpdateV2(Y.mergeUpdatesV2(updates.slice(0, j)))
const diffed = enc.diffUpdate(mergedUpdates, targetSV)
const diffedMeta = enc.parseUpdateMeta(diffed)
const decDiffedSV = Y.decodeStateVector(enc.encodeStateVectorFromUpdate(diffed))
t.compare(partMeta, diffedMeta)
t.compare(decDiffedSV, partMeta.to)
{
// We can'd do the following
// - t.compare(diffed, mergedDeletes)
// because diffed contains the set of all deletes.
// So we add all deletes from `diffed` to `partDeletes` and compare then
const decoder = decoding.createDecoder(diffed)
const updateDecoder = new UpdateDecoderV2(decoder)
readClientsStructRefs(updateDecoder, new Y.Doc())
const ds = readDeleteSet(updateDecoder)
const updateEncoder = new UpdateEncoderV2()
encoding.writeVarUint(updateEncoder.restEncoder, 0) // 0 structs
writeDeleteSet(updateEncoder, ds)
const deletesUpdate = updateEncoder.toUint8Array()
const mergedDeletes = Y.mergeUpdatesV2([deletesUpdate, partMerged])
if (!hasDeletes || enc !== encDoc) {
// deletes will almost definitely lead to different encoders because of the mergeStruct feature that is present in encDoc
t.compare(diffed, mergedDeletes)
}
}
}
}
const meta = enc.parseUpdateMeta(mergedUpdates)
meta.from.forEach((clock, client) => t.assert(clock === 0))
meta.to.forEach((clock, client) => {
const structs = /** @type {Array<Y.Item>} */ (merged.store.clients.get(client))
const lastStruct = structs[structs.length - 1]
t.assert(lastStruct.id.clock + lastStruct.length === clock)
})
})
}
/**
* @param {t.TestCase} tc
*/
export const testMergeUpdates1 = tc => {
encoders.forEach((enc, i) => {
t.info(`Using encoder: ${enc.description}`)
const ydoc = new Y.Doc({ gc: false })
const updates = /** @type {Array<Uint8Array>} */ ([])
ydoc.on(enc.updateEventName, update => { updates.push(update) })
const array = ydoc.getArray()
array.insert(0, [1])
array.insert(0, [2])
array.insert(0, [3])
array.insert(0, [4])
checkUpdateCases(ydoc, updates, enc, false)
})
}
/**
* @param {t.TestCase} tc
*/
export const testMergeUpdates2 = tc => {
encoders.forEach((enc, i) => {
t.info(`Using encoder: ${enc.description}`)
const ydoc = new Y.Doc({ gc: false })
const updates = /** @type {Array<Uint8Array>} */ ([])
ydoc.on(enc.updateEventName, update => { updates.push(update) })
const array = ydoc.getArray()
array.insert(0, [1, 2])
array.delete(1, 1)
array.insert(0, [3, 4])
array.delete(1, 2)
checkUpdateCases(ydoc, updates, enc, true)
})
}
/**
* @todo be able to apply Skip structs to Yjs docs
*/

30
tests/y-array.tests.js
View File

@@ -64,7 +64,7 @@ export const testInsertThreeElementsTryRegetProperty = tc => {
* @param {t.TestCase} tc
*/
export const testConcurrentInsertWithThreeConflicts = tc => {
var { users, array0, array1, array2 } = init(tc, { users: 3 })
const { users, array0, array1, array2 } = init(tc, { users: 3 })
array0.insert(0, [0])
array1.insert(0, [1])
array2.insert(0, [2])
@@ -107,7 +107,7 @@ export const testInsertionsInLateSync = tc => {
* @param {t.TestCase} tc
*/
export const testDisconnectReallyPreventsSendingMessages = tc => {
var { testConnector, users, array0, array1 } = init(tc, { users: 3 })
const { testConnector, users, array0, array1 } = init(tc, { users: 3 })
array0.insert(0, ['x', 'y'])
testConnector.flushAllMessages()
users[1].disconnect()
@@ -388,13 +388,13 @@ const getUniqueNumber = () => _uniqueNumber++
const arrayTransactions = [
function insert (user, gen) {
const yarray = user.getArray('array')
var uniqueNumber = getUniqueNumber()
var content = []
var len = prng.int32(gen, 1, 4)
for (var i = 0; i < len; i++) {
const uniqueNumber = getUniqueNumber()
const content = []
const len = prng.int32(gen, 1, 4)
for (let i = 0; i < len; i++) {
content.push(uniqueNumber)
}
var pos = prng.int32(gen, 0, yarray.length)
const pos = prng.int32(gen, 0, yarray.length)
const oldContent = yarray.toArray()
yarray.insert(pos, content)
oldContent.splice(pos, 0, ...content)
@@ -402,28 +402,28 @@ const arrayTransactions = [
},
function insertTypeArray (user, gen) {
const yarray = user.getArray('array')
var pos = prng.int32(gen, 0, yarray.length)
const pos = prng.int32(gen, 0, yarray.length)
yarray.insert(pos, [new Y.Array()])
var array2 = yarray.get(pos)
const array2 = yarray.get(pos)
array2.insert(0, [1, 2, 3, 4])
},
function insertTypeMap (user, gen) {
const yarray = user.getArray('array')
var pos = prng.int32(gen, 0, yarray.length)
const pos = prng.int32(gen, 0, yarray.length)
yarray.insert(pos, [new Y.Map()])
var map = yarray.get(pos)
const map = yarray.get(pos)
map.set('someprop', 42)
map.set('someprop', 43)
map.set('someprop', 44)
},
function _delete (user, gen) {
const yarray = user.getArray('array')
var length = yarray.length
const length = yarray.length
if (length > 0) {
var somePos = prng.int32(gen, 0, length - 1)
var delLength = prng.int32(gen, 1, math.min(2, length - somePos))
let somePos = prng.int32(gen, 0, length - 1)
let delLength = prng.int32(gen, 1, math.min(2, length - somePos))
if (prng.bool(gen)) {
var type = yarray.get(somePos)
const type = yarray.get(somePos)
if (type.length > 0) {
somePos = prng.int32(gen, 0, type.length - 1)
delLength = prng.int32(gen, 0, math.min(2, type.length - somePos))

16
tests/y-map.tests.js
View File

@@ -138,7 +138,7 @@ export const testGetAndSetOfMapPropertySyncs = tc => {
t.compare(map0.get('stuff'), 'stuffy')
testConnector.flushAllMessages()
for (const user of users) {
var u = user.getMap('map')
const u = user.getMap('map')
t.compare(u.get('stuff'), 'stuffy')
}
compare(users)
@@ -153,7 +153,7 @@ export const testGetAndSetOfMapPropertyWithConflict = tc => {
map1.set('stuff', 'c1')
testConnector.flushAllMessages()
for (const user of users) {
var u = user.getMap('map')
const u = user.getMap('map')
t.compare(u.get('stuff'), 'c1')
}
compare(users)
@@ -183,7 +183,7 @@ export const testGetAndSetAndDeleteOfMapProperty = tc => {
map1.delete('stuff')
testConnector.flushAllMessages()
for (const user of users) {
var u = user.getMap('map')
const u = user.getMap('map')
t.assert(u.get('stuff') === undefined)
}
compare(users)
@@ -200,7 +200,7 @@ export const testGetAndSetOfMapPropertyWithThreeConflicts = tc => {
map2.set('stuff', 'c3')
testConnector.flushAllMessages()
for (const user of users) {
var u = user.getMap('map')
const u = user.getMap('map')
t.compare(u.get('stuff'), 'c3')
}
compare(users)
@@ -223,7 +223,7 @@ export const testGetAndSetAndDeleteOfMapPropertyWithThreeConflicts = tc => {
map3.delete('stuff')
testConnector.flushAllMessages()
for (const user of users) {
var u = user.getMap('map')
const u = user.getMap('map')
t.assert(u.get('stuff') === undefined)
}
compare(users)
@@ -296,7 +296,7 @@ export const testObserversUsingObservedeep = tc => {
* @param {Object<string,any>} should
*/
const compareEvent = (is, should) => {
for (var key in should) {
for (const key in should) {
t.compare(should[key], is[key])
}
}
@@ -474,12 +474,12 @@ export const testYmapEventHasCorrectValueWhenSettingAPrimitiveFromOtherUser = tc
const mapTransactions = [
function set (user, gen) {
const key = prng.oneOf(gen, ['one', 'two'])
var value = prng.utf16String(gen)
const value = prng.utf16String(gen)
user.getMap('map').set(key, value)
},
function setType (user, gen) {
const key = prng.oneOf(gen, ['one', 'two'])
var type = prng.oneOf(gen, [new Y.Array(), new Y.Map()])
const type = prng.oneOf(gen, [new Y.Array(), new Y.Map()])
user.getMap('map').set(key, type)
if (type instanceof Y.Array) {
type.insert(0, [1, 2, 3, 4])