From 7a111de186c9ba9f94819ed7684f6354646d71e7 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sun, 7 Apr 2019 23:08:08 +0200 Subject: [PATCH] refactor read/write of structs --- src/index.js | 4 +- src/structs/AbstractItem.js | 62 +++++++++-- src/structs/AbstractStruct.js | 7 +- src/structs/GC.js | 7 +- src/structs/ItemBinary.js | 52 ++++----- src/structs/ItemDeleted.js | 50 ++++----- src/structs/ItemEmbed.js | 52 ++++----- src/structs/ItemFormat.js | 54 ++++------ src/structs/ItemJSON.js | 50 ++++----- src/structs/ItemString.js | 51 ++++----- src/structs/ItemType.js | 52 ++++----- src/types/AbstractType.js | 1 + src/utils/DeleteSet.js | 20 +++- src/utils/StructStore.js | 12 ++- src/utils/structEncoding.js | 194 +++++++++++++++++++++++++++------- tests/encoding.tests.js | 36 +++++++ tests/index.js | 3 +- tests/testHelper.js | 11 +- tests/y-map.tests.js | 4 +- 19 files changed, 418 insertions(+), 304 deletions(-) create mode 100644 tests/encoding.tests.js diff --git a/src/index.js b/src/index.js index 44dbdcbf..281b92bb 100644 --- a/src/index.js +++ b/src/index.js @@ -29,5 +29,7 @@ export { writeStates, readDeleteSet, writeDeleteSet, - createDeleteSetFromStructStore + createDeleteSetFromStructStore, + writeModel, + readModel } from './internals.js' diff --git a/src/structs/AbstractItem.js b/src/structs/AbstractItem.js index df8eba31..ba8c02b5 100644 --- a/src/structs/AbstractItem.js +++ b/src/structs/AbstractItem.js @@ -14,7 +14,10 @@ import { findRootTypeKey, compareIDs, getItem, - StructStore, ID, AbstractType, Y, Transaction // eslint-disable-line + getItemType, + getItemCleanEnd, + getItemCleanStart, + YEvent, StructStore, ID, AbstractType, Y, Transaction // eslint-disable-line } from '../internals.js' import * as error from 'lib0/error.js' @@ -67,19 +70,10 @@ export class AbstractItem extends AbstractStruct { * @param {ID | null} origin * @param {AbstractItem | null} right * @param {ID | null} rightOrigin - * @param {AbstractType | null} parent + * @param {AbstractType} parent * @param {string | null} parentSub */ constructor (id, left, origin, right, rightOrigin, parent, parentSub) { - if (left !== null) { - parent = left.parent - parentSub = left.parentSub - } else if (right !== null) { - parent = right.parent - parentSub = right.parentSub - } else if (parent === null) { - throw error.unexpectedCase() - } super(id) /** * The item that was originally to the left of this item. @@ -575,3 +569,49 @@ export const changeItemRefOffset = (item, offset) => { item.id = createID(item.id.client, item.id.clock + offset) item.left = createID(item.id.client, item.id.clock - 1) } + +/** + * Outsourcing some of the logic of computing the item params from a received struct. + * If parent === null, it is expected to gc the read struct. Otherwise apply it. + * + * @param {Y} y + * @param {StructStore} store + * @param {ID|null} leftid + * @param {ID|null} rightid + * @param {ID|null} parentid + * @param {string|null} parentSub + * @param {string|null} parentYKey + * @return {{left:AbstractItem?,right:AbstractItem?,parent:AbstractType?,parentSub:string?}} + */ +export const computeItemParams = (y, store, leftid, rightid, parentid, parentSub, parentYKey) => { + const left = leftid === null ? null : getItemCleanEnd(store, leftid) + const right = rightid === null ? null : getItemCleanStart(store, rightid) + let parent = null + if (parentid !== null) { + const parentItem = getItemType(store, parentid) + switch (parentItem.constructor) { + case ItemDeleted: + case GC: + break + default: + parent = parentItem.type + } + } else if (parentYKey !== null) { + parent = y.get(parentYKey) + } else if (left !== null) { + if (left.constructor !== GC) { + parent = left.parent + parentSub = left.parentSub + } + } else if (right !== null) { + if (right.constructor !== GC) { + parent = right.parent + parentSub = right.parentSub + } + } else { + throw error.unexpectedCase() + } + return { + left, right, parent, parentSub + } +} diff --git a/src/structs/AbstractStruct.js b/src/structs/AbstractStruct.js index 2670cc9f..83e7ed60 100644 --- a/src/structs/AbstractStruct.js +++ b/src/structs/AbstractStruct.js @@ -1,6 +1,6 @@ import { - ID, Transaction // eslint-disable-line + Y, StructStore, ID, Transaction // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' // eslint-disable-line @@ -76,11 +76,12 @@ export class AbstractRef { return this._missing } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {AbstractStruct} */ - toStruct (transaction, offset) { + toStruct (y, store, offset) { throw error.methodUnimplemented() } /** diff --git a/src/structs/GC.js b/src/structs/GC.js index f2951b45..f79b3ecd 100644 --- a/src/structs/GC.js +++ b/src/structs/GC.js @@ -7,7 +7,7 @@ import { createID, writeID, addStruct, - Transaction, ID // eslint-disable-line + Y, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' import * as decoding from 'lib0/decoding.js' @@ -89,11 +89,12 @@ export class GCRef extends AbstractRef { ] } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {GC} */ - toStruct (transaction, offset) { + toStruct (y, store, offset) { if (offset > 0) { // @ts-ignore this.id = createID(this.id.client, this.id.clock + offset) diff --git a/src/structs/ItemBinary.js b/src/structs/ItemBinary.js index b790746f..96b67bc8 100644 --- a/src/structs/ItemBinary.js +++ b/src/structs/ItemBinary.js @@ -7,12 +7,9 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, + computeItemParams, GC, - ItemDeleted, - Transaction, ID, AbstractType // eslint-disable-line + StructStore, Y, AbstractType, ID, YEvent // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -74,37 +71,24 @@ export class ItemBinaryRef extends AbstractItemRef { this.content = decoding.readPayload(decoder) } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemBinary|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store - - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, 1) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - - return new ItemBinary( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.content - ) + toStruct (y, store, offset) { + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemBinary( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.content + ) } } diff --git a/src/structs/ItemDeleted.js b/src/structs/ItemDeleted.js index 8447e3ef..eb8c1655 100644 --- a/src/structs/ItemDeleted.js +++ b/src/structs/ItemDeleted.js @@ -7,14 +7,12 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, + computeItemParams, changeItemRefOffset, GC, splitItem, addToDeleteSet, - StructStore, Transaction, ID, AbstractType // eslint-disable-line + Y, StructStore, Transaction, ID, AbstractType // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -112,41 +110,29 @@ export class ItemDeletedRef extends AbstractItemRef { return this.len } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemDeleted|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store + toStruct (y, store, offset) { if (offset > 0) { changeItemRefOffset(this, offset) this.len = this.len - offset } - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, 1) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - - return new ItemDeleted( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.len - ) + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemDeleted( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.len + ) } } diff --git a/src/structs/ItemEmbed.js b/src/structs/ItemEmbed.js index 8b39a564..3f4a9719 100644 --- a/src/structs/ItemEmbed.js +++ b/src/structs/ItemEmbed.js @@ -5,12 +5,9 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, - ItemDeleted, + computeItemParams, GC, - Transaction, ID, AbstractType // eslint-disable-line + Y, StructStore, ID, AbstractType // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -69,37 +66,24 @@ export class ItemEmbedRef extends AbstractItemRef { this.embed = JSON.parse(decoding.readVarString(decoder)) } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemEmbed|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store - - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, 1) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - - return new ItemEmbed( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.embed - ) + toStruct (y, store, offset) { + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemEmbed( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.embed + ) } } diff --git a/src/structs/ItemFormat.js b/src/structs/ItemFormat.js index e2618abc..244aa295 100644 --- a/src/structs/ItemFormat.js +++ b/src/structs/ItemFormat.js @@ -5,12 +5,9 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, - ItemDeleted, + computeItemParams, GC, - Transaction, ID, AbstractType // eslint-disable-line + Y, StructStore, ID, AbstractType // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -76,38 +73,25 @@ export class ItemFormatRef extends AbstractItemRef { this.value = JSON.parse(decoding.readVarString(decoder)) } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemFormat|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store - - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, 1) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - - return new ItemFormat( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.key, - this.value - ) + toStruct (y, store, offset) { + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemFormat( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.key, + this.value + ) } } diff --git a/src/structs/ItemJSON.js b/src/structs/ItemJSON.js index 2db923c1..4bdccb71 100644 --- a/src/structs/ItemJSON.js +++ b/src/structs/ItemJSON.js @@ -5,14 +5,11 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, + computeItemParams, splitItem, changeItemRefOffset, GC, - ItemDeleted, - StructStore, Transaction, ID, AbstractType // eslint-disable-line + StructStore, Y, ID, AbstractType // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -122,39 +119,28 @@ export class ItemJSONRef extends AbstractItemRef { return this.content.length } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemJSON|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store + toStruct (y, store, offset) { if (offset > 0) { changeItemRefOffset(this, offset) this.content = this.content.slice(offset) } - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, this.content.length) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - return new ItemJSON( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.content - ) + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemJSON( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.content + ) } } diff --git a/src/structs/ItemString.js b/src/structs/ItemString.js index fa854dc2..5179ab69 100644 --- a/src/structs/ItemString.js +++ b/src/structs/ItemString.js @@ -4,14 +4,11 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, + computeItemParams, splitItem, changeItemRefOffset, - ItemDeleted, GC, - StructStore, Transaction, ID, AbstractType // eslint-disable-line + StructStore, Y, ID, AbstractType // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -108,41 +105,29 @@ export class ItemStringRef extends AbstractItemRef { return this.string.length } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemString|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store + toStruct (y, store, offset) { if (offset > 0) { changeItemRefOffset(this, offset) this.string = this.string.slice(offset) } - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, this.string.length) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - - return new ItemString( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.string - ) + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemString( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.string + ) } } diff --git a/src/structs/ItemType.js b/src/structs/ItemType.js index 07435c9c..ea30a6c7 100644 --- a/src/structs/ItemType.js +++ b/src/structs/ItemType.js @@ -7,9 +7,7 @@ import { AbstractItem, AbstractItemRef, - getItemCleanEnd, - getItemCleanStart, - getItemType, + computeItemParams, readYArray, readYMap, readYText, @@ -17,7 +15,7 @@ import { readYXmlFragment, readYXmlHook, readYXmlText, - Y, GC, ItemDeleted, Transaction, ID, AbstractType // eslint-disable-line + StructStore, Y, GC, ItemDeleted, Transaction, ID, AbstractType // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' // eslint-disable-line @@ -171,38 +169,24 @@ export class ItemTypeRef extends AbstractItemRef { this.type = typeRefs[typeRef](decoder) } /** - * @param {Transaction} transaction + * @param {Y} y + * @param {StructStore} store * @param {number} offset * @return {ItemType|GC} */ - toStruct (transaction, offset) { - const y = transaction.y - const store = y.store - - let parent - if (this.parent !== null) { - const parentItem = getItemType(store, this.parent) - switch (parentItem.constructor) { - case ItemDeleted: - case GC: - return new GC(this.id, 1) - } - parent = parentItem.type - } else { - // @ts-ignore - parent = y.get(this.parentYKey) - } - - // TODO: we can probably only feed AbstractType with origins - return new ItemType( - this.id, - this.left === null ? null : getItemCleanEnd(store, this.left), - this.left, - this.right === null ? null : getItemCleanStart(store, this.right), - this.right, - parent, - this.parentSub, - this.type - ) + toStruct (y, store, offset) { + const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey) + return parent === null + ? new GC(this.id, this.length) + : new ItemType( + this.id, + left, + this.left, + right, + this.right, + parent, + parentSub, + this.type + ) } } diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 20fee63f..ee4c5c81 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -267,6 +267,7 @@ export const typeArrayCreateIterator = type => { // we found n, so we can set currentContent currentContent = n.getContent() currentContentIndex = 0 + n = n.right // we used the content of n, now iterate to next } const value = currentContent[currentContentIndex++] // check if we need to empty currentContent diff --git a/src/utils/DeleteSet.js b/src/utils/DeleteSet.js index 6801eb28..c02440e1 100644 --- a/src/utils/DeleteSet.js +++ b/src/utils/DeleteSet.js @@ -1,6 +1,8 @@ import { findIndexSS, + createID, + getState, AbstractItem, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' @@ -163,21 +165,24 @@ export const writeDeleteSet = (encoder, ds) => { /** * @param {decoding.Decoder} decoder - * @param {StructStore} store * @param {Transaction} transaction + * @param {StructStore} store */ -export const readDeleteSet = (decoder, store, transaction) => { +export const readDeleteSet = (decoder, transaction, store) => { + const unappliedDS = new DeleteSet() const numClients = decoding.readVarUint(decoder) for (let i = 0; i < numClients; i++) { const client = decoding.readVarUint(decoder) const numberOfDeletes = decoding.readVarUint(decoder) const structs = store.clients.get(client) || [] - const lastStruct = structs[structs.length - 1] - const state = lastStruct.id.clock + lastStruct.length + const state = getState(store, client) for (let i = 0; i < numberOfDeletes; i++) { const clock = decoding.readVarUint(decoder) const len = decoding.readVarUint(decoder) if (clock < state) { + if (state < clock + len) { + addToDeleteSet(unappliedDS, createID(client, state), clock + len - state) + } let index = findIndexSS(structs, clock) /** * We can ignore the case of GC and Delete structs, because we are going to skip them @@ -206,7 +211,14 @@ export const readDeleteSet = (decoder, store, transaction) => { break } } + } else { + addToDeleteSet(unappliedDS, createID(client, state), len) } } } + if (unappliedDS.clients.size > 0) { + const unappliedDSEncoder = encoding.createEncoder() + writeDeleteSet(unappliedDSEncoder, unappliedDS) + store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toBuffer(unappliedDSEncoder))) + } } diff --git a/src/utils/StructStore.js b/src/utils/StructStore.js index 601b46e7..44fb523f 100644 --- a/src/utils/StructStore.js +++ b/src/utils/StructStore.js @@ -1,6 +1,6 @@ import { - Transaction, ID, ItemType, AbstractItem, AbstractStruct // eslint-disable-line + AbstractRef, ID, ItemType, AbstractItem, AbstractStruct // eslint-disable-line } from '../internals.js' import * as math from 'lib0/math.js' @@ -14,6 +14,16 @@ export class StructStore { * @type {Map>} */ this.clients = new Map() + /** + * Store uncompleted struct readers here + * @see tryResumePendingReaders + * @type {Set<{stack:Array,structReaders:Map>,missing:ID}>} + */ + this.pendingStructReaders = new Set() + /** + * @type {Array} + */ + this.pendingDeleteReaders = [] } } diff --git a/src/utils/structEncoding.js b/src/utils/structEncoding.js index e324c00a..65ae50e5 100644 --- a/src/utils/structEncoding.js +++ b/src/utils/structEncoding.js @@ -2,8 +2,8 @@ import { findIndexSS, exists, - ItemBinaryRef, GCRef, + ItemBinaryRef, ItemDeletedRef, ItemEmbedRef, ItemFormatRef, @@ -15,6 +15,9 @@ import { readID, getState, getStates, + readDeleteSet, + writeDeleteSet, + createDeleteSetFromStructStore, Transaction, AbstractStruct, AbstractRef, StructStore, ID // eslint-disable-line } from '../internals.js' @@ -28,9 +31,9 @@ import * as iterator from 'lib0/iterator.js' * @typedef {Map} StateMap */ -const structRefs = [ - ItemBinaryRef, +export const structRefs = [ GCRef, + ItemBinaryRef, ItemDeletedRef, ItemEmbedRef, ItemFormatRef, @@ -44,7 +47,7 @@ const structRefs = [ * @param {number} structsLen * @param {ID} nextID * @param {number} localState next expected clock by nextID.client - * @return {Iterator} + * @return {IterableIterator} */ const createStructReaderIterator = (decoder, structsLen, nextID, localState) => iterator.createIterator(() => { let done = false @@ -83,6 +86,7 @@ export const writeStructs = (encoder, store, _sm) => { // we use it in readStructs to jump ahead to the end of the message encoding.writeUint32(encoder, 0) _sm.forEach((clock, client) => { + // only write if new structs are available if (getState(store, client) > clock) { sm.set(client, clock) } @@ -125,30 +129,17 @@ export const writeStructs = (encoder, store, _sm) => { } /** - * Read the next Item in a Decoder and fill this Item with the read data. - * - * This is called when data is received from a remote peer. - * * @param {decoding.Decoder} decoder The decoder object to read data from. - * @param {Transaction} transaction - * @param {StructStore} store - * - * @private + * @param {Map} localState + * @return {Map>} */ -export const readStructs = (decoder, transaction, store) => { +const readStructReaders = (decoder, localState) => { /** - * @type {Map>} + * @type {Map>} */ const structReaders = new Map() const endOfMessagePos = decoder.pos + decoding.readUint32(decoder) const clientbeforeState = decoding.readVarUint(decoder) - /** - * Stack of pending structs waiting for struct dependencies. - * Maximum length of stack is structReaders.size. - * @type {Array} - */ - const stack = [] - const localState = getStates(store) for (let i = 0; i < clientbeforeState; i++) { const nextID = readID(decoder) const decoderPos = decoder.pos + decoding.readUint32(decoder) @@ -160,29 +151,152 @@ export const readStructs = (decoder, transaction, store) => { // Jump ahead to end of message so that reading can continue. // We will use the created struct readers for the remaining part of this workflow. decoder.pos = endOfMessagePos - for (const it of structReaders.values()) { - // todo try for in of it - for (let res = it.next(); !res.done; res = it.next()) { - stack.push(res.value) - while (stack.length > 0) { - const ref = stack[stack.length - 1] - const m = ref._missing - while (m.length > 0) { - const nextMissing = m[m.length - 1] - if (!exists(store, nextMissing)) { - // @ts-ignore must not be undefined, otherwise unexpected case - stack.push(structReaders.get(nextMissing.client).next().value) - break + return structReaders +} + +/** + * Resume computing structs generated by struct readers. + * + * While there is something to do, we integrate structs in this order + * 1. top element on stack, if stack is not empty + * 2. next element from current struct reader (if empty, use next struct reader) + * + * If struct causally depends on another struct (ref.missing), we put next reader of + * `ref.id.client` on top of stack. + * + * At some point we find a struct that has no causal dependencies, + * then we start emptying the stack. + * + * It is not possible to have circles: i.e. struct1 (from client1) depends on struct2 (from client2) + * depends on struct3 (from client1). Therefore the max stack size is eqaul to `structReaders.length`. + * + * This method is implemented in a way so that we can resume computation if this update + * causally depends on another update. + * + * @param {Transaction} transaction + * @param {StructStore} store + * @param {Map} localState + * @param {Map>} structReaders + * @param {Array} stack Stack of pending structs waiting for struct dependencies. + * Maximum length of stack is structReaders.size. + * + * @todo reimplement without iterators - read everything in arrays instead + */ +const execStructReaders = (transaction, store, localState, structReaders, stack) => { + // iterate over all struct readers until we are done + const structReaderIterator = structReaders.values() + let structReaderIteratorResult = structReaderIterator.next() + while (stack.length !== 0 || !structReaderIteratorResult.done) { + if (stack.length === 0) { + // stack is empty. We know that there there are more structReaders to be processed + const nextStructRes = structReaderIteratorResult.value.next() + if (nextStructRes.done) { + // current structReaderIteratorResult is empty, use next one + structReaderIteratorResult = structReaderIterator.next() + } else { + stack.push(nextStructRes.value) + } + } else { + const ref = stack[stack.length - 1] + const m = ref._missing + while (m.length > 0) { + const missing = m[m.length - 1] + if (!exists(store, missing)) { + // get the struct reader that has the missing struct + const reader = structReaders.get(missing.client) + const nextRef = reader === undefined ? undefined : reader.next().value + if (nextRef === undefined) { + // This update message causally depends on another update message. + // Store current stack and readers in StructStore and resume the computation at another time + store.pendingStructReaders.add({ stack, structReaders, missing }) + return } - ref._missing.pop() + stack.push(nextRef) + break } - if (m.length === 0) { - const localClock = (localState.get(ref.id.client) || 0) - const offset = ref.id.clock < localClock ? localClock - ref.id.clock : 0 - ref.toStruct(transaction, offset).integrate(transaction) - stack.pop() + ref._missing.pop() + } + if (m.length === 0) { + const localClock = (localState.get(ref.id.client) || 0) + const offset = ref.id.clock < localClock ? localClock - ref.id.clock : 0 + if (offset < ref.length) { + ref.toStruct(transaction.y, store, offset).integrate(transaction) } + stack.pop() } } } } + +/** + * Try to resume pending struct readers in `store.pendingReaders` while `pendingReaders.nextMissing` + * exists. + * + * @param {Transaction} transaction + * @param {StructStore} store + */ +const tryResumePendingStructReaders = (transaction, store) => { + let resume = true + const pendingReaders = store.pendingStructReaders + while (resume) { + resume = false + for (const pendingReader of pendingReaders) { + if (exists(store, pendingReader.missing)) { + resume = true // found at least one more reader to execute + pendingReaders.delete(pendingReader) + execStructReaders(transaction, store, getStates(store), pendingReader.structReaders, pendingReader.stack) + } + } + } +} + +/** + * @param {Transaction} transaction + * @param {StructStore} store + */ +export const tryResumePendingDeleteReaders = (transaction, store) => { + const pendingReaders = store.pendingDeleteReaders + store.pendingDeleteReaders = [] + for (let i = 0; i < pendingReaders.length; i++) { + readDeleteSet(pendingReaders[i], transaction, store) + } +} + +/** + * Read the next Item in a Decoder and fill this Item with the read data. + * + * This is called when data is received from a remote peer. + * + * @param {decoding.Decoder} decoder The decoder object to read data from. + * @param {Transaction} transaction + * @param {StructStore} store + * + * @private + */ +export const readStructs = (decoder, transaction, store) => { + const localState = getStates(store) + const readers = readStructReaders(decoder, localState) + execStructReaders(transaction, store, localState, readers, []) + tryResumePendingStructReaders(transaction, store) + tryResumePendingDeleteReaders(transaction, store) +} + +/** + * @param {decoding.Decoder} decoder + * @param {Transaction} transaction + * @param {StructStore} store + */ +export const readModel = (decoder, transaction, store) => { + readStructs(decoder, transaction, store) + readDeleteSet(decoder, transaction, store) +} + +/** + * @param {encoding.Encoder} encoder + * @param {StructStore} store + * @param {Map} [targetState] The state of the target that receives the update. Leave empty to write all known structs + */ +export const writeModel = (encoder, store, targetState = new Map()) => { + writeStructs(encoder, store, targetState) + writeDeleteSet(encoder, createDeleteSetFromStructStore(store)) +} diff --git a/tests/encoding.tests.js b/tests/encoding.tests.js new file mode 100644 index 00000000..45d2e5e7 --- /dev/null +++ b/tests/encoding.tests.js @@ -0,0 +1,36 @@ +import * as t from 'lib0/testing.js' + +import { + structRefs, + structGCRefNumber, + structBinaryRefNumber, + structDeletedRefNumber, + structEmbedRefNumber, + structFormatRefNumber, + structJSONRefNumber, + structStringRefNumber, + structTypeRefNumber, + GCRef, + ItemBinaryRef, + ItemDeletedRef, + ItemEmbedRef, + ItemFormatRef, + ItemJSONRef, + ItemStringRef, + ItemTypeRef +} from '../src/internals.js' + +/** + * @param {t.TestCase} tc + */ +export const testStructReferences = tc => { + t.assert(structRefs.length === 8) + t.assert(structRefs[structGCRefNumber] === GCRef) + t.assert(structRefs[structBinaryRefNumber] === ItemBinaryRef) + t.assert(structRefs[structDeletedRefNumber] === ItemDeletedRef) + t.assert(structRefs[structEmbedRefNumber] === ItemEmbedRef) + t.assert(structRefs[structFormatRefNumber] === ItemFormatRef) + t.assert(structRefs[structJSONRefNumber] === ItemJSONRef) + t.assert(structRefs[structStringRefNumber] === ItemStringRef) + t.assert(structRefs[structTypeRefNumber] === ItemTypeRef) +} diff --git a/tests/index.js b/tests/index.js index ca509794..d2b8e9ef 100644 --- a/tests/index.js +++ b/tests/index.js @@ -3,6 +3,7 @@ import * as array from './y-array.tests.js' import * as map from './y-map.tests.js' import * as text from './y-text.tests.js' import * as xml from './y-xml.tests.js' +import * as encoding from './encoding.tests.js' import { runTests } from 'lib0/testing.js' import { isBrowser, isNode } from 'lib0/environment.js' @@ -12,7 +13,7 @@ if (isBrowser) { log.createVConsole(document.body) } runTests({ - map, array, text, xml + map, array, text, xml, encoding }).then(success => { /* istanbul ignore next */ if (isNode) { diff --git a/tests/testHelper.js b/tests/testHelper.js index 2bd991ac..a73e11e1 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -81,14 +81,14 @@ export class TestYInstance extends Y.Y { if (!this.tc.onlineConns.has(this)) { this.tc.onlineConns.add(this) const encoder = encoding.createEncoder() - syncProtocol.writeSyncStep1(encoder, this) + syncProtocol.writeSyncStep1(encoder, this.store) // publish SyncStep1 broadcastMessage(this, encoding.toBuffer(encoder)) this.tc.onlineConns.forEach(remoteYInstance => { if (remoteYInstance !== this) { // remote instance sends instance to this instance const encoder = encoding.createEncoder() - syncProtocol.writeSyncStep1(encoder, remoteYInstance) + syncProtocol.writeSyncStep1(encoder, remoteYInstance.store) this._receive(encoding.toBuffer(encoder), remoteYInstance) } }) @@ -271,8 +271,11 @@ export const compare = users => { const userMapValues = users.map(u => u.getMap('map').toJSON()) const userXmlValues = users.map(u => u.get('xml', Y.XmlElement).toString()) const userTextValues = users.map(u => u.getText('text').toDelta()) - for (var i = 0; i < users.length - 1; i++) { - t.describe(`Comparing user${i} with user${i + 1}`) + for (const u of users) { + t.assert(u.store.pendingDeleteReaders.length === 0) + t.assert(u.store.pendingStructReaders.size === 0) + } + for (let i = 0; i < users.length - 1; i++) { t.compare(userArrayValues[i].length, users[i].getArray('array').length) t.compare(userArrayValues[i], userArrayValues[i + 1]) t.compare(userMapValues[i], userMapValues[i + 1]) diff --git a/tests/y-map.tests.js b/tests/y-map.tests.js index c50fbd5a..20abb0e4 100644 --- a/tests/y-map.tests.js +++ b/tests/y-map.tests.js @@ -221,7 +221,7 @@ export const testObserveDeepProperties = tc => { export const testObserversUsingObservedeep = tc => { const { users, map0 } = init(tc, { users: 2 }) /** - * @type {Array>} + * @type {Array>} */ const pathes = [] let calls = 0 @@ -348,7 +348,7 @@ const mapTransactions = [ * @param {t.TestCase} tc */ export const testRepeatGeneratingYmapTests20 = tc => { - applyRandomTests(tc, mapTransactions, 20) + applyRandomTests(tc, mapTransactions, 8) } /**