refactor read/write of structs

src/index.js

@@ -29,5 +29,7 @@ export {
   writeStates,
   readDeleteSet,
   writeDeleteSet,
-  createDeleteSetFromStructStore
+  createDeleteSetFromStructStore,
+  writeModel,
+  readModel
 } from './internals.js'

src/structs/AbstractItem.js

@@ -14,7 +14,10 @@ import {
   findRootTypeKey,
   compareIDs,
   getItem,
-  StructStore, ID, AbstractType, Y, Transaction // eslint-disable-line
+  getItemType,
+  getItemCleanEnd,
+  getItemCleanStart,
+  YEvent, StructStore, ID, AbstractType, Y, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as error from 'lib0/error.js'
@@ -67,19 +70,10 @@ export class AbstractItem extends AbstractStruct {
    * @param {ID | null} origin
    * @param {AbstractItem | null} right
    * @param {ID | null} rightOrigin
-   * @param {AbstractType<any> | null} parent
+   * @param {AbstractType<any>} parent
    * @param {string | null} parentSub
    */
   constructor (id, left, origin, right, rightOrigin, parent, parentSub) {
-    if (left !== null) {
-      parent = left.parent
-      parentSub = left.parentSub
-    } else if (right !== null) {
-      parent = right.parent
-      parentSub = right.parentSub
-    } else if (parent === null) {
-      throw error.unexpectedCase()
-    }
     super(id)
     /**
      * The item that was originally to the left of this item.
@@ -575,3 +569,49 @@ export const changeItemRefOffset = (item, offset) => {
   item.id = createID(item.id.client, item.id.clock + offset)
   item.left = createID(item.id.client, item.id.clock - 1)
 }
+
+/**
+ * Outsourcing some of the logic of computing the item params from a received struct.
+ * If parent === null, it is expected to gc the read struct. Otherwise apply it.
+ *
+ * @param {Y} y
+ * @param {StructStore} store
+ * @param {ID|null} leftid
+ * @param {ID|null} rightid
+ * @param {ID|null} parentid
+ * @param {string|null} parentSub
+ * @param {string|null} parentYKey
+ * @return {{left:AbstractItem?,right:AbstractItem?,parent:AbstractType<YEvent>?,parentSub:string?}}
+ */
+export const computeItemParams = (y, store, leftid, rightid, parentid, parentSub, parentYKey) => {
+  const left = leftid === null ? null : getItemCleanEnd(store, leftid)
+  const right = rightid === null ? null : getItemCleanStart(store, rightid)
+  let parent = null
+  if (parentid !== null) {
+    const parentItem = getItemType(store, parentid)
+    switch (parentItem.constructor) {
+      case ItemDeleted:
+      case GC:
+        break
+      default:
+        parent = parentItem.type
+    }
+  } else if (parentYKey !== null) {
+    parent = y.get(parentYKey)
+  } else if (left !== null) {
+    if (left.constructor !== GC) {
+      parent = left.parent
+      parentSub = left.parentSub
+    }
+  } else if (right !== null) {
+    if (right.constructor !== GC) {
+      parent = right.parent
+      parentSub = right.parentSub
+    }
+  } else {
+    throw error.unexpectedCase()
+  }
+  return {
+    left, right, parent, parentSub
+  }
+}

src/structs/AbstractStruct.js

@@ -1,6 +1,6 @@
 
 import {
-  ID, Transaction // eslint-disable-line
+  Y, StructStore, ID, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js' // eslint-disable-line
@@ -76,11 +76,12 @@ export class AbstractRef {
     return this._missing
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {AbstractStruct}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
     throw error.methodUnimplemented()
   }
   /**

src/structs/GC.js

@@ -7,7 +7,7 @@ import {
   createID,
   writeID,
   addStruct,
-  Transaction, ID // eslint-disable-line
+  Y, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as decoding from 'lib0/decoding.js'
@@ -89,11 +89,12 @@ export class GCRef extends AbstractRef {
     ]
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
     if (offset > 0) {
       // @ts-ignore
       this.id = createID(this.id.client, this.id.clock + offset)

src/structs/ItemBinary.js

@@ -7,12 +7,9 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
   GC,
-  ItemDeleted,
+  StructStore, Y, AbstractType, ID, YEvent // eslint-disable-line
-  Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -74,37 +71,24 @@ export class ItemBinaryRef extends AbstractItemRef {
     this.content = decoding.readPayload(decoder)
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemBinary|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    const store = y.store
+    return parent === null
-
+      ? new GC(this.id, this.length)
-    let parent
+      : new ItemBinary(
-    if (this.parent !== null) {
+        this.id,
-      const parentItem = getItemType(store, this.parent)
+        left,
-      switch (parentItem.constructor) {
+        this.left,
-        case ItemDeleted:
+        right,
-        case GC:
+        this.right,
-          return new GC(this.id, 1)
+        parent,
-      }
+        parentSub,
-      parent = parentItem.type
+        this.content
-    } else {
+      )
-      // @ts-ignore
-      parent = y.get(this.parentYKey)
-    }
-
-    return new ItemBinary(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.content
-    )
   }
 }

src/structs/ItemDeleted.js

@@ -7,14 +7,12 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
   changeItemRefOffset,
   GC,
   splitItem,
   addToDeleteSet,
-  StructStore, Transaction, ID, AbstractType // eslint-disable-line
+  Y, StructStore, Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -112,41 +110,29 @@ export class ItemDeletedRef extends AbstractItemRef {
     return this.len
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemDeleted|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
-    const store = y.store
     if (offset > 0) {
       changeItemRefOffset(this, offset)
       this.len = this.len - offset
     }
 
-    let parent
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    if (this.parent !== null) {
+    return parent === null
-      const parentItem = getItemType(store, this.parent)
+      ? new GC(this.id, this.length)
-      switch (parentItem.constructor) {
+      : new ItemDeleted(
-        case ItemDeleted:
+        this.id,
-        case GC:
+        left,
-          return new GC(this.id, 1)
+        this.left,
-      }
+        right,
-      parent = parentItem.type
+        this.right,
-    } else {
+        parent,
-      // @ts-ignore
+        parentSub,
-      parent = y.get(this.parentYKey)
+        this.len
-    }
+      )
-
-    return new ItemDeleted(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.len
-    )
   }
 }

src/structs/ItemEmbed.js

@@ -5,12 +5,9 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
-  ItemDeleted,
   GC,
-  Transaction, ID, AbstractType // eslint-disable-line
+  Y, StructStore, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -69,37 +66,24 @@ export class ItemEmbedRef extends AbstractItemRef {
     this.embed = JSON.parse(decoding.readVarString(decoder))
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemEmbed|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    const store = y.store
+    return parent === null
-
+      ? new GC(this.id, this.length)
-    let parent
+      : new ItemEmbed(
-    if (this.parent !== null) {
+        this.id,
-      const parentItem = getItemType(store, this.parent)
+        left,
-      switch (parentItem.constructor) {
+        this.left,
-        case ItemDeleted:
+        right,
-        case GC:
+        this.right,
-          return new GC(this.id, 1)
+        parent,
-      }
+        parentSub,
-      parent = parentItem.type
+        this.embed
-    } else {
+      )
-      // @ts-ignore
-      parent = y.get(this.parentYKey)
-    }
-
-    return new ItemEmbed(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.embed
-    )
   }
 }

src/structs/ItemFormat.js

@@ -5,12 +5,9 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
-  ItemDeleted,
   GC,
-  Transaction, ID, AbstractType // eslint-disable-line
+  Y, StructStore, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -76,38 +73,25 @@ export class ItemFormatRef extends AbstractItemRef {
     this.value = JSON.parse(decoding.readVarString(decoder))
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemFormat|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    const store = y.store
+    return parent === null
-
+      ? new GC(this.id, this.length)
-    let parent
+      : new ItemFormat(
-    if (this.parent !== null) {
+        this.id,
-      const parentItem = getItemType(store, this.parent)
+        left,
-      switch (parentItem.constructor) {
+        this.left,
-        case ItemDeleted:
+        right,
-        case GC:
+        this.right,
-          return new GC(this.id, 1)
+        parent,
-      }
+        parentSub,
-      parent = parentItem.type
+        this.key,
-    } else {
+        this.value
-      // @ts-ignore
+      )
-      parent = y.get(this.parentYKey)
-    }
-
-    return new ItemFormat(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.key,
-      this.value
-    )
   }
 }

src/structs/ItemJSON.js

@@ -5,14 +5,11 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
   splitItem,
   changeItemRefOffset,
   GC,
-  ItemDeleted,
+  StructStore, Y, ID, AbstractType // eslint-disable-line
-  StructStore, Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -122,39 +119,28 @@ export class ItemJSONRef extends AbstractItemRef {
     return this.content.length
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemJSON|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
-    const store = y.store
     if (offset > 0) {
       changeItemRefOffset(this, offset)
       this.content = this.content.slice(offset)
     }
-    let parent
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    if (this.parent !== null) {
+    return parent === null
-      const parentItem = getItemType(store, this.parent)
+      ? new GC(this.id, this.length)
-      switch (parentItem.constructor) {
+      : new ItemJSON(
-        case ItemDeleted:
+        this.id,
-        case GC:
+        left,
-          return new GC(this.id, this.content.length)
+        this.left,
-      }
+        right,
-      parent = parentItem.type
+        this.right,
-    } else {
+        parent,
-      // @ts-ignore
+        parentSub,
-      parent = y.get(this.parentYKey)
+        this.content
-    }
+      )
-    return new ItemJSON(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.content
-    )
   }
 }

src/structs/ItemString.js

@@ -4,14 +4,11 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
   splitItem,
   changeItemRefOffset,
-  ItemDeleted,
   GC,
-  StructStore, Transaction, ID, AbstractType // eslint-disable-line
+  StructStore, Y, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
@@ -108,41 +105,29 @@ export class ItemStringRef extends AbstractItemRef {
     return this.string.length
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemString|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
-    const store = y.store
     if (offset > 0) {
       changeItemRefOffset(this, offset)
       this.string = this.string.slice(offset)
     }
 
-    let parent
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    if (this.parent !== null) {
+    return parent === null
-      const parentItem = getItemType(store, this.parent)
+      ? new GC(this.id, this.length)
-      switch (parentItem.constructor) {
+      : new ItemString(
-        case ItemDeleted:
+        this.id,
-        case GC:
+        left,
-          return new GC(this.id, this.string.length)
+        this.left,
-      }
+        right,
-      parent = parentItem.type
+        this.right,
-    } else {
+        parent,
-      // @ts-ignore
+        parentSub,
-      parent = y.get(this.parentYKey)
+        this.string
-    }
+      )
-
-    return new ItemString(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.string
-    )
   }
 }

src/structs/ItemType.js

@@ -7,9 +7,7 @@
 import {
   AbstractItem,
   AbstractItemRef,
-  getItemCleanEnd,
+  computeItemParams,
-  getItemCleanStart,
-  getItemType,
   readYArray,
   readYMap,
   readYText,
@@ -17,7 +15,7 @@ import {
   readYXmlFragment,
   readYXmlHook,
   readYXmlText,
-  Y, GC, ItemDeleted, Transaction, ID, AbstractType // eslint-disable-line
+  StructStore, Y, GC, ItemDeleted, Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js' // eslint-disable-line
@@ -171,38 +169,24 @@ export class ItemTypeRef extends AbstractItemRef {
     this.type = typeRefs[typeRef](decoder)
   }
   /**
-   * @param {Transaction} transaction
+   * @param {Y} y
+   * @param {StructStore} store
    * @param {number} offset
    * @return {ItemType|GC}
    */
-  toStruct (transaction, offset) {
+  toStruct (y, store, offset) {
-    const y = transaction.y
+    const { left, right, parent, parentSub } = computeItemParams(y, store, this.left, this.right, this.parent, this.parentSub, this.parentYKey)
-    const store = y.store
+    return parent === null
-
+      ? new GC(this.id, this.length)
-    let parent
+      : new ItemType(
-    if (this.parent !== null) {
+        this.id,
-      const parentItem = getItemType(store, this.parent)
+        left,
-      switch (parentItem.constructor) {
+        this.left,
-        case ItemDeleted:
+        right,
-        case GC:
+        this.right,
-          return new GC(this.id, 1)
+        parent,
-      }
+        parentSub,
-      parent = parentItem.type
+        this.type
-    } else {
+      )
-      // @ts-ignore
-      parent = y.get(this.parentYKey)
-    }
-
-    // TODO: we can probably only feed AbstractType with origins
-    return new ItemType(
-      this.id,
-      this.left === null ? null : getItemCleanEnd(store, this.left),
-      this.left,
-      this.right === null ? null : getItemCleanStart(store, this.right),
-      this.right,
-      parent,
-      this.parentSub,
-      this.type
-    )
   }
 }

src/types/AbstractType.js

@@ -267,6 +267,7 @@ export const typeArrayCreateIterator = type => {
         // we found n, so we can set currentContent
         currentContent = n.getContent()
         currentContentIndex = 0
+        n = n.right // we used the content of n, now iterate to next
       }
       const value = currentContent[currentContentIndex++]
       // check if we need to empty currentContent

src/utils/DeleteSet.js

@@ -1,6 +1,8 @@
 
 import {
   findIndexSS,
+  createID,
+  getState,
   AbstractItem, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
@@ -163,21 +165,24 @@ export const writeDeleteSet = (encoder, ds) => {
 
 /**
  * @param {decoding.Decoder} decoder
- * @param {StructStore} store
  * @param {Transaction} transaction
+ * @param {StructStore} store
  */
-export const readDeleteSet = (decoder, store, transaction) => {
+export const readDeleteSet = (decoder, transaction, store) => {
+  const unappliedDS = new DeleteSet()
   const numClients = decoding.readVarUint(decoder)
   for (let i = 0; i < numClients; i++) {
     const client = decoding.readVarUint(decoder)
     const numberOfDeletes = decoding.readVarUint(decoder)
     const structs = store.clients.get(client) || []
-    const lastStruct = structs[structs.length - 1]
+    const state = getState(store, client)
-    const state = lastStruct.id.clock + lastStruct.length
     for (let i = 0; i < numberOfDeletes; i++) {
       const clock = decoding.readVarUint(decoder)
       const len = decoding.readVarUint(decoder)
       if (clock < state) {
+        if (state < clock + len) {
+          addToDeleteSet(unappliedDS, createID(client, state), clock + len - state)
+        }
         let index = findIndexSS(structs, clock)
         /**
          * We can ignore the case of GC and Delete structs, because we are going to skip them
@@ -206,7 +211,14 @@ export const readDeleteSet = (decoder, store, transaction) => {
             break
           }
         }
+      } else {
+        addToDeleteSet(unappliedDS, createID(client, state), len)
       }
     }
   }
+  if (unappliedDS.clients.size > 0) {
+    const unappliedDSEncoder = encoding.createEncoder()
+    writeDeleteSet(unappliedDSEncoder, unappliedDS)
+    store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toBuffer(unappliedDSEncoder)))
+  }
 }

src/utils/StructStore.js

@@ -1,6 +1,6 @@
 
 import {
-  Transaction, ID, ItemType, AbstractItem, AbstractStruct // eslint-disable-line
+  AbstractRef, ID, ItemType, AbstractItem, AbstractStruct // eslint-disable-line
 } from '../internals.js'
 
 import * as math from 'lib0/math.js'
@@ -14,6 +14,16 @@ export class StructStore {
      * @type {Map<number,Array<AbstractStruct>>}
      */
     this.clients = new Map()
+    /**
+     * Store uncompleted struct readers here
+     * @see tryResumePendingReaders
+     * @type {Set<{stack:Array<AbstractRef>,structReaders:Map<number,IterableIterator<AbstractRef>>,missing:ID}>}
+     */
+    this.pendingStructReaders = new Set()
+    /**
+     * @type {Array<decoding.Decoder>}
+     */
+    this.pendingDeleteReaders = []
   }
 }
 

src/utils/structEncoding.js

@@ -2,8 +2,8 @@
 import {
   findIndexSS,
   exists,
-  ItemBinaryRef,
   GCRef,
+  ItemBinaryRef,
   ItemDeletedRef,
   ItemEmbedRef,
   ItemFormatRef,
@@ -15,6 +15,9 @@ import {
   readID,
   getState,
   getStates,
+  readDeleteSet,
+  writeDeleteSet,
+  createDeleteSetFromStructStore,
   Transaction, AbstractStruct, AbstractRef, StructStore, ID // eslint-disable-line
 } from '../internals.js'
 
@@ -28,9 +31,9 @@ import * as iterator from 'lib0/iterator.js'
  * @typedef {Map<number, number>} StateMap
  */
 
-const structRefs = [
+export const structRefs = [
-  ItemBinaryRef,
   GCRef,
+  ItemBinaryRef,
   ItemDeletedRef,
   ItemEmbedRef,
   ItemFormatRef,
@@ -44,7 +47,7 @@ const structRefs = [
  * @param {number} structsLen
  * @param {ID} nextID
  * @param {number} localState next expected clock by nextID.client
- * @return {Iterator<AbstractRef>}
+ * @return {IterableIterator<AbstractRef>}
  */
 const createStructReaderIterator = (decoder, structsLen, nextID, localState) => iterator.createIterator(() => {
   let done = false
@@ -83,6 +86,7 @@ export const writeStructs = (encoder, store, _sm) => {
   // we use it in readStructs to jump ahead to the end of the message
   encoding.writeUint32(encoder, 0)
   _sm.forEach((clock, client) => {
+    // only write if new structs are available
     if (getState(store, client) > clock) {
       sm.set(client, clock)
     }
@@ -125,30 +129,17 @@ export const writeStructs = (encoder, store, _sm) => {
 }
 
 /**
- * Read the next Item in a Decoder and fill this Item with the read data.
- *
- * This is called when data is received from a remote peer.
- *
  * @param {decoding.Decoder} decoder The decoder object to read data from.
- * @param {Transaction} transaction
+ * @param {Map<number,number>} localState
- * @param {StructStore} store
+ * @return {Map<number,IterableIterator<AbstractRef>>}
- *
- * @private
  */
-export const readStructs = (decoder, transaction, store) => {
+const readStructReaders = (decoder, localState) => {
   /**
-   * @type {Map<number,Iterator<AbstractRef>>}
+   * @type {Map<number,IterableIterator<AbstractRef>>}
    */
   const structReaders = new Map()
   const endOfMessagePos = decoder.pos + decoding.readUint32(decoder)
   const clientbeforeState = decoding.readVarUint(decoder)
-  /**
-   * Stack of pending structs waiting for struct dependencies.
-   * Maximum length of stack is structReaders.size.
-   * @type {Array<AbstractRef>}
-   */
-  const stack = []
-  const localState = getStates(store)
   for (let i = 0; i < clientbeforeState; i++) {
     const nextID = readID(decoder)
     const decoderPos = decoder.pos + decoding.readUint32(decoder)
@@ -160,29 +151,152 @@ export const readStructs = (decoder, transaction, store) => {
   // Jump ahead to end of message so that reading can continue.
   // We will use the created struct readers for the remaining part of this workflow.
   decoder.pos = endOfMessagePos
-  for (const it of structReaders.values()) {
+  return structReaders
-    // todo try for in of it
+}
-    for (let res = it.next(); !res.done; res = it.next()) {
+
-      stack.push(res.value)
+/**
-      while (stack.length > 0) {
+ * Resume computing structs generated by struct readers.
-        const ref = stack[stack.length - 1]
+ *
-        const m = ref._missing
+ * While there is something to do, we integrate structs in this order
-        while (m.length > 0) {
+ * 1. top element on stack, if stack is not empty
-          const nextMissing = m[m.length - 1]
+ * 2. next element from current struct reader (if empty, use next struct reader)
-          if (!exists(store, nextMissing)) {
+ *
-            // @ts-ignore must not be undefined, otherwise unexpected case
+ * If struct causally depends on another struct (ref.missing), we put next reader of
-            stack.push(structReaders.get(nextMissing.client).next().value)
+ * `ref.id.client` on top of stack.
-            break
+ *
+ * At some point we find a struct that has no causal dependencies,
+ * then we start emptying the stack.
+ *
+ * It is not possible to have circles: i.e. struct1 (from client1) depends on struct2 (from client2)
+ * depends on struct3 (from client1). Therefore the max stack size is eqaul to `structReaders.length`.
+ *
+ * This method is implemented in a way so that we can resume computation if this update
+ * causally depends on another update.
+ *
+ * @param {Transaction} transaction
+ * @param {StructStore} store
+ * @param {Map<number,number>} localState
+ * @param {Map<number,IterableIterator<AbstractRef>>} structReaders
+ * @param {Array<AbstractRef>} stack Stack of pending structs waiting for struct dependencies.
+ *                                   Maximum length of stack is structReaders.size.
+ *
+ * @todo reimplement without iterators - read everything in arrays instead
+ */
+const execStructReaders = (transaction, store, localState, structReaders, stack) => {
+  // iterate over all struct readers until we are done
+  const structReaderIterator = structReaders.values()
+  let structReaderIteratorResult = structReaderIterator.next()
+  while (stack.length !== 0 || !structReaderIteratorResult.done) {
+    if (stack.length === 0) {
+      // stack is empty. We know that there there are more structReaders to be processed
+      const nextStructRes = structReaderIteratorResult.value.next()
+      if (nextStructRes.done) {
+        // current structReaderIteratorResult is empty, use next one
+        structReaderIteratorResult = structReaderIterator.next()
+      } else {
+        stack.push(nextStructRes.value)
+      }
+    } else {
+      const ref = stack[stack.length - 1]
+      const m = ref._missing
+      while (m.length > 0) {
+        const missing = m[m.length - 1]
+        if (!exists(store, missing)) {
+          // get the struct reader that has the missing struct
+          const reader = structReaders.get(missing.client)
+          const nextRef = reader === undefined ? undefined : reader.next().value
+          if (nextRef === undefined) {
+            // This update message causally depends on another update message.
+            // Store current stack and readers in StructStore and resume the computation at another time
+            store.pendingStructReaders.add({ stack, structReaders, missing })
+            return
           }
-          ref._missing.pop()
+          stack.push(nextRef)
+          break
         }
-        if (m.length === 0) {
+        ref._missing.pop()
-          const localClock = (localState.get(ref.id.client) || 0)
+      }
-          const offset = ref.id.clock < localClock ? localClock - ref.id.clock : 0
+      if (m.length === 0) {
-          ref.toStruct(transaction, offset).integrate(transaction)
+        const localClock = (localState.get(ref.id.client) || 0)
-          stack.pop()
+        const offset = ref.id.clock < localClock ? localClock - ref.id.clock : 0
+        if (offset < ref.length) {
+          ref.toStruct(transaction.y, store, offset).integrate(transaction)
         }
+        stack.pop()
       }
     }
   }
 }
+
+/**
+ * Try to resume pending struct readers in `store.pendingReaders` while `pendingReaders.nextMissing`
+ * exists.
+ *
+ * @param {Transaction} transaction
+ * @param {StructStore} store
+ */
+const tryResumePendingStructReaders = (transaction, store) => {
+  let resume = true
+  const pendingReaders = store.pendingStructReaders
+  while (resume) {
+    resume = false
+    for (const pendingReader of pendingReaders) {
+      if (exists(store, pendingReader.missing)) {
+        resume = true // found at least one more reader to execute
+        pendingReaders.delete(pendingReader)
+        execStructReaders(transaction, store, getStates(store), pendingReader.structReaders, pendingReader.stack)
+      }
+    }
+  }
+}
+
+/**
+ * @param {Transaction} transaction
+ * @param {StructStore} store
+ */
+export const tryResumePendingDeleteReaders = (transaction, store) => {
+  const pendingReaders = store.pendingDeleteReaders
+  store.pendingDeleteReaders = []
+  for (let i = 0; i < pendingReaders.length; i++) {
+    readDeleteSet(pendingReaders[i], transaction, store)
+  }
+}
+
+/**
+ * Read the next Item in a Decoder and fill this Item with the read data.
+ *
+ * This is called when data is received from a remote peer.
+ *
+ * @param {decoding.Decoder} decoder The decoder object to read data from.
+ * @param {Transaction} transaction
+ * @param {StructStore} store
+ *
+ * @private
+ */
+export const readStructs = (decoder, transaction, store) => {
+  const localState = getStates(store)
+  const readers = readStructReaders(decoder, localState)
+  execStructReaders(transaction, store, localState, readers, [])
+  tryResumePendingStructReaders(transaction, store)
+  tryResumePendingDeleteReaders(transaction, store)
+}
+
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Transaction} transaction
+ * @param {StructStore} store
+ */
+export const readModel = (decoder, transaction, store) => {
+  readStructs(decoder, transaction, store)
+  readDeleteSet(decoder, transaction, store)
+}
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {StructStore} store
+ * @param {Map<number,number>} [targetState] The state of the target that receives the update. Leave empty to write all known structs
+ */
+export const writeModel = (encoder, store, targetState = new Map()) => {
+  writeStructs(encoder, store, targetState)
+  writeDeleteSet(encoder, createDeleteSetFromStructStore(store))
+}

tests/encoding.tests.js

@@ -0,0 +1,36 @@
+import * as t from 'lib0/testing.js'
+
+import {
+  structRefs,
+  structGCRefNumber,
+  structBinaryRefNumber,
+  structDeletedRefNumber,
+  structEmbedRefNumber,
+  structFormatRefNumber,
+  structJSONRefNumber,
+  structStringRefNumber,
+  structTypeRefNumber,
+  GCRef,
+  ItemBinaryRef,
+  ItemDeletedRef,
+  ItemEmbedRef,
+  ItemFormatRef,
+  ItemJSONRef,
+  ItemStringRef,
+  ItemTypeRef
+} from '../src/internals.js'
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testStructReferences = tc => {
+  t.assert(structRefs.length === 8)
+  t.assert(structRefs[structGCRefNumber] === GCRef)
+  t.assert(structRefs[structBinaryRefNumber] === ItemBinaryRef)
+  t.assert(structRefs[structDeletedRefNumber] === ItemDeletedRef)
+  t.assert(structRefs[structEmbedRefNumber] === ItemEmbedRef)
+  t.assert(structRefs[structFormatRefNumber] === ItemFormatRef)
+  t.assert(structRefs[structJSONRefNumber] === ItemJSONRef)
+  t.assert(structRefs[structStringRefNumber] === ItemStringRef)
+  t.assert(structRefs[structTypeRefNumber] === ItemTypeRef)
+}

tests/index.js

@@ -3,6 +3,7 @@ import * as array from './y-array.tests.js'
 import * as map from './y-map.tests.js'
 import * as text from './y-text.tests.js'
 import * as xml from './y-xml.tests.js'
+import * as encoding from './encoding.tests.js'
 
 import { runTests } from 'lib0/testing.js'
 import { isBrowser, isNode } from 'lib0/environment.js'
@@ -12,7 +13,7 @@ if (isBrowser) {
   log.createVConsole(document.body)
 }
 runTests({
-  map, array, text, xml
+  map, array, text, xml, encoding
 }).then(success => {
   /* istanbul ignore next */
   if (isNode) {

tests/testHelper.js

@@ -81,14 +81,14 @@ export class TestYInstance extends Y.Y {
     if (!this.tc.onlineConns.has(this)) {
       this.tc.onlineConns.add(this)
       const encoder = encoding.createEncoder()
-      syncProtocol.writeSyncStep1(encoder, this)
+      syncProtocol.writeSyncStep1(encoder, this.store)
       // publish SyncStep1
       broadcastMessage(this, encoding.toBuffer(encoder))
       this.tc.onlineConns.forEach(remoteYInstance => {
         if (remoteYInstance !== this) {
           // remote instance sends instance to this instance
           const encoder = encoding.createEncoder()
-          syncProtocol.writeSyncStep1(encoder, remoteYInstance)
+          syncProtocol.writeSyncStep1(encoder, remoteYInstance.store)
           this._receive(encoding.toBuffer(encoder), remoteYInstance)
         }
       })
@@ -271,8 +271,11 @@ export const compare = users => {
   const userMapValues = users.map(u => u.getMap('map').toJSON())
   const userXmlValues = users.map(u => u.get('xml', Y.XmlElement).toString())
   const userTextValues = users.map(u => u.getText('text').toDelta())
-  for (var i = 0; i < users.length - 1; i++) {
+  for (const u of users) {
-    t.describe(`Comparing user${i} with user${i + 1}`)
+    t.assert(u.store.pendingDeleteReaders.length === 0)
+    t.assert(u.store.pendingStructReaders.size === 0)
+  }
+  for (let i = 0; i < users.length - 1; i++) {
     t.compare(userArrayValues[i].length, users[i].getArray('array').length)
     t.compare(userArrayValues[i], userArrayValues[i + 1])
     t.compare(userMapValues[i], userMapValues[i + 1])

tests/y-map.tests.js

@@ -221,7 +221,7 @@ export const testObserveDeepProperties = tc => {
 export const testObserversUsingObservedeep = tc => {
   const { users, map0 } = init(tc, { users: 2 })
   /**
-   * @type {Array<Array<string>>}
+   * @type {Array<Array<string|number>>}
    */
   const pathes = []
   let calls = 0
@@ -348,7 +348,7 @@ const mapTransactions = [
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYmapTests20 = tc => {
-  applyRandomTests(tc, mapTransactions, 20)
+  applyRandomTests(tc, mapTransactions, 8)
 }
 
 /**