Create Structs based on offset, if necessary

implement offset parameter in Ref.toStruct

src/structs/AbstractItem.js

@@ -139,9 +139,6 @@ export class AbstractItem extends AbstractStruct {
     const parent = this.parent
     const parentSub = this.parentSub
     const length = this.length
-    const left = this.left
-    const right = this.right
-    // integrate
     /*
     # $this has to find a unique position between origin and the next known character
     # case 1: $origin equals $o.origin: the $creator parameter decides if left or right
@@ -163,8 +160,8 @@ export class AbstractItem extends AbstractStruct {
      */
     let o
     // set o to the first conflicting item
-    if (left !== null) {
-      o = left.right
+    if (this.left !== null) {
+      o = this.left.right
     } else if (parentSub !== null) {
       o = parent._map.get(parentSub) || null
     } else {
@@ -175,7 +172,7 @@ export class AbstractItem extends AbstractStruct {
     // Let c in conflictingItems, b in itemsBeforeOrigin
     // ***{origin}bbbb{this}{c,b}{c,b}{o}***
     // Note that conflictingItems is a subset of itemsBeforeOrigin
-    while (o !== null && o !== right) {
+    while (o !== null && o !== this.right) {
       itemsBeforeOrigin.add(o)
       conflictingItems.add(o)
       if (this.origin === o.origin) {
@@ -199,10 +196,10 @@ export class AbstractItem extends AbstractStruct {
       o = o.right
     }
     // reconnect left/right + update parent map/start if necessary
-    if (left !== null) {
-      const right = left.right
+    if (this.left !== null) {
+      const right = this.left.right
       this.right = right
-      left.right = this
+      this.left.right = this
       if (right !== null) {
         right.left = this
       }
@@ -230,12 +227,12 @@ export class AbstractItem extends AbstractStruct {
       maplib.setIfUndefined(transaction.changed, parent, set.create).add(parentSub)
     }
     // @ts-ignore
-    if ((parent._item !== null && parent._item.deleted) || (left !== null && parentSub !== null)) {
+    if ((parent._item !== null && parent._item.deleted) || (this.left !== null && parentSub !== null)) {
       // delete if parent is deleted or if this is not the current attribute value of parent
       this.delete(transaction)
-    } else if (parentSub !== null && left === null && right !== null) {
+    } else if (parentSub !== null && this.left === null && this.right !== null) {
       // this is the current attribute value of parent. delete right
-      right.delete(transaction)
+      this.right.delete(transaction)
     }
   }
 
@@ -529,3 +526,12 @@ export class AbstractItemRef extends AbstractRef {
     }
   }
 }
+
+/**
+ * @param {AbstractItemRef} item
+ * @param {number} offset
+ */
+export const changeItemRefOffset = (item, offset) => {
+  item.id = createID(item.id.client, item.id.clock + offset)
+  item.left = createID(item.id.client, item.id.clock - 1)
+}

src/structs/AbstractStruct.js

@@ -1,6 +1,6 @@
 
 import {
-  Y, ID, Transaction // eslint-disable-line
+  ID, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js' // eslint-disable-line
@@ -77,9 +77,10 @@ export class AbstractRef {
   }
   /**
    * @param {Transaction} transaction
+   * @param {number} offset
    * @return {AbstractStruct}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     throw error.methodUnimplemented()
   }
   /**

src/structs/GC.js

@@ -6,7 +6,8 @@ import {
   AbstractStruct,
   createID,
   writeID,
-  ID // eslint-disable-line
+  addStruct,
+  Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as decoding from 'lib0/decoding.js'
@@ -42,6 +43,13 @@ export class GC extends AbstractStruct {
     return true
   }
 
+  /**
+   * @param {Transaction} transaction
+   */
+  integrate (transaction) {
+    addStruct(transaction.y.store, this)
+  }
+
   /**
    * @param {encoding.Encoder} encoder
    * @param {number} offset
@@ -65,10 +73,6 @@ export class GCRef extends AbstractRef {
    */
   constructor (decoder, id, info) {
     super(id)
-    /**
-     * @type {ID}
-     */
-    this.id = id
     /**
      * @type {number}
      */
@@ -83,9 +87,16 @@ export class GCRef extends AbstractRef {
     ]
   }
   /**
+   * @param {Transaction} transaction
+   * @param {number} offset
    * @return {GC}
    */
-  toStruct () {
+  toStruct (transaction, offset) {
+    if (offset > 0) {
+      // @ts-ignore
+      this.id = createID(this.id.client, this.id.clock + offset)
+      this._len = this._len - offset
+    }
     return new GC(
       this.id,
       this._len

src/structs/ItemBinary.js

@@ -10,6 +10,8 @@ import {
   getItemCleanEnd,
   getItemCleanStart,
   getItemType,
+  GC,
+  ItemDeleted,
   Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -69,17 +71,32 @@ export class ItemBinaryRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemBinary}
+   * @param {number} offset
+   * @return {ItemBinary|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, 1)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
+
     return new ItemBinary(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.content
     )

src/structs/ItemDeleted.js

@@ -10,6 +10,8 @@ import {
   getItemCleanEnd,
   getItemCleanStart,
   getItemType,
+  changeItemRefOffset,
+  GC,
   Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -84,17 +86,36 @@ export class ItemDeletedRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemDeleted}
+   * @param {number} offset
+   * @return {ItemDeleted|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+    if (offset > 0) {
+      changeItemRefOffset(this, offset)
+      this.len = this.len - offset
+    }
+
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, 1)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
+
     return new ItemDeleted(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.len
     )

src/structs/ItemEmbed.js

@@ -8,6 +8,8 @@ import {
   getItemCleanEnd,
   getItemCleanStart,
   getItemType,
+  ItemDeleted,
+  GC,
   Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -64,17 +66,32 @@ export class ItemEmbedRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemEmbed}
+   * @param {number} offset
+   * @return {ItemEmbed|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, 1)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
+
     return new ItemEmbed(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.embed
     )

src/structs/ItemFormat.js

@@ -8,6 +8,8 @@ import {
   getItemCleanEnd,
   getItemCleanStart,
   getItemType,
+  ItemDeleted,
+  GC,
   Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -71,17 +73,32 @@ export class ItemFormatRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemFormat}
+   * @param {number} offset
+   * @return {ItemFormat|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, 1)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
+
     return new ItemFormat(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.key,
       this.value

src/structs/ItemJSON.js

@@ -9,6 +9,9 @@ import {
   getItemCleanStart,
   getItemType,
   splitItem,
+  changeItemRefOffset,
+  GC,
+  ItemDeleted,
   Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -106,6 +109,9 @@ export class ItemJSONRef extends AbstractItemRef {
         cs.push(JSON.parse(c))
       }
     }
+    /**
+     * @type {Array<any>}
+     */
     this.content = cs
   }
   get length () {
@@ -113,17 +119,34 @@ export class ItemJSONRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemJSON}
+   * @param {number} offset
+   * @return {ItemJSON|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+    if (offset > 0) {
+      changeItemRefOffset(this, offset)
+      this.content = this.content.slice(offset)
+    }
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, this.content.length)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
     return new ItemJSON(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.content
     )

src/structs/ItemString.js

@@ -8,6 +8,9 @@ import {
   getItemCleanStart,
   getItemType,
   splitItem,
+  changeItemRefOffset,
+  ItemDeleted,
+  GC,
   Transaction, ID, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -102,17 +105,36 @@ export class ItemStringRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemString}
+   * @param {number} offset
+   * @return {ItemString|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+    if (offset > 0) {
+      changeItemRefOffset(this, offset)
+      this.string = this.string.slice(offset)
+    }
+
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, this.string.length)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
+
     return new ItemString(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.string
     )

src/structs/ItemType.js

@@ -168,17 +168,32 @@ export class ItemTypeRef extends AbstractItemRef {
   }
   /**
    * @param {Transaction} transaction
-   * @return {ItemType}
+   * @param {number} offset
+   * @return {ItemType|GC}
    */
-  toStruct (transaction) {
+  toStruct (transaction, offset) {
     const y = transaction.y
     const store = y.store
+
+    let parent
+    if (this.parent !== null) {
+      const parentItem = getItemType(store, this.parent)
+      switch (parentItem.constructor) {
+        case ItemDeleted:
+        case GC:
+          return new GC(this.id, 1)
+      }
+      parent = parentItem.type
+    } else {
+      // @ts-ignore
+      parent = y.get(this.parentYKey)
+    }
+
     return new ItemType(
       this.id,
       this.left === null ? null : getItemCleanEnd(store, transaction, this.left),
       this.right === null ? null : getItemCleanStart(store, transaction, this.right),
-      // @ts-ignore
-      this.parent === null ? y.get(this.parentYKey) : getItemType(store, this.parent).type,
+      parent,
       this.parentSub,
       this.type
     )

src/utils/StructStore.js

@@ -19,20 +19,20 @@ export class StructStore {
 }
 
 /**
- * Return the states as an array of {client,clock} pairs.
+ * Return the states as a Map<client,clock>.
  * Note that clock refers to the next expected clock id.
  *
  * @param {StructStore} store
- * @return {Array<{client:number,clock:number}>}
+ * @return {Map<number,number>}
  */
-export const getStates = store =>
-  map.map(store.clients, (structs, client) => {
+export const getStates = store => {
+  const sm = new Map()
+  store.clients.forEach((structs, client) => {
     const struct = structs[structs.length - 1]
-    return {
-      client,
-      clock: struct.id.clock + struct.length
-    }
+    sm.set(client, struct.id.clock + struct.length)
   })
+  return sm
+}
 
 /**
  * @param {StructStore} store

src/utils/Transaction.js

@@ -56,8 +56,11 @@ export class Transaction {
      * Holds the state before the transaction started.
      * @type {Map<Number,Number>}
      */
-    this.beforeState = new Map()
-    getStates(y.store).forEach(({client, clock}) => { this.beforeState.set(client, clock) })
+    this.beforeState = getStates(y.store)
+    /**
+     * Holds the state after the transaction.
+     * @type {Map<Number,Number>}
+     */
     this.afterState = new Map()
     /**
      * All types that were directly modified (property added or child

src/utils/Y.js

@@ -96,9 +96,7 @@ export class Y extends Observable {
         // only call afterTransaction listeners if anything changed
         const transactionChangedContent = transaction.changedParentTypes.size !== 0
         if (transactionChangedContent) {
-          getStates(transaction.y.store).forEach(({client, clock}) => {
-            transaction.afterState.set(client, clock)
-          })
+          transaction.afterState = getStates(transaction.y.store)
           // when all changes & events are processed, emit afterTransaction event
           this.emit('afterTransaction', [this, transaction])
           // transaction cleanup
@@ -152,7 +150,8 @@ export class Y extends Observable {
               // @ts-ignore
               const structs = store.clients.get(client)
               // we iterate from right to left so we can safely remove entries
-              for (let i = structs.length - 1; i >= math.max(findIndexSS(structs, clock), 1); i--) {
+              const firstChangePos = math.max(findIndexSS(structs, clock), 1)
+              for (let i = structs.length - 1; i >= firstChangePos; i--) {
                 tryToMergeWithLeft(structs, i)
               }
             }

src/utils/structEncoding.js

@@ -43,19 +43,23 @@ const structRefs = [
  * @param {decoding.Decoder} decoder
  * @param {number} structsLen
  * @param {ID} nextID
+ * @param {number} localState next expected clock by nextID.client
  * @return {Iterator<AbstractRef>}
  */
-const createStructReaderIterator = (decoder, structsLen, nextID) => iterator.createIterator(() => {
+const createStructReaderIterator = (decoder, structsLen, nextID, localState) => iterator.createIterator(() => {
   let done = false
   let value
-  if (structsLen === 0) {
-    done = true
-  } else {
+  do {
+    if (structsLen === 0) {
+      done = true
+      value = undefined
+      break
+    }
     const info = decoding.readUint8(decoder)
     value = new structRefs[binary.BITS5 & info](decoder, nextID, info)
     nextID = createID(nextID.client, nextID.clock + value.length)
     structsLen--
-  }
+  } while (nextID.clock <= localState) // read until we find something new (check nextID.clock instead because it equals `clock+len`)
   return { done, value }
 })
 
@@ -78,7 +82,7 @@ export const writeStructs = (encoder, store, _sm) => {
       sm.set(client, clock)
     }
   })
-  getStates(store).forEach(({client}) => {
+  getStates(store).forEach((clock, client) => {
     if (!_sm.has(client)) {
       sm.set(client, 0)
     }
@@ -131,17 +135,18 @@ export const readStructs = (decoder, transaction, store) => {
    */
   const structReaders = new Map()
   const clientbeforeState = decoding.readVarUint(decoder)
+  /**
+   * @type {Array<AbstractRef>}
+   */
+  const stack = []
+  const localState = getStates(store)
   for (let i = 0; i < clientbeforeState; i++) {
     const nextID = readID(decoder)
     const decoderPos = decoder.pos + decoding.readUint32(decoder)
     const structReaderDecoder = decoding.clone(decoder, decoderPos)
     const numberOfStructs = decoding.readVarUint(structReaderDecoder)
-    structReaders.set(nextID.client, createStructReaderIterator(structReaderDecoder, numberOfStructs, nextID))
+    structReaders.set(nextID.client, createStructReaderIterator(structReaderDecoder, numberOfStructs, nextID, localState.get(nextID.client) || 0))
   }
-  /**
-   * @type {Array<AbstractRef>}
-   */
-  const stack = []
   for (const it of structReaders.values()) {
     // todo try for in of it
     for (let res = it.next(); !res.done; res = it.next()) {
@@ -159,7 +164,9 @@ export const readStructs = (decoder, transaction, store) => {
           ref._missing.pop()
         }
         if (m.length === 0) {
-          ref.toStruct(transaction).integrate(transaction)
+          const localClock = (localState.get(ref.id.client) || 0)
+          const offset = ref.id.clock < localClock ? localClock - ref.id.clock : 0
+          ref.toStruct(transaction, offset).integrate(transaction)
           stack.pop()
         }
       }

tests/testHelper.js

@@ -243,6 +243,7 @@ export const init = (tc, { users = 5 } = {}) => {
   result.testConnector = testConnector
   for (let i = 0; i < users; i++) {
     const y = testConnector.createY(i)
+    y.clientID = i
     result.users.push(y)
     result['array' + i] = y.get('array', Y.Array)
     result['map' + i] = y.get('map', Y.Map)