more type fixes and rethinking writeStructs

2019-04-02 23:08:58 +02:00
parent 73c28952c2
commit e23582b1cd
35 changed files with 952 additions and 695 deletions
--- a/src/utils/DeleteSet.js
+++ b/src/utils/DeleteSet.js
@@ -1,6 +1,7 @@
 import * as map from 'lib0/map.js'
 import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
+import * as math from 'lib0/math.js'
 import { StructStore, getItemRange } from './StructStore.js' // eslint-disable-line
 import { Transaction } from './Transaction.js' // eslint-disable-line
 import { ID } from './ID.js' // eslint-disable-line
@@ -39,13 +40,38 @@ export class DeleteSet {
  }
 }

+/**
+ * @param {Array<DeleteItem>} dis
+ * @param {number} clock
+ * @return {number|null}
+ */
+export const findIndexSS = (dis, clock) => {
+  let left = 0
+  let right = dis.length
+  while (left <= right) {
+    const midindex = math.floor((left + right) / 2)
+    const mid = dis[midindex]
+    const midclock = mid.clock
+    if (midclock <= clock) {
+      if (clock < midclock + mid.len) {
+        return midindex
+      }
+      left = midindex
+    } else {
+      right = midindex
+    }
+  }
+  return null
+}
+
 /**
 * @param {DeleteSet} ds
 * @param {ID} id
 * @return {boolean}
 */
 export const isDeleted = (ds, id) => {
-  
+  const dis = ds.clients.get(id.client)
+  return dis !== undefined && findIndexSS(dis, id.clock) !== null
 }

 /**
@@ -75,15 +101,12 @@ export const sortAndMergeDeleteSet = ds => {
 }

 /**
- * @param {Transaction} transaction
+ * @param {DeleteSet} ds
+ * @param {ID} id
+ * @param {number} length
 */
-export const createDeleteSetFromTransaction = transaction => {
-  const ds = new DeleteSet()
-  transaction.deleted.forEach(item => {
-    map.setIfUndefined(ds.clients, item.id.client, () => []).push(new DeleteItem(item.id.clock, item.length))
-  })
-  sortAndMergeDeleteSet(ds)
-  return ds
+export const addToDeleteSet = (ds, id, length) => {
+  map.setIfUndefined(ds.clients, id.client, () => []).push(new DeleteItem(id.clock, length))
 }

 /**
--- a/src/utils/ID.js
+++ b/src/utils/ID.js
@@ -4,6 +4,8 @@

 import * as decoding from 'lib0/decoding.js'
 import * as encoding from 'lib0/encoding.js'
+import * as error from 'lib0/error.js'
+import { AbstractType } from '../types/AbstractType.js' // eslint-disable-line

 export class ID {
  /**
@@ -46,14 +48,19 @@ export class ID {
  }
 }

+/**
+ * @param {ID} a
+ * @param {ID} b
+ * @return {boolean}
+ */
+export const compareIDs = (a, b) => a === b || (a !== null && b !== null && a.client === b.client && a.clock === b.clock)
+
 /**
 * @param {number} client
 * @param {number} clock
 */
 export const createID = (client, clock) => new ID(client, clock)

-const isNullID = 0xFFFFFF
-
 /**
 * @param {encoding.Encoder} encoder
 * @param {ID} id
@@ -63,21 +70,31 @@ export const writeID = (encoder, id) => {
  encoding.writeVarUint(encoder, id.clock)
 }

-/**
- * @param {encoding.Encoder} encoder
- */
-export const writeNullID = (encoder) =>
-  encoding.writeVarUint(encoder, isNullID)
-
 /**
 * Read ID.
 * * If first varUint read is 0xFFFFFF a RootID is returned.
 * * Otherwise an ID is returned
 *
 * @param {decoding.Decoder} decoder
- * @return {ID | null}
+ * @return {ID}
 */
-export const readID = decoder => {
-  const client = decoding.readVarUint(decoder)
-  return client === isNullID ? null : createID(client, decoding.readVarUint(decoder))
+export const readID = decoder =>
+  createID(decoding.readVarUint(decoder), decoding.readVarUint(decoder))
+
+/**
+ * The top types are mapped from y.share.get(keyname) => type.
+ * `type` does not store any information about the `keyname`.
+ * This function finds the correct `keyname` for `type` and throws otherwise.
+ *
+ * @param {AbstractType} type
+ * @return {string}
+ */
+export const findRootTypeKey = type => {
+  // @ts-ignore _y must be defined, otherwise unexpected case
+  for (let [key, value] of type._y.share) {
+    if (value === type) {
+      return key
+    }
+  }
+  throw error.unexpectedCase()
 }
--- a/src/utils/Snapshot.js
+++ b/src/utils/Snapshot.js
@@ -18,6 +18,6 @@ export class Snapshot {
 * @param {AbstractItem} item
 * @param {Snapshot} [snapshot]
 */
-export const isVisible = (item, snapshot) => snapshot === undefined ? !item._deleted : (
-  snapshot.sm.has(item.id.client) && (snapshot.sm.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item._id)
+export const isVisible = (item, snapshot) => snapshot === undefined ? !item.deleted : (
+  snapshot.sm.has(item.id.client) && (snapshot.sm.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item.id)
 )
--- a/src/utils/StructStore.js
+++ b/src/utils/StructStore.js
@@ -5,6 +5,7 @@ import { ID } from './ID.js' // eslint-disable-line
 import { Transaction } from './Transaction.js' // eslint-disable-line
 import * as map from 'lib0/map.js'
 import * as math from 'lib0/math.js'
+import * as error from 'lib0/error.js'

 export class StructStore {
  constructor () {
@@ -68,13 +69,13 @@ export const addStruct = (store, struct) => {
 }

 /**
- * Expects that id is actually in store. This function throws or is an infinite loop otherwise.
- * @param {Array<AbstractStruct>} structs // ordered structs without holes
+ * Perform a binary search on a sorted array
+ * @param {Array<any>} structs
 * @param {number} clock
 * @return {number}
 * @private
 */
-export const findIndex = (structs, clock) => {
+export const findIndexSS = (structs, clock) => {
  let left = 0
  let right = structs.length
  while (left <= right) {
@@ -90,7 +91,7 @@ export const findIndex = (structs, clock) => {
      right = midindex
    }
  }
-  throw new Error('ID does not exist')
+  throw error.unexpectedCase()
 }

 /**
@@ -101,13 +102,13 @@ export const findIndex = (structs, clock) => {
 * @return {AbstractStruct}
 * @private
 */
-const find = (store, id) => {
+export const find = (store, id) => {
  /**
   * @type {Array<AbstractStruct>}
   */
  // @ts-ignore
  const structs = store.clients.get(id.client)
-  return structs[findIndex(structs, id.clock)]
+  return structs[findIndexSS(structs, id.clock)]
 }

 /**
@@ -135,14 +136,13 @@ export const getItemCleanStart = (store, transaction, id) => {
   */
  // @ts-ignore
  const structs = store.clients.get(id.client)
-  const index = findIndex(structs, id.clock)
+  const index = findIndexSS(structs, id.clock)
  /**
   * @type {AbstractItem}
   */
  let struct = structs[index]
  if (struct.id.clock < id.clock) {
-    struct.splitAt()
-    struct = splitStruct(transaction, struct, id.clock - struct.id.clock)
+    struct = struct.splitAt(transaction, id.clock - struct.id.clock)
    structs.splice(index, 0, struct)
  }
  return struct
@@ -163,10 +163,10 @@ export const getItemCleanEnd = (store, transaction, id) => {
   */
  // @ts-ignore
  const structs = store.clients.get(id.client)
-  const index = findIndex(structs, id.clock)
+  const index = findIndexSS(structs, id.clock)
  const struct = structs[index]
  if (id.clock !== struct.id.clock + struct.length - 1) {
-    structs.splice(index, 0, splitStruct(transaction, struct, id.clock - struct.id.clock + 1))
+    structs.splice(index, 0, struct.splitAt(transaction, id.clock - struct.id.clock + 1))
  }
  return struct
 }
@@ -188,11 +188,11 @@ export const getItemRange = (store, transaction, client, clock, len) => {
   */
  // @ts-ignore
  const structs = store.clients.get(client)
-  let index = findIndex(structs, clock)
+  let index = findIndexSS(structs, clock)
  let struct = structs[index]
  let range = []
  if (struct.id.clock < clock) {
-    struct = splitStruct(transaction, struct, clock - struct.id.clock)
+    struct = struct.splitAt(transaction, clock - struct.id.clock)
    structs.splice(index, 0, struct)
  }
  while (struct.id.clock + struct.length <= clock + len) {
@@ -200,7 +200,7 @@ export const getItemRange = (store, transaction, client, clock, len) => {
    struct = structs[++index]
  }
  if (clock < struct.id.clock + struct.length) {
-    structs.splice(index, 0, splitStruct(transaction, struct, clock + len - struct.id.clock))
+    structs.splice(index, 0, struct.splitAt(transaction, clock + len - struct.id.clock))
    range.push(struct)
  }
  return range
@@ -218,5 +218,12 @@ export const replaceStruct = (store, struct, newStruct) => {
   */
  // @ts-ignore
  const structs = store.clients.get(struct.id.client)
-  structs[findIndex(structs, struct.id.clock)] = newStruct
+  structs[findIndexSS(structs, struct.id.clock)] = newStruct
 }
+
+/**
+ * @param {StructStore} store
+ * @param {ID} id
+ * @return {boolean}
+ */
+export const exists = (store, id) => id.clock < getState(store, id.client)
--- a/src/utils/Transaction.js
+++ b/src/utils/Transaction.js
@@ -10,7 +10,7 @@ import { YEvent } from './YEvent.js' // eslint-disable-line
 import { ItemType } from '../structs/ItemType.js' // eslint-disable-line
 import { writeStructsFromTransaction } from './structEncoding.js'
 import { createID } from './ID.js' // eslint-disable-line
-import { createDeleteSetFromTransaction, writeDeleteSet } from './DeleteSet.js'
+import { writeDeleteSet, DeleteSet, sortAndMergeDeleteSet } from './DeleteSet.js'
 import { getState } from './StructStore.js'

 /**
@@ -46,15 +46,10 @@ export class Transaction {
     */
    this.y = y
    /**
-     * All new items that are added during a transaction.
-     * @type {Set<AbstractItem>}
+     * Describes the set of deleted items by ids
+     * @type {DeleteSet}
     */
-    this.added = new Set()
-    /**
-     * Set of all deleted items
-     * @type {Set<AbstractItem>}
-     */
-    this.deleted = new Set()
+    this.deleteSet = new DeleteSet()
    /**
     * If a state was modified, the original value is saved here.
     * Use `stateUpdates` to compute the original state before the transaction,
@@ -87,7 +82,8 @@ export class Transaction {
    if (this._updateMessage === null) {
      const encoder = encoding.createEncoder()
      writeStructsFromTransaction(encoder, this)
-      writeDeleteSet(encoder, createDeleteSetFromTransaction(this))
+      sortAndMergeDeleteSet(this.deleteSet)
+      writeDeleteSet(encoder, this.deleteSet)
      this._updateMessage = encoder
    }
    return this._updateMessage
--- a/src/utils/Y.js
+++ b/src/utils/Y.js
@@ -71,20 +71,20 @@ export class Y extends Observable {
      console.error(e)
    }
    if (initialCall) {
-      this.emit('beforeObserverCalls', [this, this._transaction, remote])
      const transaction = this._transaction
      this._transaction = null
-      // emit change events on changed types
-      transaction.changed.forEach((subs, itemtype) => {
-        if (!itemtype._item.deleted) {
-          itemtype.type._callObserver(transaction, subs, remote)
-        }
-      })
-      transaction.changedParentTypes.forEach((events, type) => {
-        if (!type._deleted) {
+      // only call event listeners / observers if anything changed
+      const transactionChangedContent = transaction.changedParentTypes.size !== 0
+      if (transactionChangedContent) {
+        this.emit('beforeObserverCalls', [this, this._transaction, remote])
+        // emit change events on changed types
+        transaction.changed.forEach((subs, itemtype) => {
+          itemtype._callObserver(transaction, subs)
+        })
+        transaction.changedParentTypes.forEach((events, type) => {
          events = events
            .filter(event =>
-              !event.target._deleted
+              event.target._item === null || !event.target._item.deleted
            )
          events
            .forEach(event => {
@@ -92,11 +92,15 @@ export class Y extends Observable {
            })
          // we don't have to check for events.length
          // because there is no way events is empty..
-          type.type._deepEventHandler.callEventListeners(transaction, events)
-        }
-      })
-      // when all changes & events are processed, emit afterTransaction event
-      this.emit('afterTransaction', [this, transaction, remote])
+          type._deepEventHandler.callEventListeners(transaction, events)
+        })
+        // when all changes & events are processed, emit afterTransaction event
+        this.emit('afterTransaction', [this, transaction, remote])
+        // transaction cleanup
+        // todo: replace deleted items with ItemDeleted
+        // todo: replace items with deleted parent with ItemGC
+        // todo: on all affected store.clients props, try to merge
+      }
    }
  }
  /**
@@ -120,6 +124,7 @@ export class Y extends Observable {
   *   }
   *
   * @TODO: implement getText, getArray, ..
+   * @TODO: Decide wether to use define() or get() and then use it consistently
   *
   * @param {string} name
   * @param {Function} TypeConstructor The constructor of the type definition
@@ -127,7 +132,7 @@ export class Y extends Observable {
   */
  get (name, TypeConstructor = AbstractType) {
    // @ts-ignore
-    const type = map.setTfUndefined(this.share, name, () => new TypeConstructor())
+    const type = map.setIfUndefined(this.share, name, () => new TypeConstructor())
    const Constr = type.constructor
    if (Constr !== TypeConstructor) {
      if (Constr === AbstractType) {
--- a/src/utils/integrateRemoteStructs.js
+++ b/src/utils/integrateRemoteStructs.js
@@ -1,112 +0,0 @@
-/**
- * @module utils
- */
-
-import * as decoding from 'lib0/decoding.js'
-import { GC } from '../structs/GC.js'
-import { Y } from '../utils/Y.js' // eslint-disable-line
-
-class MissingEntry {
-  constructor (decoder, missing, struct) {
-    this.decoder = decoder
-    this.missing = missing.length
-    this.struct = struct
-  }
-}
-
-/**
- * @private
- * Integrate remote struct
- * When a remote struct is integrated, other structs might be ready to ready to
- * integrate.
- * @param {Y} y
- * @param {Item} struct
- */
-function _integrateRemoteStructHelper (y, struct) {
-  const id = struct._id
-  if (id === undefined) {
-    struct._integrate(y)
-  } else {
-    if (y.ss.getState(id.user) > id.clock) {
-      return
-    }
-    if (!y.gcEnabled || struct.constructor === GC || (struct._parent.constructor !== GC && struct._parent._deleted === false)) {
-      // Is either a GC or Item with an undeleted parent
-      // save to integrate
-      struct._integrate(y)
-    } else {
-      // Is an Item. parent was deleted.
-      struct._gc(y)
-    }
-    let msu = y._missingStructs.get(id.user)
-    if (msu != null) {
-      let clock = id.clock
-      const finalClock = clock + struct._length
-      for (;clock < finalClock; clock++) {
-        const missingStructs = msu.get(clock)
-        if (missingStructs !== undefined) {
-          missingStructs.forEach(missingDef => {
-            missingDef.missing--
-            if (missingDef.missing === 0) {
-              y._readyToIntegrate.push(missingDef)
-            }
-          })
-          msu.delete(clock)
-        }
-      }
-      if (msu.size === 0) {
-        y._missingStructs.delete(id.user)
-      }
-    }
-  }
-}
-
-/**
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- */
-export const integrateRemoteStructs = (decoder, y) => {
-  const len = decoding.readUint32(decoder)
-  for (let i = 0; i < len; i++) {
-    let reference = decoding.readVarUint(decoder)
-    let Constr = getStruct(reference)
-    let struct = new Constr()
-    let decoderPos = decoder.pos
-    let missing = struct._fromBinary(y, decoder)
-    if (missing.length === 0) {
-      while (struct !== null) {
-        _integrateRemoteStructHelper(y, struct)
-        struct = null
-        if (y._readyToIntegrate.length > 0) {
-          const missingDef = y._readyToIntegrate.shift()
-          const decoder = missingDef.decoder
-          let oldPos = decoder.pos
-          let missing = missingDef.struct._fromBinary(y, decoder)
-          decoder.pos = oldPos
-          if (missing.length === 0) {
-            struct = missingDef.struct
-          } else {
-            throw new Error('Missing should be empty')
-          }
-        }
-      }
-    } else {
-      let _decoder = decoding.createDecoder(decoder.arr.buffer)
-      _decoder.pos = decoderPos
-      let missingEntry = new MissingEntry(_decoder, missing, struct)
-      let missingStructs = y._missingStructs
-      for (let i = missing.length - 1; i >= 0; i--) {
-        let m = missing[i]
-        if (!missingStructs.has(m.user)) {
-          missingStructs.set(m.user, new Map())
-        }
-        let msu = missingStructs.get(m.user)
-        if (!msu.has(m.clock)) {
-          msu.set(m.clock, [])
-        }
-        let mArray = msu = msu.get(m.clock)
-        mArray.push(missingEntry)
-      }
-    }
-  }
-}
--- a/src/utils/relativePosition.js
+++ b/src/utils/relativePosition.js
@@ -3,9 +3,13 @@
 */

 import * as ID from './ID.js'
-import { GC } from '../structs/GC.js'
-
-// TODO: Implement function to describe ranges
+import { AbstractType } from '../types/AbstractType.js' // eslint-disable-line
+import { AbstractItem } from '../structs/AbstractItem.js' // eslint-disable-line
+import * as encoding from 'lib0/encoding.js'
+import * as decoding from 'lib0/decoding.js'
+import * as error from 'lib0/error.js'
+import { find, exists, getItemType, StructStore } from './StructStore.js' // eslint-disable-line
+import { Y } from './Y.js' // eslint-disable-line

 /**
 * A relative position that is based on the Yjs model. In contrast to an
@@ -18,9 +22,7 @@ import { GC } from '../structs/GC.js'
 * {@link getRelativePosition} and it can be transformed to an absolute position
 * with {@link fromRelativePosition}.
 *
- * Pro tip: Use this to implement shared cursor locations in YText or YXml!
- * The relative position is {@link encodable}, so you can send it to other
- * clients.
+ * One of the properties must be defined.
 *
 * @example
 * // Current cursor position is at position 10
@@ -33,98 +35,220 @@ import { GC } from '../structs/GC.js'
 * absolutePosition.type // => yText
 * console.log('cursor location is ' + absolutePosition.offset) // => cursor location is 3
 *
- * @typedef {encodable} RelativePosition
 */
+export class RelativePosition {
+  /**
+   * @param {ID.ID|null} type
+   * @param {string|null} tname
+   * @param {ID.ID|null} item
+   */
+  constructor (type, tname, item) {
+    /**
+     * @type {ID.ID|null}
+     */
+    this.type = type
+    /**
+     * @type {string|null}
+     */
+    this.tname = tname
+    /**
+     * @type {ID.ID | null}
+     */
+    this.item = item
+  }
+}
+
+export class AbsolutePosition {
+  /**
+   * @param {AbstractType} type
+   * @param {number} offset
+   */
+  constructor (type, offset) {
+    /**
+     * @type {AbstractType}
+     */
+    this.type = type
+    /**
+     * @type {number}
+     */
+    this.offset = offset
+  }
+}
+
+/**
+ * @param {AbstractType} type
+ * @param {number} offset
+ */
+export const createAbsolutePosition = (type, offset) => new AbsolutePosition(type, offset)
+
+/**
+ * @param {AbstractType} type
+ * @param {ID.ID|null} item
+ */
+export const createRelativePosition = (type, item) => {
+  let typeid = null
+  let tname = null
+  if (type._item === null) {
+    tname = ID.findRootTypeKey(type)
+  } else {
+    typeid = type._item.id
+  }
+  return new RelativePosition(typeid, tname, item)
+}

 /**
 * Create a relativePosition based on a absolute position.
 *
- * @param {YType} type The base type (e.g. YText or YArray).
- * @param {Integer} offset The absolute position.
+ * @param {AbstractType} type The base type (e.g. YText or YArray).
+ * @param {number} offset The absolute position.
+ * @return {RelativePosition}
 */
-export const getRelativePosition = (type, offset) => {
-  // TODO: rename to createRelativePosition
+export const createRelativePositionByOffset = (type, offset) => {
  let t = type._start
  while (t !== null) {
-    if (!t._deleted && t._countable) {
-      if (t._length > offset) {
-        return [t._id.user, t._id.clock + offset]
+    if (!t.deleted && t.countable) {
+      if (t.length > offset) {
+        // case 1: found position somewhere in the linked list
+        return createRelativePosition(type, ID.createID(t.id.client, t.id.clock + offset))
      }
-      offset -= t._length
+      offset -= t.length
    }
-    t = t._right
+    t = t.right
  }
-  return ['endof', type._id.user, type._id.clock || null, type._id.name || null, type._id.type || null]
+  return createRelativePosition(type, null)
 }

 /**
- * @typedef {Object} AbsolutePosition The result of {@link fromRelativePosition}
- * @property {YType} type The type on which to apply the absolute position.
- * @property {number} offset The absolute offset.r
+ * @param {encoding.Encoder} encoder
+ * @param {RelativePosition} rpos
 */
+export const writeRelativePosition = (encoder, rpos) => {
+  const { type, tname, item } = rpos
+  if (item !== null) {
+    encoding.writeVarUint(encoder, 0)
+    ID.writeID(encoder, item)
+  } else if (tname !== null) {
+    // case 2: found position at the end of the list and type is stored in y.share
+    encoding.writeUint8(encoder, 1)
+    encoding.writeVarString(encoder, tname)
+  } else if (type !== null) {
+    // case 3: found position at the end of the list and type is attached to an item
+    encoding.writeUint8(encoder, 2)
+    ID.writeID(encoder, type)
+  } else {
+    throw error.unexpectedCase()
+  }
+  return encoder
+}

 /**
- * Transforms a relative position back to a relative position.
- *
- * @param {Y} y The Yjs instance in which to query for the absolute position.
- * @param {RelativePosition} rpos The relative position.
- * @return {AbsolutePosition} The absolute position in the Yjs model
- *                            (type + offset).
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ * @param {StructStore} store
+ * @return {RelativePosition|null}
 */
-export const fromRelativePosition = (y, rpos) => {
-  if (rpos === null) {
+export const readRelativePosition = (decoder, y, store) => {
+  let type = null
+  let tname = null
+  let itemID = null
+  switch (decoding.readVarUint(decoder)) {
+    case 0:
+      // case 1: found position somewhere in the linked list
+      itemID = ID.readID(decoder)
+      break
+    case 1:
+      // case 2: found position at the end of the list and type is stored in y.share
+      tname = decoding.readVarString(decoder)
+      break
+    case 2: {
+      // case 3: found position at the end of the list and type is attached to an item
+      type = ID.readID(decoder)
+    }
+  }
+  return new RelativePosition(type, tname, itemID)
+}
+
+/**
+ * @param {RelativePosition} rpos
+ * @param {StructStore} store
+ * @param {Y} y
+ * @return {AbsolutePosition|null}
+ */
+export const toAbsolutePosition = (rpos, store, y) => {
+  const rightID = rpos.item
+  const typeID = rpos.type
+  const tname = rpos.tname
+  let type = null
+  let offset = 0
+  if (rightID !== null) {
+    if (!exists(store, rightID)) {
+      return null
+    }
+    const right = find(store, rightID)
+    if (!(right instanceof AbstractItem)) {
+      return null
+    }
+    offset = right.deleted ? 0 : rightID.clock - right.id.clock
+    let n = right.left
+    while (n !== null) {
+      if (!n.deleted && n.countable) {
+        offset += n.length
+      }
+      n = n.left
+    }
+    type = right.parent
+  } else {
+    if (tname !== null) {
+      type = y.get(tname)
+    } else if (typeID !== null) {
+      type = getItemType(store, typeID).type
+    } else {
+      throw error.unexpectedCase()
+    }
+    offset = type._length
+  }
+  if (type._item !== null && type._item.deleted) {
    return null
  }
-  if (rpos[0] === 'endof') {
-    let id
-    if (rpos[3] === null) {
-      id = ID.createID(rpos[1], rpos[2])
-    } else {
-      id = ID.createRootID(rpos[3], rpos[4])
-    }
-    let type = y.os.get(id)
-    if (type === null) {
-      return null
-    }
-    while (type._redone !== null) {
-      type = type._redone
-    }
-    if (type === null || type.constructor === GC) {
-      return null
-    }
-    return {
-      type,
-      offset: type.length
-    }
-  } else {
-    let offset = 0
-    let struct = y.os.findNodeWithUpperBound(ID.createID(rpos[0], rpos[1])).val
-    if (struct === null || struct._id.user === ID.RootFakeUserID) {
-      return null // TODO: support fake ids?
-    }
-    const diff = rpos[1] - struct._id.clock
-    while (struct._redone !== null) {
-      struct = struct._redone
-    }
-    const parent = struct._parent
-    if (struct.constructor === GC || parent._deleted) {
-      return null
-    }
-    if (!struct._deleted && struct._countable) {
-      offset = diff
-    }
-    struct = struct._left
-    while (struct !== null) {
-      if (!struct._deleted && struct._countable) {
-        offset += struct._length
-      }
-      struct = struct._left
-    }
-    return {
-      type: parent,
-      offset: offset
-    }
-  }
+  return createAbsolutePosition(type, offset)
 }

-export const equal = (posa, posb) => posa === posb || (posa !== null && posb !== null && posa.length === posb.length && posa.every((v, i) => v === posb[i]))
+/**
+ * Transforms an absolute to a relative position.
+ *
+ * @param {AbsolutePosition} apos The absolute position.
+ * @param {Y} y The Yjs instance in which to query for the absolute position.
+ * @return {RelativePosition} The absolute position in the Yjs model
+ *                            (type + offset).
+ */
+export const toRelativePosition = (apos, y) => {
+  const type = apos.type
+  if (type._length === apos.offset) {
+    return createRelativePosition(type, null)
+  } else {
+    let offset = apos.offset
+    let n = type._start
+    while (n !== null) {
+      if (!n.deleted && n.countable) {
+        if (n.length > offset) {
+          return createRelativePosition(type, ID.createID(n.id.client, n.id.clock + offset))
+        }
+        offset -= n.length
+      }
+      n = n.right
+    }
+  }
+  throw error.unexpectedCase()
+}
+
+/**
+ * @param {RelativePosition|null} a
+ * @param {RelativePosition|null} b
+ */
+export const compareRelativePositions = (a, b) => a === b || (
+  a !== null && b !== null && (
+    (a.item !== null && b.item !== null && ID.compareIDs(a.item, b.item)) ||
+    (a.tname !== null && a.tname === b.tname) ||
+    (a.type !== null && b.type !== null && ID.compareIDs(a.type, b.type))
+  )
+)
--- a/src/utils/structEncoding.js
+++ b/src/utils/structEncoding.js
@@ -1,45 +1,147 @@
 import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
-import { AbstractStruct, AbstractRef } from '../structs/AbstractStruct.js'
+import * as map from 'lib0/map.js'
+import { AbstractStruct, AbstractRef } from '../structs/AbstractStruct.js' // eslint-disable-line
 import * as binary from 'lib0/binary.js'
-import { Transaction } from './Transaction.js'
-import { findIndex } from './StructStore.js'
+import { Transaction } from './Transaction.js' // eslint-disable-line
+import { findIndexSS, exists, StructStore } from './StructStore.js' // eslint-disable-line
+import { writeID, createID, readID, ID } from './ID.js' // eslint-disable-line
+import * as iterator from 'lib0/iterator.js'
+import { ItemBinaryRef } from '../structs/ItemBinary.js'
+import { GCRef } from '../structs/GC.js'
+import { ItemDeletedRef } from '../structs/ItemDeleted.js'
+import { ItemEmbedRef } from '../structs/ItemEmbed.js'
+import { ItemFormatRef } from '../structs/ItemFormat.js'
+import { ItemJSONRef } from '../structs/ItemJSON.js'
+import { ItemStringRef } from '../structs/ItemString.js'
+import { ItemTypeRef } from '../structs/ItemType.js'
+
+/**
+ * @typedef {Map<number, number>} StateMap
+ */

 const structRefs = [
-  ItemBinaryRef
+  ItemBinaryRef,
+  GCRef,
+  ItemDeletedRef,
+  ItemEmbedRef,
+  ItemFormatRef,
+  ItemJSONRef,
+  ItemStringRef,
+  ItemTypeRef
 ]

+/**
+ * @param {decoding.Decoder} decoder
+ * @param {number} structsLen
+ * @param {ID} nextID
+ * @return {Iterator<AbstractRef>}
+ */
+const createStructReaderIterator = (decoder, structsLen, nextID) => iterator.createIterator(() => {
+  let done = false
+  let value
+  if (structsLen === 0) {
+    done = true
+  } else {
+    const info = decoding.readUint8(decoder)
+    value = new structRefs[binary.BITS5 & info](decoder, nextID, info)
+    nextID = createID(nextID.client, nextID.clock)
+  }
+  return { done, value }
+})
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {Transaction} transaction
+ */
+export const writeStructsFromTransaction = (encoder, transaction) => writeStructs(encoder, transaction.y.store, transaction.stateUpdates)
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {StructStore} store
+ * @param {StateMap} sm
+ */
+export const writeStructs = (encoder, store, sm) => {
+  const encoderUserPosMap = map.create()
+  // write # states that were updated
+  encoding.writeVarUint(encoder, sm.size)
+  sm.forEach((client, clock) => {
+    // write first id
+    writeID(encoder, createID(client, clock))
+    encoderUserPosMap.set(client, encoding.length(encoder))
+    // write diff to pos where structs are written
+    // We will fill out this value later *)
+    encoding.writeUint32(encoder, 0)
+  })
+  sm.forEach((client, clock) => {
+    const decPos = encoderUserPosMap.get(client)
+    encoding.setUint32(encoder, decPos, encoding.length(encoder) - decPos)
+    /**
+     * @type {Array<AbstractStruct>}
+     */
+    // @ts-ignore
+    const structs = store.clients.get(client)
+    const startNewStructs = findIndexSS(structs, clock)
+    // write # encoded structs
+    encoding.writeVarUint(encoder, structs.length - startNewStructs)
+    const firstStruct = structs[startNewStructs]
+    // write first struct with an offset (may be 0)
+    firstStruct.write(encoder, clock - firstStruct.id.clock, 0)
+    for (let i = startNewStructs + 1; i < structs.length; i++) {
+      structs[i].write(encoder, 0, 0)
+    }
+  })
+}
+
 /**
 * Read the next Item in a Decoder and fill this Item with the read data.
 *
 * This is called when data is received from a remote peer.
 *
 * @param {decoding.Decoder} decoder The decoder object to read data from.
- * @return {AbstractRef}
+ * @param {Transaction} transaction
+ * @param {StructStore} store
 *
 * @private
 */
-export const read = decoder => {
-  const info = decoding.readUint8(decoder)
-  return new structRefs[binary.BITS5 & info](decoder, info)
-}
-
-/**
- * @param {encoding.Encoder} encoder
- * @param {Transaction} transaction
- */
-export const writeStructsFromTransaction = (encoder, transaction) => {
-  const stateUpdates = transaction.stateUpdates
-  const y = transaction.y
-  encoding.writeVarUint(encoder, stateUpdates.size)
-  stateUpdates.forEach((clock, client) => {
-    /**
-     * @type {Array<AbstractStruct>}
-     */
-    // @ts-ignore
-    const structs = y.store.clients.get(client)
-    for (let i = findIndex(structs, clock); i < structs.length; i++) {
-      structs[i].write(encoder, 0)
+export const readStructs = (decoder, transaction, store) => {
+  /**
+   * @type {Map<number,Iterator<AbstractRef>>}
+   */
+  const structReaders = new Map()
+  const clientStateUpdates = decoding.readVarUint(decoder)
+  for (let i = 0; i < clientStateUpdates; i++) {
+    const nextID = readID(decoder)
+    const decoderPos = decoder.pos + decoding.readUint32(decoder)
+    const structReaderDecoder = decoding.clone(decoder, decoderPos)
+    const numberOfStructs = decoding.readVarUint(structReaderDecoder)
+    structReaders.set(nextID.client, createStructReaderIterator(structReaderDecoder, numberOfStructs, nextID))
+  }
+  /**
+   * @type {Array<AbstractRef>}
+   */
+  const stack = []
+  for (const it of structReaders.values()) {
+    // todo try for in of it
+    for (let res = it.next(); !res.done; res = it.next()) {
+      stack.push(res.value)
+      while (stack.length > 0) {
+        const ref = stack[stack.length - 1]
+        const m = ref._missing
+        while (m.length > 0) {
+          const nextMissing = m[m.length - 1]
+          if (!exists(store, nextMissing)) {
+            // @ts-ignore must not be undefined, otherwise unexpected case
+            stack.push(structReaders.get(nextMissing.client).next().value)
+            break
+          }
+          ref._missing.pop()
+        }
+        if (m.length === 0) {
+          ref.toStruct(transaction).integrate(transaction)
+          stack.pop()
+        }
+      }
    }
-  })
+  }
 }