restructuring the project

src/utils/BindMapping.js

@@ -0,0 +1,67 @@
+/**
+ * Type that maps from Yjs type to Target type.
+ * Used to implement double bindings.
+ *
+ * @private
+ * @template Y
+ * @template T
+ */
+export class BindMapping {
+  /**
+   */
+  constructor () {
+    /**
+     * @type Map<Y, T>
+     */
+    this.yt = new Map()
+    /**
+     * @type Map<T, Y>
+     */
+    this.ty = new Map()
+  }
+  /**
+   * Map y to t. Removes all existing bindings from y and t
+   * @param {Y} y
+   * @param {T} t
+   */
+  bind (y, t) {
+    const existingT = this.yt.get(y)
+    if (existingT !== undefined) {
+      this.ty.delete(existingT)
+    }
+    const existingY = this.ty.get(t)
+    if (existingY !== undefined) {
+      this.yt.delete(existingY)
+    }
+    this.yt.set(y, t)
+    this.ty.set(t, y)
+  }
+  /**
+   * @param {Y} y
+   * @return {boolean}
+   */
+  hasY (y) {
+    return this.yt.has(y)
+  }
+  /**
+   * @param {T} t
+   * @return {boolean}
+   */
+  hasT (t) {
+    return this.ty.has(t)
+  }
+  /**
+   * @param {Y} y
+   * @return {T}
+   */
+  getY (y) {
+    return this.yt.get(y)
+  }
+  /**
+   * @param {T} t
+   * @return {Y}
+   */
+  getT (t) {
+    return this.ty.get(t)
+  }
+}

src/utils/DeleteStore.js

@@ -0,0 +1,89 @@
+/**
+ * @module utils
+ */
+
+import { Tree } from 'funlib/tree.js'
+import * as ID from './ID.js'
+
+export class DSNode {
+  constructor (id, len, gc) {
+    this._id = id
+    this.len = len
+    this.gc = gc
+  }
+  clone () {
+    return new DSNode(this._id, this.len, this.gc)
+  }
+}
+
+export class DeleteStore extends Tree {
+  logTable () {
+    const deletes = []
+    this.iterate(null, null, n => {
+      deletes.push({
+        user: n._id.user,
+        clock: n._id.clock,
+        len: n.len,
+        gc: n.gc
+      })
+    })
+    console.table(deletes)
+  }
+  isDeleted (id) {
+    var n = this.findWithUpperBound(id)
+    return n !== null && n._id.user === id.user && id.clock < n._id.clock + n.len
+  }
+  mark (id, length, gc) {
+    if (length === 0) return
+    // Step 1. Unmark range
+    const leftD = this.findWithUpperBound(ID.createID(id.user, id.clock - 1))
+    // Resize left DSNode if necessary
+    if (leftD !== null && leftD._id.user === id.user) {
+      if (leftD._id.clock < id.clock && id.clock < leftD._id.clock + leftD.len) {
+        // node is overlapping. need to resize
+        if (id.clock + length < leftD._id.clock + leftD.len) {
+          // overlaps new mark range and some more
+          // create another DSNode to the right of new mark
+          this.put(new DSNode(ID.createID(id.user, id.clock + length), leftD._id.clock + leftD.len - id.clock - length, leftD.gc))
+        }
+        // resize left DSNode
+        leftD.len = id.clock - leftD._id.clock
+      } // Otherwise there is no overlapping
+    }
+    // Resize right DSNode if necessary
+    const upper = ID.createID(id.user, id.clock + length - 1)
+    const rightD = this.findWithUpperBound(upper)
+    if (rightD !== null && rightD._id.user === id.user) {
+      if (rightD._id.clock < id.clock + length && id.clock <= rightD._id.clock && id.clock + length < rightD._id.clock + rightD.len) { // we only consider the case where we resize the node
+        const d = id.clock + length - rightD._id.clock
+        rightD._id = ID.createID(rightD._id.user, rightD._id.clock + d)
+        rightD.len -= d
+      }
+    }
+    // Now we only have to delete all inner marks
+    const deleteNodeIds = []
+    this.iterate(id, upper, m => {
+      deleteNodeIds.push(m._id)
+    })
+    for (let i = deleteNodeIds.length - 1; i >= 0; i--) {
+      this.delete(deleteNodeIds[i])
+    }
+    let newMark = new DSNode(id, length, gc)
+    // Step 2. Check if we can extend left or right
+    if (leftD !== null && leftD._id.user === id.user && leftD._id.clock + leftD.len === id.clock && leftD.gc === gc) {
+      // We can extend left
+      leftD.len += length
+      newMark = leftD
+    }
+    const rightNext = this.find(ID.createID(id.user, id.clock + length))
+    if (rightNext !== null && rightNext._id.user === id.user && id.clock + length === rightNext._id.clock && gc === rightNext.gc) {
+      // We can merge newMark and rightNext
+      newMark.len += rightNext.len
+      this.delete(rightNext._id)
+    }
+    if (leftD !== newMark) {
+      // only put if we didn't extend left
+      this.put(newMark)
+    }
+  }
+}

src/utils/EventHandler.js

@@ -0,0 +1,70 @@
+/**
+ * @module utils
+ */
+
+/**
+ * General event handler implementation.
+ */
+export class EventHandler {
+  constructor () {
+    this.eventListeners = []
+  }
+
+  /**
+   * To prevent memory leaks, call this method when the eventListeners won't be
+   * used anymore.
+   */
+  destroy () {
+    this.eventListeners = null
+  }
+
+  /**
+   * Adds an event listener that is called when
+   * {@link EventHandler#callEventListeners} is called.
+   *
+   * @param {Function} f The event handler.
+   */
+  addEventListener (f) {
+    this.eventListeners.push(f)
+  }
+
+  /**
+   * Removes an event listener.
+   *
+   * @param {Function} f The event handler that was added with
+   *                     {@link EventHandler#addEventListener}
+   */
+  removeEventListener (f) {
+    this.eventListeners = this.eventListeners.filter(g => f !== g)
+  }
+
+  /**
+   * Removes all event listeners.
+   */
+  removeAllEventListeners () {
+    this.eventListeners = []
+  }
+
+  /**
+   * Call all event listeners that were added via
+   * {@link EventHandler#addEventListener}.
+   *
+   * @param {Transaction} transaction The transaction object
+   * @param {YEvent} event An event object that describes the change on a type.
+   */
+  callEventListeners (transaction, event) {
+    for (var i = 0; i < this.eventListeners.length; i++) {
+      try {
+        const f = this.eventListeners[i]
+        f(event, transaction)
+      } catch (e) {
+        /*
+          Your observer threw an error. This error was caught so that Yjs
+          can ensure data consistency! In order to debug this error you
+          have to check "Pause On Caught Exceptions" in developer tools.
+        */
+        console.error(e)
+      }
+    }
+  }
+}

src/utils/ID.js

@@ -0,0 +1,94 @@
+/**
+ * @module utils
+ */
+
+import { getStructReference } from 'y-protocols/sync.js'
+import * as decoding from 'funlib/decoding.js'
+import * as encoding from 'funlib/encoding.js'
+
+export class ID {
+  constructor (user, clock) {
+    this.user = user // TODO: rename to client
+    this.clock = clock
+  }
+  clone () {
+    return new ID(this.user, this.clock)
+  }
+  equals (id) {
+    return id !== null && id.user === this.user && id.clock === this.clock
+  }
+  lessThan (id) {
+    if (id.constructor === ID) {
+      return this.user < id.user || (this.user === id.user && this.clock < id.clock)
+    } else {
+      return false
+    }
+  }
+  /**
+   * @param {encoding.Encoder} encoder
+   */
+  encode (encoder) {
+    encoding.writeVarUint(encoder, this.user)
+    encoding.writeVarUint(encoder, this.clock)
+  }
+}
+
+export const createID = (user, clock) => new ID(user, clock)
+
+export const RootFakeUserID = 0xFFFFFF
+
+export class RootID {
+  constructor (name, typeConstructor) {
+    this.user = RootFakeUserID
+    this.name = name
+    this.type = getStructReference(typeConstructor)
+  }
+  equals (id) {
+    return id !== null && id.user === this.user && id.name === this.name && id.type === this.type
+  }
+  lessThan (id) {
+    if (id.constructor === RootID) {
+      return this.user < id.user || (this.user === id.user && (this.name < id.name || (this.name === id.name && this.type < id.type)))
+    } else {
+      return true
+    }
+  }
+  /**
+   * @param {encoding.Encoder} encoder
+   */
+  encode (encoder) {
+    encoding.writeVarUint(encoder, this.user)
+    encoding.writeVarString(encoder, this.name)
+    encoding.writeVarUint(encoder, this.type)
+  }
+}
+
+/**
+ * Create a new root id.
+ *
+ * @example
+ *   y.define('name', Y.Array) // name, and typeConstructor
+ *
+ * @param {string} name
+ * @param {Function} typeConstructor must be defined in structReferences
+ */
+export const createRootID = (name, typeConstructor) => new RootID(name, typeConstructor)
+
+/**
+ * Read ID.
+ * * If first varUint read is 0xFFFFFF a RootID is returned.
+ * * Otherwise an ID is returned
+ *
+ * @param {decoding.Decoder} decoder
+ * @return {ID|RootID}
+ */
+export const decode = decoder => {
+  const user = decoding.readVarUint(decoder)
+  if (user === RootFakeUserID) {
+    // read property name and type id
+    const rid = createRootID(decoding.readVarString(decoder), null)
+    rid.type = decoding.readVarUint(decoder)
+    return rid
+  }
+  return createID(user, decoding.readVarUint(decoder))
+}

src/utils/OperationStore.js

@@ -0,0 +1,98 @@
+/**
+ * @module utils
+ */
+
+import { Tree } from 'funlib/tree.js'
+import * as ID from '../utils/ID.js'
+import { getStruct } from 'y-protocols/sync.js'
+import { GC } from '../structs/GC.js'
+import * as stringify from 'y-protocols/utils/structStringify.js'
+
+export class OperationStore extends Tree {
+  constructor (y) {
+    super()
+    this.y = y
+  }
+  logTable () {
+    const items = []
+    this.iterate(null, null, item => {
+      if (item.constructor === GC) {
+        items.push({
+          id: stringify.stringifyItemID(item),
+          content: item._length,
+          deleted: 'GC'
+        })
+      } else {
+        items.push({
+          id: stringify.stringifyItemID(item),
+          origin: item._origin === null ? '()' : stringify.stringifyID(item._origin._lastId),
+          left: item._left === null ? '()' : stringify.stringifyID(item._left._lastId),
+          right: stringify.stringifyItemID(item._right),
+          right_origin: stringify.stringifyItemID(item._right_origin),
+          parent: stringify.stringifyItemID(item._parent),
+          parentSub: item._parentSub,
+          deleted: item._deleted,
+          content: JSON.stringify(item._content)
+        })
+      }
+    })
+    console.table(items)
+  }
+  get (id) {
+    let struct = this.find(id)
+    if (struct === null && id instanceof ID.RootID) {
+      const Constr = getStruct(id.type)
+      const y = this.y
+      struct = new Constr()
+      struct._id = id
+      struct._parent = y
+      y.transact(() => {
+        struct._integrate(y)
+      })
+      this.put(struct)
+    }
+    return struct
+  }
+  // Use getItem for structs with _length > 1
+  getItem (id) {
+    var item = this.findWithUpperBound(id)
+    if (item === null) {
+      return null
+    }
+    const itemID = item._id
+    if (id.user === itemID.user && id.clock < itemID.clock + item._length) {
+      return item
+    } else {
+      return null
+    }
+  }
+  // Return an insertion such that id is the first element of content
+  // This function manipulates an item, if necessary
+  getItemCleanStart (id) {
+    var ins = this.getItem(id)
+    if (ins === null || ins._length === 1) {
+      return ins
+    }
+    const insID = ins._id
+    if (insID.clock === id.clock) {
+      return ins
+    } else {
+      return ins._splitAt(this.y, id.clock - insID.clock)
+    }
+  }
+  // Return an insertion such that id is the last element of content
+  // This function manipulates an operation, if necessary
+  getItemCleanEnd (id) {
+    var ins = this.getItem(id)
+    if (ins === null || ins._length === 1) {
+      return ins
+    }
+    const insID = ins._id
+    if (insID.clock + ins._length - 1 === id.clock) {
+      return ins
+    } else {
+      ins._splitAt(this.y, id.clock - insID.clock + 1)
+      return ins
+    }
+  }
+}

src/utils/StateStore.js

@@ -0,0 +1,53 @@
+/**
+ * @module utils
+ */
+
+import * as ID from '../utils/ID.js'
+
+/**
+ */
+export class StateStore {
+  constructor (y) {
+    this.y = y
+    this.state = new Map()
+  }
+  logTable () {
+    const entries = []
+    for (let [user, state] of this.state) {
+      entries.push({
+        user, state
+      })
+    }
+    console.table(entries)
+  }
+  getNextID (len) {
+    const user = this.y.userID
+    const state = this.getState(user)
+    this.setState(user, state + len)
+    return ID.createID(user, state)
+  }
+  updateRemoteState (struct) {
+    let user = struct._id.user
+    let userState = this.state.get(user)
+    while (struct !== null && struct._id.clock === userState) {
+      userState += struct._length
+      struct = this.y.os.get(ID.createID(user, userState))
+    }
+    this.state.set(user, userState)
+  }
+  getState (user) {
+    let state = this.state.get(user)
+    if (state == null) {
+      return 0
+    }
+    return state
+  }
+  setState (user, state) {
+    // TODO: modify missingi structs here
+    const beforeState = this.y._transaction.beforeState
+    if (!beforeState.has(user)) {
+      beforeState.set(user, this.getState(user))
+    }
+    this.state.set(user, state)
+  }
+}

src/utils/Transaction.js

@@ -0,0 +1,69 @@
+/**
+ * @module utils
+ */
+
+import * as encoding from 'funlib/encoding.js'
+
+/**
+ * A transaction is created for every change on the Yjs model. It is possible
+ * to bundle changes on the Yjs model in a single transaction to
+ * minimize the number on messages sent and the number of observer calls.
+ * If possible the user of this library should bundle as many changes as
+ * possible. Here is an example to illustrate the advantages of bundling:
+ *
+ * @example
+ * const map = y.define('map', YMap)
+ * // Log content when change is triggered
+ * map.observe(() => {
+ *   console.log('change triggered')
+ * })
+ * // Each change on the map type triggers a log message:
+ * map.set('a', 0) // => "change triggered"
+ * map.set('b', 0) // => "change triggered"
+ * // When put in a transaction, it will trigger the log after the transaction:
+ * y.transact(() => {
+ *   map.set('a', 1)
+ *   map.set('b', 1)
+ * }) // => "change triggered"
+ *
+ */
+export class Transaction {
+  constructor (y) {
+    /**
+     * @type {Y} The Yjs instance.
+     */
+    this.y = y
+    /**
+     * All new types that are added during a transaction.
+     * @type {Set<Item>}
+     */
+    this.newTypes = new Set()
+    /**
+     * All types that were directly modified (property added or child
+     * inserted/deleted). New types are not included in this Set.
+     * Maps from type to parentSubs (`item._parentSub = null` for YArray)
+     * @type {Map<Type|Y,String>}
+     */
+    this.changedTypes = new Map()
+    // TODO: rename deletedTypes
+    /**
+     * Set of all deleted Types and Structs.
+     * @type {Set<Item>}
+     */
+    this.deletedStructs = new Set()
+    /**
+     * Saves the old state set of the Yjs instance. If a state was modified,
+     * the original value is saved here.
+     * @type {Map<Number,Number>}
+     */
+    this.beforeState = new Map()
+    /**
+     * Stores the events for the types that observe also child elements.
+     * It is mainly used by `observeDeep`.
+     * @type {Map<Type,Array<YEvent>>}
+     */
+    this.changedParentTypes = new Map()
+    this.encodedStructsLen = 0
+    this.encodedStructs = encoding.createEncoder()
+  }
+}

src/utils/UndoManager.js

@@ -0,0 +1,180 @@
+
+import * as ID from './ID.js'
+import { isParentOf } from './isParentOf.js'
+
+class ReverseOperation {
+  constructor (y, transaction, bindingInfos) {
+    this.created = new Date()
+    const beforeState = transaction.beforeState
+    if (beforeState.has(y.userID)) {
+      this.toState = ID.createID(y.userID, y.ss.getState(y.userID) - 1)
+      this.fromState = ID.createID(y.userID, beforeState.get(y.userID))
+    } else {
+      this.toState = null
+      this.fromState = null
+    }
+    this.deletedStructs = new Set()
+    transaction.deletedStructs.forEach(struct => {
+      this.deletedStructs.add({
+        from: struct._id,
+        len: struct._length
+      })
+    })
+    /**
+     * Maps from binding to binding information (e.g. cursor information)
+     */
+    this.bindingInfos = bindingInfos
+  }
+}
+
+function applyReverseOperation (y, scope, reverseBuffer) {
+  let performedUndo = false
+  let undoOp = null
+  y.transact(() => {
+    while (!performedUndo && reverseBuffer.length > 0) {
+      undoOp = reverseBuffer.pop()
+      // make sure that it is possible to iterate {from}-{to}
+      if (undoOp.fromState !== null) {
+        y.os.getItemCleanStart(undoOp.fromState)
+        y.os.getItemCleanEnd(undoOp.toState)
+        y.os.iterate(undoOp.fromState, undoOp.toState, op => {
+          while (op._deleted && op._redone !== null) {
+            op = op._redone
+          }
+          if (op._deleted === false && isParentOf(scope, op)) {
+            performedUndo = true
+            op._delete(y)
+          }
+        })
+      }
+      const redoitems = new Set()
+      for (let del of undoOp.deletedStructs) {
+        const fromState = del.from
+        const toState = ID.createID(fromState.user, fromState.clock + del.len - 1)
+        y.os.getItemCleanStart(fromState)
+        y.os.getItemCleanEnd(toState)
+        y.os.iterate(fromState, toState, op => {
+          if (
+            isParentOf(scope, op) &&
+            op._parent !== y &&
+            (
+              op._id.user !== y.userID ||
+              undoOp.fromState === null ||
+              op._id.clock < undoOp.fromState.clock ||
+              op._id.clock > undoOp.toState.clock
+            )
+          ) {
+            redoitems.add(op)
+          }
+        })
+      }
+      redoitems.forEach(op => {
+        const opUndone = op._redo(y, redoitems)
+        performedUndo = performedUndo || opUndone
+      })
+    }
+  })
+  if (performedUndo && undoOp !== null) {
+    // should be performed after the undo transaction
+    undoOp.bindingInfos.forEach((info, binding) => {
+      binding._restoreUndoStackInfo(info)
+    })
+  }
+  return performedUndo
+}
+
+/**
+ * Saves a history of locally applied operations. The UndoManager handles the
+ * undoing and redoing of locally created changes.
+ */
+export class UndoManager {
+  /**
+   * @param {YType} scope The scope on which to listen for changes.
+   * @param {Object} options Optionally provided configuration.
+   */
+  constructor (scope, options = {}) {
+    this.options = options
+    this._bindings = new Set(options.bindings)
+    options.captureTimeout = options.captureTimeout == null ? 500 : options.captureTimeout
+    this._undoBuffer = []
+    this._redoBuffer = []
+    this._scope = scope
+    this._undoing = false
+    this._redoing = false
+    this._lastTransactionWasUndo = false
+    const y = scope._y
+    this.y = y
+    y._hasUndoManager = true
+    let bindingInfos
+    y.on('beforeTransaction', (y, transaction, remote) => {
+      if (!remote) {
+        // Store binding information before transaction is executed
+        // By restoring the binding information, we can make sure that the state
+        // before the transaction can be recovered
+        bindingInfos = new Map()
+        this._bindings.forEach(binding => {
+          bindingInfos.set(binding, binding._getUndoStackInfo())
+        })
+      }
+    })
+    y.on('afterTransaction', (y, transaction, remote) => {
+      if (!remote && transaction.changedParentTypes.has(scope)) {
+        let reverseOperation = new ReverseOperation(y, transaction, bindingInfos)
+        if (!this._undoing) {
+          let lastUndoOp = this._undoBuffer.length > 0 ? this._undoBuffer[this._undoBuffer.length - 1] : null
+          if (
+            this._redoing === false &&
+            this._lastTransactionWasUndo === false &&
+            lastUndoOp !== null &&
+            (options.captureTimeout < 0 || reverseOperation.created - lastUndoOp.created <= options.captureTimeout)
+          ) {
+            lastUndoOp.created = reverseOperation.created
+            if (reverseOperation.toState !== null) {
+              lastUndoOp.toState = reverseOperation.toState
+              if (lastUndoOp.fromState === null) {
+                lastUndoOp.fromState = reverseOperation.fromState
+              }
+            }
+            reverseOperation.deletedStructs.forEach(lastUndoOp.deletedStructs.add, lastUndoOp.deletedStructs)
+          } else {
+            this._lastTransactionWasUndo = false
+            this._undoBuffer.push(reverseOperation)
+          }
+          if (!this._redoing) {
+            this._redoBuffer = []
+          }
+        } else {
+          this._lastTransactionWasUndo = true
+          this._redoBuffer.push(reverseOperation)
+        }
+      }
+    })
+  }
+
+  /**
+   * Enforce that the next change is created as a separate item in the undo stack
+   */
+  flushChanges () {
+    this._lastTransactionWasUndo = true
+  }
+
+  /**
+   * Undo the last locally created change.
+   */
+  undo () {
+    this._undoing = true
+    const performedUndo = applyReverseOperation(this.y, this._scope, this._undoBuffer)
+    this._undoing = false
+    return performedUndo
+  }
+
+  /**
+   * Redo the last locally created change.
+   */
+  redo () {
+    this._redoing = true
+    const performedRedo = applyReverseOperation(this.y, this._scope, this._redoBuffer)
+    this._redoing = false
+    return performedRedo
+  }
+}

src/utils/Y.js

@@ -0,0 +1,181 @@
+import { DeleteStore } from './DeleteStore.js'
+import { OperationStore } from './OperationStore.js'
+import { StateStore } from './StateStore.js'
+import * as random from 'funlib/random.js'
+import { createRootID } from './ID.js'
+import { Observable } from 'funlib/observable.js'
+import { Transaction } from './Transaction.js'
+
+/**
+ * Anything that can be encoded with `JSON.stringify` and can be decoded with
+ * `JSON.parse`.
+ *
+ * The following property should hold:
+ * `JSON.parse(JSON.stringify(key))===key`
+ *
+ * At the moment the only safe values are number and string.
+ *
+ * @typedef {(number|string|Object)} encodable
+ */
+
+/**
+ * A Yjs instance handles the state of shared data.
+ */
+export class Y extends Observable {
+  /**
+   * @param {Object} [conf] configuration
+   */
+  constructor (conf = {}) {
+    super()
+    this.gcEnabled = conf.gc || false
+    this._contentReady = false
+    this.userID = random.uint32()
+    // TODO: This should be a Map so we can use encodables as keys
+    this._map = new Map()
+    this.ds = new DeleteStore()
+    this.os = new OperationStore(this)
+    this.ss = new StateStore(this)
+    this._missingStructs = new Map()
+    this._readyToIntegrate = []
+    this._transaction = null
+    this.connected = false
+    // for compatibility with isParentOf
+    this._parent = null
+    this._hasUndoManager = false
+    this._deleted = false // for compatiblity of having this as a parent for types
+    this._id = null
+  }
+  _beforeChange () {}
+  _callObserver (transaction, subs, remote) {}
+  /**
+   * Changes that happen inside of a transaction are bundled. This means that
+   * the observer fires _after_ the transaction is finished and that all changes
+   * that happened inside of the transaction are sent as one message to the
+   * other peers.
+   *
+   * @param {Function} f The function that should be executed as a transaction
+   * @param {?Boolean} remote Optional. Whether this transaction is initiated by
+   *                          a remote peer. This should not be set manually!
+   *                          Defaults to false.
+   */
+  transact (f, remote = false) {
+    let initialCall = this._transaction === null
+    if (initialCall) {
+      this._transaction = new Transaction(this)
+      this.emit('beforeTransaction', [this, this._transaction, remote])
+    }
+    try {
+      f(this)
+    } catch (e) {
+      console.error(e)
+    }
+    if (initialCall) {
+      this.emit('beforeObserverCalls', [this, this._transaction, remote])
+      const transaction = this._transaction
+      this._transaction = null
+      // emit change events on changed types
+      transaction.changedTypes.forEach((subs, type) => {
+        if (!type._deleted) {
+          type._callObserver(transaction, subs, remote)
+        }
+      })
+      transaction.changedParentTypes.forEach((events, type) => {
+        if (!type._deleted) {
+          events = events
+            .filter(event =>
+              !event.target._deleted
+            )
+          events
+            .forEach(event => {
+              event.currentTarget = type
+            })
+          // we don't have to check for events.length
+          // because there is no way events is empty..
+          type._deepEventHandler.callEventListeners(transaction, events)
+        }
+      })
+      // when all changes & events are processed, emit afterTransaction event
+      this.emit('afterTransaction', [this, transaction, remote])
+    }
+  }
+
+  /**
+   * Fake _start for root properties (y.set('name', type))
+   *
+   * @private
+   */
+  get _start () {
+    return null
+  }
+
+  /**
+   * Fake _start for root properties (y.set('name', type))
+   *
+   * @private
+   */
+  set _start (start) {}
+
+  /**
+   * Define a shared data type.
+   *
+   * Multiple calls of `y.define(name, TypeConstructor)` yield the same result
+   * and do not overwrite each other. I.e.
+   * `y.define(name, type) === y.define(name, type)`
+   *
+   * After this method is called, the type is also available on `y._map.get(name)`.
+   *
+   * *Best Practices:*
+   * Either define all types right after the Yjs instance is created or always
+   * use `y.define(..)` when accessing a type.
+   *
+   * @example
+   *   // Option 1
+   *   const y = new Y(..)
+   *   y.define('myArray', YArray)
+   *   y.define('myMap', YMap)
+   *   // .. when accessing the type use y._map.get(name)
+   *   y.share.myArray.insert(..)
+   *   y.share.myMap.set(..)
+   *
+   *   // Option2
+   *   const y = new Y(..)
+   *   // .. when accessing the type use `y.define(..)`
+   *   y.define('myArray', YArray).insert(..)
+   *   y.define('myMap', YMap).set(..)
+   *
+   * @param {String} name
+   * @param {Function} TypeConstructor The constructor of the type definition
+   * @returns {any} The created type. Constructed with TypeConstructor
+   */
+  define (name, TypeConstructor) {
+    let id = createRootID(name, TypeConstructor)
+    let type = this.os.get(id)
+    if (this._map.get(name) === undefined) {
+      this._map.set(name, type)
+    } else if (this._map.get(name) !== type) {
+      throw new Error('Type is already defined with a different constructor')
+    }
+    return type
+  }
+
+  /**
+   * Get a defined type. The type must be defined locally. First define the
+   * type with {@link define}.
+   *
+   * This returns the same value as `y.share[name]`
+   *
+   * @param {String} name The typename
+   * @return {any}
+   */
+  get (name) {
+    return this._map.get(name)
+  }
+
+  /**
+   * Disconnect from the room, and destroy all traces of this Yjs instance.
+   */
+  destroy () {
+    this.emit('destroyed', [true])
+    super.destroy()
+  }
+}

src/utils/YEvent.js

@@ -0,0 +1,39 @@
+/**
+ * @module utils
+ */
+
+/**
+ * YEvent describes the changes on a YType.
+ */
+export class YEvent {
+  /**
+   * @param {Type} target The changed type.
+   */
+  constructor (target) {
+    /**
+     * The type on which this event was created on.
+     * @type {Type}
+     */
+    this.target = target
+    /**
+     * The current target on which the observe callback is called.
+     * @type {Type}
+     */
+    this.currentTarget = target
+  }
+
+  /**
+   * Computes the path from `y` to the changed type.
+   *
+   * The following property holds:
+   * @example
+   *   let type = y
+   *   event.path.forEach(dir => {
+   *     type = type.get(dir)
+   *   })
+   *   type === event.target // => true
+   */
+  get path () {
+    return this.currentTarget.getPathTo(this.target)
+  }
+}

src/utils/defragmentItemContent.js

@@ -0,0 +1,61 @@
+
+/**
+ * @module utils
+ */
+
+import * as ID from '../utils/ID.js'
+import { ItemJSON } from '../structs/ItemJSON.js'
+import { ItemString } from '../structs/ItemString.js'
+
+/**
+ * Try to merge all items in os with their successors.
+ *
+ * Some transformations (like delete) fragment items.
+ * Item(c: 'ab') + Delete(1,1) + Delete(0, 1) -> Item(c: 'a',deleted);Item(c: 'b',deleted)
+ *
+ * This functions merges the fragmented nodes together:
+ * Item(c: 'a',deleted);Item(c: 'b',deleted) -> Item(c: 'ab', deleted)
+ *
+ * TODO: The Tree implementation does not support deletions in-spot.
+ *       This is why all deletions must be performed after the traversal.
+ *
+ */
+export const defragmentItemContent = y => {
+  const os = y.os
+  if (os.length < 2) {
+    return
+  }
+  let deletes = []
+  let node = os.findSmallestNode()
+  let next = node.next()
+  while (next !== null) {
+    let a = node.val
+    let b = next.val
+    if (
+      (a instanceof ItemJSON || a instanceof ItemString) &&
+      a.constructor === b.constructor &&
+      a._deleted === b._deleted &&
+      a._right === b &&
+      (ID.createID(a._id.user, a._id.clock + a._length)).equals(b._id)
+    ) {
+      a._right = b._right
+      if (a instanceof ItemJSON) {
+        a._content = a._content.concat(b._content)
+      } else if (a instanceof ItemString) {
+        a._content += b._content
+      }
+      // delete b later
+      deletes.push(b._id)
+      // do not iterate node!
+      // !(node = next)
+    } else {
+      // not able to merge node, get next node
+      node = next
+    }
+    // update next
+    next = next.next()
+  }
+  for (let i = deletes.length - 1; i >= 0; i--) {
+    os.delete(deletes[i])
+  }
+}

src/utils/integrateRemoteStructs.js

@@ -0,0 +1,146 @@
+/**
+ * @module utils
+ */
+
+import { getStruct } from 'y-protocols/sync.js'
+import * as decoding from 'funlib/decoding.js'
+import { GC } from '../structs/GC.js'
+import { Y } from '../utils/Y.js' // eslint-disable-line
+import { Item } from '../structs/Item.js' // eslint-disable-line
+
+class MissingEntry {
+  constructor (decoder, missing, struct) {
+    this.decoder = decoder
+    this.missing = missing.length
+    this.struct = struct
+  }
+}
+
+/**
+ * @private
+ * Integrate remote struct
+ * When a remote struct is integrated, other structs might be ready to ready to
+ * integrate.
+ * @param {Y} y
+ * @param {Item} struct
+ */
+function _integrateRemoteStructHelper (y, struct) {
+  const id = struct._id
+  if (id === undefined) {
+    struct._integrate(y)
+  } else {
+    if (y.ss.getState(id.user) > id.clock) {
+      return
+    }
+    if (!y.gcEnabled || struct.constructor === GC || (struct._parent.constructor !== GC && struct._parent._deleted === false)) {
+      // Is either a GC or Item with an undeleted parent
+      // save to integrate
+      struct._integrate(y)
+    } else {
+      // Is an Item. parent was deleted.
+      struct._gc(y)
+    }
+    let msu = y._missingStructs.get(id.user)
+    if (msu != null) {
+      let clock = id.clock
+      const finalClock = clock + struct._length
+      for (;clock < finalClock; clock++) {
+        const missingStructs = msu.get(clock)
+        if (missingStructs !== undefined) {
+          missingStructs.forEach(missingDef => {
+            missingDef.missing--
+            if (missingDef.missing === 0) {
+              const decoder = missingDef.decoder
+              let oldPos = decoder.pos
+              let missing = missingDef.struct._fromBinary(y, decoder)
+              decoder.pos = oldPos
+              if (missing.length === 0) {
+                y._readyToIntegrate.push(missingDef.struct)
+              } else {
+                // TODO: throw error here
+              }
+            }
+          })
+          msu.delete(clock)
+        }
+      }
+      if (msu.size === 0) {
+        y._missingStructs.delete(id.user)
+      }
+    }
+  }
+}
+
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ */
+export const integrateRemoteStructs = (decoder, y) => {
+  const len = decoding.readUint32(decoder)
+  for (let i = 0; i < len; i++) {
+    let reference = decoding.readVarUint(decoder)
+    let Constr = getStruct(reference)
+    let struct = new Constr()
+    let decoderPos = decoder.pos
+    let missing = struct._fromBinary(y, decoder)
+    if (missing.length === 0) {
+      while (struct != null) {
+        _integrateRemoteStructHelper(y, struct)
+        struct = y._readyToIntegrate.shift()
+      }
+    } else {
+      let _decoder = decoding.createDecoder(decoder.arr.buffer)
+      _decoder.pos = decoderPos
+      let missingEntry = new MissingEntry(_decoder, missing, struct)
+      let missingStructs = y._missingStructs
+      for (let i = missing.length - 1; i >= 0; i--) {
+        let m = missing[i]
+        if (!missingStructs.has(m.user)) {
+          missingStructs.set(m.user, new Map())
+        }
+        let msu = missingStructs.get(m.user)
+        if (!msu.has(m.clock)) {
+          msu.set(m.clock, [])
+        }
+        let mArray = msu = msu.get(m.clock)
+        mArray.push(missingEntry)
+      }
+    }
+  }
+}
+
+// TODO: use this above / refactor
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ */
+export const integrateRemoteStruct = (decoder, y) => {
+  let reference = decoding.readVarUint(decoder)
+  let Constr = getStruct(reference)
+  let struct = new Constr()
+  let decoderPos = decoder.pos
+  let missing = struct._fromBinary(y, decoder)
+  if (missing.length === 0) {
+    while (struct != null) {
+      _integrateRemoteStructHelper(y, struct)
+      struct = y._readyToIntegrate.shift()
+    }
+  } else {
+    let _decoder = decoding.createDecoder(decoder.arr.buffer)
+    _decoder.pos = decoderPos
+    let missingEntry = new MissingEntry(_decoder, missing, struct)
+    let missingStructs = y._missingStructs
+    for (let i = missing.length - 1; i >= 0; i--) {
+      let m = missing[i]
+      if (!missingStructs.has(m.user)) {
+        missingStructs.set(m.user, new Map())
+      }
+      let msu = missingStructs.get(m.user)
+      if (!msu.has(m.clock)) {
+        msu.set(m.clock, [])
+      }
+      let mArray = msu = msu.get(m.clock)
+      mArray.push(missingEntry)
+    }
+  }
+}

src/utils/isParentOf.js

@@ -0,0 +1,26 @@
+/**
+ * @module utils
+ */
+
+import { Y } from '../utils/Y.js' // eslint-disable-line
+import { Type } from '../structs/Type.js' // eslint-disable-line
+
+/**
+ * Check if `parent` is a parent of `child`.
+ *
+ * @param {Type | Y} parent
+ * @param {Type | Y} child
+ * @return {Boolean} Whether `parent` is a parent of `child`.
+ *
+ * @public
+ */
+export const isParentOf = (parent, child) => {
+  child = child._parent
+  while (child !== null) {
+    if (child === parent) {
+      return true
+    }
+    child = child._parent
+  }
+  return false
+}

src/utils/relativePosition.js

@@ -0,0 +1,130 @@
+/**
+ * @module utils
+ */
+
+import * as ID from './ID.js'
+import { GC } from '../structs/GC.js'
+
+// TODO: Implement function to describe ranges
+
+/**
+ * A relative position that is based on the Yjs model. In contrast to an
+ * absolute position (position by index), the relative position can be
+ * recomputed when remote changes are received. For example:
+ *
+ * ```Insert(0, 'x')('a|bc') = 'xa|bc'``` Where | is the cursor position.
+ *
+ * A relative cursor position can be obtained with the function
+ * {@link getRelativePosition} and it can be transformed to an absolute position
+ * with {@link fromRelativePosition}.
+ *
+ * Pro tip: Use this to implement shared cursor locations in YText or YXml!
+ * The relative position is {@link encodable}, so you can send it to other
+ * clients.
+ *
+ * @example
+ * // Current cursor position is at position 10
+ * let relativePosition = getRelativePosition(yText, 10)
+ * // modify yText
+ * yText.insert(0, 'abc')
+ * yText.delete(3, 10)
+ * // Compute the cursor position
+ * let absolutePosition = fromRelativePosition(y, relativePosition)
+ * absolutePosition.type // => yText
+ * console.log('cursor location is ' + absolutePosition.offset) // => cursor location is 3
+ *
+ * @typedef {encodable} RelativePosition
+ */
+
+/**
+ * Create a relativePosition based on a absolute position.
+ *
+ * @param {YType} type The base type (e.g. YText or YArray).
+ * @param {Integer} offset The absolute position.
+ */
+export const getRelativePosition = (type, offset) => {
+  // TODO: rename to createRelativePosition
+  let t = type._start
+  while (t !== null) {
+    if (!t._deleted && t._countable) {
+      if (t._length > offset) {
+        return [t._id.user, t._id.clock + offset]
+      }
+      offset -= t._length
+    }
+    t = t._right
+  }
+  return ['endof', type._id.user, type._id.clock || null, type._id.name || null, type._id.type || null]
+}
+
+/**
+ * @typedef {Object} AbsolutePosition The result of {@link fromRelativePosition}
+ * @property {YType} type The type on which to apply the absolute position.
+ * @property {number} offset The absolute offset.r
+ */
+
+/**
+ * Transforms a relative position back to a relative position.
+ *
+ * @param {Y} y The Yjs instance in which to query for the absolute position.
+ * @param {RelativePosition} rpos The relative position.
+ * @return {AbsolutePosition} The absolute position in the Yjs model
+ *                            (type + offset).
+ */
+export const fromRelativePosition = (y, rpos) => {
+  if (rpos === null) {
+    return null
+  }
+  if (rpos[0] === 'endof') {
+    let id
+    if (rpos[3] === null) {
+      id = ID.createID(rpos[1], rpos[2])
+    } else {
+      id = ID.createRootID(rpos[3], rpos[4])
+    }
+    let type = y.os.get(id)
+    if (type === null) {
+      return null
+    }
+    while (type._redone !== null) {
+      type = type._redone
+    }
+    if (type === null || type.constructor === GC) {
+      return null
+    }
+    return {
+      type,
+      offset: type.length
+    }
+  } else {
+    let offset = 0
+    let struct = y.os.findNodeWithUpperBound(ID.createID(rpos[0], rpos[1])).val
+    if (struct === null || struct._id.user === ID.RootFakeUserID) {
+      return null // TODO: support fake ids?
+    }
+    const diff = rpos[1] - struct._id.clock
+    while (struct._redone !== null) {
+      struct = struct._redone
+    }
+    const parent = struct._parent
+    if (struct.constructor === GC || parent._deleted) {
+      return null
+    }
+    if (!struct._deleted && struct._countable) {
+      offset = diff
+    }
+    struct = struct._left
+    while (struct !== null) {
+      if (!struct._deleted && struct._countable) {
+        offset += struct._length
+      }
+      struct = struct._left
+    }
+    return {
+      type: parent,
+      offset: offset
+    }
+  }
+}
+
+export const equal = (posa, posb) => posa === posb || (posa !== null && posb !== null && posa.length === posb.length && posa.every((v, i) => v === posb[i]))

src/utils/snapshot.js

@@ -0,0 +1,7 @@
+
+/**
+ *
+ * @param {Item} item
+ * @param {import("../protocols/history").HistorySnapshot} [snapshot]
+ */
+export const isVisible = (item, snapshot) => snapshot === undefined ? !item._deleted : (snapshot.sm.has(item._id.user) && snapshot.sm.get(item._id.user) > item._id.clock && !snapshot.ds.isDeleted(item._id))

src/utils/structEncoding.js

@@ -0,0 +1,5 @@
+
+export const writeStructToTransaction = (transaction, struct) => {
+  transaction.encodedStructsLen++
+  struct._toBinary(transaction.encodedStructs)
+}

src/utils/structManipulation.js

@@ -0,0 +1,35 @@
+
+import * as ID from '../utils/ID.js'
+
+/**
+ * @private
+ * Delete all items in an ID-range.
+ * Does not create delete operations!
+ * TODO: implement getItemCleanStartNode for better performance (only one lookup).
+ */
+export const deleteItemRange = (y, user, clock, range, gcChildren) => {
+  let item = y.os.getItemCleanStart(ID.createID(user, clock))
+  if (item !== null) {
+    if (!item._deleted) {
+      item._splitAt(y, range)
+      item._delete(y, false, true)
+    }
+    let itemLen = item._length
+    range -= itemLen
+    clock += itemLen
+    if (range > 0) {
+      let node = y.os.findNode(ID.createID(user, clock))
+      while (node !== null && node.val !== null && range > 0 && node.val._id.equals(ID.createID(user, clock))) {
+        const nodeVal = node.val
+        if (!nodeVal._deleted) {
+          nodeVal._splitAt(y, range)
+          nodeVal._delete(y, false, gcChildren)
+        }
+        const nodeLen = nodeVal._length
+        range -= nodeLen
+        clock += nodeLen
+        node = node.next()
+      }
+    }
+  }
+}