14.0.0-1

README.md

@@ -482,8 +482,6 @@ or any of its children.
     <dd>Clone this type into a fresh Yjs type.</dd>
     <b><code>toArray():Array&lt;Y.XmlElement|Y.XmlText&gt;</code></b>
     <dd>Copies the children to a new Array.</dd>
-    <b><code>toDOM():DocumentFragment</code></b>
-    <dd>Transforms this type and all children to new DOM elements.</dd>
     <b><code>toString():string</code></b>
     <dd>Get the XML serialization of all descendants.</dd>
     <b><code>toJSON():string</code></b>
@@ -557,8 +555,6 @@ content and be actually XML compliant.
     <dd>Clone this type into a fresh Yjs type.</dd>
     <b><code>toArray():Array&lt;Y.XmlElement|Y.XmlText&gt;</code></b>
     <dd>Copies the children to a new Array.</dd>
-    <b><code>toDOM():Element</code></b>
-    <dd>Transforms this type and all children to a new DOM element.</dd>
     <b><code>toString():string</code></b>
     <dd>Get the XML serialization of all descendants.</dd>
     <b><code>toJSON():string</code></b>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yjs",
-  "version": "13.5.22",
+  "version": "14.0.0-1",
   "description": "Shared Editing Library",
   "main": "./dist/yjs.cjs",
   "module": "./dist/yjs.mjs",
@@ -19,7 +19,7 @@
     "lint": "markdownlint README.md && standard && tsc",
     "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true",
     "serve-docs": "npm run docs && http-server ./docs/",
-    "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repetition-time 1000 && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs",
+    "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs",
     "debug": "concurrently 'http-server -o test.html' 'npm run watch'",
     "trace-deopt": "clear && rollup -c  && node --trace-deopt dist/test.cjs",
     "trace-opt": "clear && rollup -c  && node --trace-opt dist/test.cjs",

src/index.js

@@ -48,7 +48,6 @@ export {
   findRootTypeKey,
   findIndexSS,
   getItem,
-  typeListToArraySnapshot,
   typeMapGetSnapshot,
   createDocFromSnapshot,
   iterateDeletedStructs,

src/internals.js

@@ -8,6 +8,7 @@ export * from './utils/encoding.js'
 export * from './utils/EventHandler.js'
 export * from './utils/ID.js'
 export * from './utils/isParentOf.js'
+export * from './utils/ListCursor.js'
 export * from './utils/logging.js'
 export * from './utils/PermanentUserData.js'
 export * from './utils/RelativePosition.js'
@@ -38,6 +39,7 @@ export * from './structs/ContentFormat.js'
 export * from './structs/ContentJSON.js'
 export * from './structs/ContentAny.js'
 export * from './structs/ContentString.js'
+export * from './structs/ContentMove.js'
 export * from './structs/ContentType.js'
 export * from './structs/Item.js'
 export * from './structs/Skip.js'

src/structs/ContentMove.js

@@ -0,0 +1,296 @@
+
+import * as error from 'lib0/error'
+import * as decoding from 'lib0/decoding'
+import * as encoding from 'lib0/encoding'
+import * as math from 'lib0/math'
+import {
+  writeID,
+  readID,
+  ID, AbstractType, ContentType, RelativePosition, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Transaction, Item, StructStore, getItem, getItemCleanStart, getItemCleanEnd, // eslint-disable-line
+  addsStruct
+} from '../internals.js'
+
+/**
+ * @param {ContentMove | { start: RelativePosition, end: RelativePosition }} moved
+ * @param {Transaction} tr
+ * @param {boolean} split
+ * @return {{ start: Item, end: Item }} $start (inclusive) is the beginning and $end (inclusive) is the end of the moved area
+ */
+export const getMovedCoords = (moved, tr, split) => {
+  const store = tr.doc.store
+  const startItem = moved.start.item
+  const endItem = moved.end.item
+  let start // this (inclusive) is the beginning of the moved area
+  let end // this (exclusive) is the first item after start that is not part of the moved area
+  if (startItem) {
+    if (moved.start.assoc < 0) {
+      // We know that the items have already been split, hence getItem suffices.
+      start = split ? getItemCleanEnd(tr, startItem) : getItem(store, startItem)
+      start = start.right
+    } else {
+      start = split ? getItemCleanStart(tr, startItem) : getItem(store, startItem)
+    }
+  } else if (moved.start.tname != null) {
+    start = tr.doc.get(moved.start.tname)._start
+  } else if (moved.start.type) {
+    start = /** @type {ContentType} */ (getItem(store, moved.start.type).content).type._start
+  } else {
+    error.unexpectedCase()
+  }
+  if (endItem) {
+    if (moved.end.assoc < 0) {
+      end = split ? getItemCleanEnd(tr, endItem) : getItem(store, endItem)
+      end = end.right
+    } else {
+      end = split ? getItemCleanStart(tr, endItem) : getItem(store, endItem)
+    }
+  } else {
+    error.unexpectedCase()
+  }
+  return { start: /** @type {Item} */ (start), end: /** @type {Item} */ (end) }
+}
+
+/**
+ * @param {Transaction} tr
+ * @param {ContentMove} moved
+ * @param {Item} movedItem
+ * @param {Set<Item>} trackedMovedItems
+ * @return {boolean} true if there is a loop
+ */
+export const findMoveLoop = (tr, moved, movedItem, trackedMovedItems) => {
+  if (trackedMovedItems.has(movedItem)) {
+    return true
+  }
+  trackedMovedItems.add(movedItem)
+  /**
+   * @type {{ start: Item | null, end: Item | null }}
+   */
+  let { start, end } = getMovedCoords(moved, tr, false)
+  while (start !== end && start != null) {
+    if (
+      !start.deleted &&
+      start.moved === movedItem &&
+      start.content.constructor === ContentMove &&
+      findMoveLoop(tr, start.content, start, trackedMovedItems)
+    ) {
+      return true
+    }
+    start = start.right
+  }
+  return false
+}
+
+/**
+ * @private
+ */
+export class ContentMove {
+  /**
+   * @param {RelativePosition} start
+   * @param {RelativePosition} end
+   * @param {number} priority if we want to move content that is already moved, we need to assign a higher priority to this move operation.
+   */
+  constructor (start, end, priority) {
+    this.start = start
+    this.end = end
+    this.priority = priority
+    /**
+     * We store which Items+ContentMove we override. Once we delete
+     * this ContentMove, we need to re-integrate the overridden items.
+     *
+     * This representation can be improved if we ever run into memory issues because of too many overrides.
+     * Ideally, we should probably just re-iterate the document and re-integrate all moved items.
+     * This is fast enough and reduces memory footprint significantly.
+     *
+     * @type {Set<Item>}
+     */
+    this.overrides = new Set()
+  }
+
+  /**
+   * @return {number}
+   */
+  getLength () {
+    return 1
+  }
+
+  /**
+   * @return {Array<any>}
+   */
+  getContent () {
+    return [null]
+  }
+
+  /**
+   * @return {boolean}
+   */
+  isCountable () {
+    return false
+  }
+
+  /**
+   * @return {ContentMove}
+   */
+  copy () {
+    return new ContentMove(this.start, this.end, this.priority)
+  }
+
+  /**
+   * @param {number} offset
+   * @return {ContentMove}
+   */
+  splice (offset) {
+    return this
+  }
+
+  /**
+   * @param {ContentMove} right
+   * @return {boolean}
+   */
+  mergeWith (right) {
+    return false
+  }
+
+  /**
+   * @param {Transaction} transaction
+   * @param {Item} item
+   */
+  integrate (transaction, item) {
+    const sm = /** @type {AbstractType<any>} */ (item.parent)._searchMarker
+    if (sm) sm.length = 0
+    const movedCoords = getMovedCoords(this, transaction, true)
+    /**
+     * @type {{ start: Item | null, end: item | null }}
+     */
+    let { start, end } = movedCoords
+    let maxPriority = 0
+    // If this ContentMove was created locally, we set prio = -1. This indicates
+    // that we want to set prio to the current prio-maximum of the moved range.
+    const adaptPriority = this.priority < 0
+    while (start !== end && start != null) {
+      const prevMove = start.moved // this is the same as prevMove
+      const nextPrio = prevMove ? /** @type {ContentMove} */ (prevMove.content).priority : -1
+      if (adaptPriority || nextPrio < this.priority || (prevMove != null && nextPrio === this.priority && (prevMove.id.client < item.id.client || (prevMove.id.client === item.id.client && prevMove.id.clock < item.id.clock)))) {
+        if (prevMove !== null) {
+          if (/** @type {ContentMove} */ (prevMove.content).isCollapsed()) {
+            prevMove.deleteAsCleanup(transaction, adaptPriority)
+          }
+          this.overrides.add(prevMove)
+          if (start !== movedCoords.start) {
+            // only add this to mergeStructs if this is not the first item
+            transaction._mergeStructs.push(start)
+          }
+        }
+        maxPriority = math.max(maxPriority, nextPrio)
+        // was already moved
+        if (prevMove && !transaction.prevMoved.has(start) && !addsStruct(transaction, prevMove)) {
+          // only override prevMoved if the prevMoved item is not new
+          // we need to know which item previously moved an item
+          transaction.prevMoved.set(start, prevMove)
+        }
+        start.moved = item
+        if (!start.deleted && start.content.constructor === ContentMove && findMoveLoop(transaction, start.content, start, new Set([item]))) {
+          item.deleteAsCleanup(transaction, adaptPriority)
+          return
+        }
+      } else if (prevMove != null) {
+        /** @type {ContentMove} */ (prevMove.content).overrides.add(item)
+      }
+      start = start.right
+    }
+    if (adaptPriority) {
+      this.priority = maxPriority + 1
+    }
+  }
+
+  /**
+   * @param {Transaction} transaction
+   * @param {Item} item
+   */
+  delete (transaction, item) {
+    /**
+     * @type {{ start: Item | null, end: Item | null }}
+     */
+    let { start, end } = getMovedCoords(this, transaction, false)
+    while (start !== end && start != null) {
+      if (start.moved === item) {
+        const prevMoved = transaction.prevMoved.get(start)
+        if (addsStruct(transaction, item)) {
+          if (prevMoved === item) {
+            // Edge case: Item has been moved by this move op and it has been created & deleted in the same transaction (hence no effect that should be emitted by the change computation)
+            transaction.prevMoved.delete(start)
+          }
+        } else if (prevMoved == null) { // && !addsStruct(tr, item)
+          // Normal case: item has been moved by this move and it has not been created & deleted in the same transaction
+          transaction.prevMoved.set(start, item)
+        }
+        start.moved = null
+      }
+      start = start.right
+    }
+    /**
+     * @param {Item} reIntegrateItem
+     */
+    const reIntegrate = reIntegrateItem => {
+      const content = /** @type {ContentMove} */ (reIntegrateItem.content)
+      // content is not yet transformed to a ContentDeleted
+      if (content.getRef() === 11) {
+        if (reIntegrateItem.deleted) {
+          // potentially we can integrate the items that reIntegrateItem overrides
+          content.overrides.forEach(reIntegrate)
+        } else {
+          content.integrate(transaction, reIntegrateItem)
+        }
+      }
+    }
+    this.overrides.forEach(reIntegrate)
+  }
+
+  /**
+   * @param {StructStore} store
+   */
+  gc (store) {}
+
+  /**
+   * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
+   * @param {number} offset
+   */
+  write (encoder, offset) {
+    const isCollapsed = this.isCollapsed()
+    encoding.writeVarUint(encoder.restEncoder, (isCollapsed ? 1 : 0) | (this.start.assoc >= 0 ? 2 : 0) | (this.end.assoc >= 0 ? 4 : 0) | this.priority << 6)
+    writeID(encoder.restEncoder, /** @type {ID} */ (this.start.item))
+    if (!isCollapsed) {
+      writeID(encoder.restEncoder, /** @type {ID} */ (this.end.item))
+    }
+  }
+
+  /**
+   * @return {number}
+   */
+  getRef () {
+    return 11
+  }
+
+  isCollapsed () {
+    return this.start.item === this.end.item && this.start.item !== null
+  }
+}
+
+/**
+ * @private
+ *
+ * @param {UpdateDecoderV1 | UpdateDecoderV2} decoder
+ * @return {ContentMove}
+ */
+export const readContentMove = decoder => {
+  const info = decoding.readVarUint(decoder.restDecoder)
+  const isCollapsed = (info & 1) === 1
+  const startAssoc = (info & 2) === 2 ? 0 : -1
+  const endAssoc = (info & 4) === 4 ? 0 : -1
+  // @TODO use BIT3 & BIT4 to indicate the case `null` is the start/end
+  // BIT5 is reserved for future extensions
+  const priority = info >>> 6
+  const startId = readID(decoder.restDecoder)
+  const start = new RelativePosition(null, null, startId, startAssoc)
+  const end = new RelativePosition(null, null, isCollapsed ? startId : readID(decoder.restDecoder), endAssoc)
+  return new ContentMove(start, end, priority)
+}

src/structs/Item.js

@@ -21,12 +21,14 @@ import {
   createID,
   readContentFormat,
   readContentType,
+  readContentMove,
   addChangedTypeToTransaction,
   UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as error from 'lib0/error'
 import * as binary from 'lib0/binary'
+import { ContentMove } from './ContentMove.js'
 
 /**
  * @todo This should return several items
@@ -116,6 +118,13 @@ export const splitItem = (transaction, leftItem, diff) => {
     /** @type {AbstractType<any>} */ (rightItem.parent)._map.set(rightItem.parentSub, rightItem)
   }
   leftItem.length = diff
+  if (leftItem.moved) {
+    rightItem.moved = leftItem.moved
+    const m = transaction.prevMoved.get(leftItem)
+    if (m) {
+      transaction.prevMoved.set(rightItem, m)
+    }
+  }
   return rightItem
 }
 
@@ -281,11 +290,18 @@ export class Item extends AbstractStruct {
      */
     this.parentSub = parentSub
     /**
-     * If this type's effect is reundone this type refers to the type that undid
+     * If this type's effect is reundone this type refers to the type-id that undid
      * this operation.
+     *
      * @type {ID | null}
      */
     this.redone = null
+    /**
+     * This property is reused by the moved prop. In this case this property refers to an Item.
+     *
+     * @type {Item | null}
+     */
+    this.moved = null
     /**
      * @type {AbstractContent}
      */
@@ -367,11 +383,21 @@ export class Item extends AbstractStruct {
     if (this.parent && this.parent.constructor === ID && this.id.client !== this.parent.client && this.parent.clock >= getState(store, this.parent.client)) {
       return this.parent.client
     }
+    if (this.content.constructor === ContentMove) {
+      const c = /** @type {ContentMove} */ (this.content)
+      const start = c.start.item
+      const end = c.isCollapsed() ? null : c.end.item
+      if (start && start.clock >= getState(store, start.client)) {
+        return start.client
+      }
+      if (end && end.clock >= getState(store, end.client)) {
+        return end.client
+      }
+    }
 
     // We have all missing ids, now find the items
-
     if (this.origin) {
-      this.left = getItemCleanEnd(transaction, store, this.origin)
+      this.left = getItemCleanEnd(transaction, this.origin)
       this.origin = this.left.lastId
     }
     if (this.rightOrigin) {
@@ -399,6 +425,7 @@ export class Item extends AbstractStruct {
         this.parent = /** @type {ContentType} */ (parentItem.content).type
       }
     }
+
     return null
   }
 
@@ -409,7 +436,7 @@ export class Item extends AbstractStruct {
   integrate (transaction, offset) {
     if (offset > 0) {
       this.id.clock += offset
-      this.left = getItemCleanEnd(transaction, transaction.doc.store, createID(this.id.client, this.id.clock - 1))
+      this.left = getItemCleanEnd(transaction, createID(this.id.client, this.id.clock - 1))
       this.origin = this.left.lastId
       this.content = this.content.splice(offset)
       this.length -= offset
@@ -508,6 +535,24 @@ export class Item extends AbstractStruct {
       if (this.parentSub === null && this.countable && !this.deleted) {
         /** @type {AbstractType<any>} */ (this.parent)._length += this.length
       }
+      // check if this item is in a moved range
+      if ((this.left && this.left.moved) || (this.right && this.right.moved)) {
+        const leftMoved = this.left && this.left.moved && /** @type {ContentMove} */ (this.left.moved.content)
+        const rightMoved = this.right && this.right.moved && /** @type {ContentMove} */ (this.right.moved.content)
+        if (leftMoved === rightMoved) {
+          this.moved = /** @type {Item} */ (this.left).moved
+        } else if (
+          (leftMoved != null && !leftMoved.isCollapsed()) ||
+          (rightMoved != null && !rightMoved.isCollapsed())
+        ) {
+          // We know that this item is on the edge of a moved range.
+          // @todo Instead, we could check to which moved-range this item belongs
+          // This approach (reintegration) is pretty expensive in some scenarios
+          leftMoved && leftMoved.integrate(transaction, /** @type {any} */ (this.left).moved)
+          rightMoved && rightMoved.integrate(transaction, /** @type {any} */ (this.right).moved)
+        }
+      }
+
       addStruct(transaction.doc.store, this)
       this.content.integrate(transaction, this)
       // add parent to transaction.changed
@@ -569,21 +614,22 @@ export class Item extends AbstractStruct {
       this.deleted === right.deleted &&
       this.redone === null &&
       right.redone === null &&
+      this.moved === right.moved &&
       this.content.constructor === right.content.constructor &&
       this.content.mergeWith(right.content)
     ) {
-      const searchMarker = /** @type {AbstractType<any>} */ (this.parent)._searchMarker
+      if (right.marker) {
-      if (searchMarker) {
+        // Right will be "forgotten", so we delete all
-        searchMarker.forEach(marker => {
+        // search markers that reference right.
-          if (marker.p === right) {
+        const searchMarker = /** @type {AbstractType<any>} */ (this.parent)._searchMarker
-            // right is going to be "forgotten" so we need to update the marker
+        if (searchMarker) {
-            marker.p = this
+          for (let i = searchMarker.length - 1; i >= 0; i--) {
-            // adjust marker index
+            if (searchMarker[i].nextItem === right) {
-            if (!this.deleted && this.countable) {
+              // @todo do something more efficient than splicing..
-              marker.index -= this.length
+              searchMarker.splice(i, 1)
             }
           }
-        })
+        }
       }
       if (right.keep) {
         this.keep = true
@@ -613,7 +659,23 @@ export class Item extends AbstractStruct {
       this.markDeleted()
       addToDeleteSet(transaction.deleteSet, this.id.client, this.id.clock, this.length)
       addChangedTypeToTransaction(transaction, parent, this.parentSub)
-      this.content.delete(transaction)
+      this.content.delete(transaction, this)
+    }
+  }
+
+  /**
+   * Similar to `this.delete(tr)`, but additionally ensures
+   * that the deleted range is broadcasted using a different
+   * origin/source in a separate update event, so that
+   * the providers don't filter this message.
+   *
+   * @param {Transaction} transaction
+   * @param {boolean} isLocal
+   */
+  deleteAsCleanup (transaction, isLocal) {
+    this.delete(transaction)
+    if (!isLocal) {
+      addToDeleteSet(transaction.cleanupDeletions, this.id.client, this.id.clock, this.length)
     }
   }
 
@@ -700,7 +762,7 @@ export const readItemContent = (decoder, info) => contentRefs[info & binary.BITS
  * @type {Array<function(UpdateDecoderV1 | UpdateDecoderV2):AbstractContent>}
  */
 export const contentRefs = [
-  () => { error.unexpectedCase() }, // GC is not ItemContent
+  error.unexpectedCase, // GC is not ItemContent
   readContentDeleted, // 1
   readContentJSON, // 2
   readContentBinary, // 3
@@ -710,7 +772,8 @@ export const contentRefs = [
   readContentType, // 7
   readContentAny, // 8
   readContentDoc, // 9
-  () => { error.unexpectedCase() } // 10 - Skip is not ItemContent
+  error.unexpectedCase, // 10 - Skip is not ItemContent
+  readContentMove // 11
 ]
 
 /**
@@ -777,8 +840,9 @@ export class AbstractContent {
 
   /**
    * @param {Transaction} transaction
+   * @param {Item} item
    */
-  delete (transaction) {
+  delete (transaction, item) {
     throw error.methodUnimplemented()
   }
 

src/types/AbstractType.js

@@ -10,8 +10,8 @@ import {
   createID,
   ContentAny,
   ContentBinary,
-  getItemCleanStart,
+  ListCursor,
-  ContentDoc, YText, YArray, UpdateEncoderV1, UpdateEncoderV2, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
+  ContentDoc, UpdateEncoderV1, UpdateEncoderV2, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
 } from '../internals.js'
 
 import * as map from 'lib0/map'
@@ -19,68 +19,8 @@ import * as iterator from 'lib0/iterator'
 import * as error from 'lib0/error'
 import * as math from 'lib0/math'
 
-const maxSearchMarker = 80
+const maxSearchMarker = 300
-
+const freshSearchMarkerDistance = 30
-/**
- * A unique timestamp that identifies each marker.
- *
- * Time is relative,.. this is more like an ever-increasing clock.
- *
- * @type {number}
- */
-let globalSearchMarkerTimestamp = 0
-
-export class ArraySearchMarker {
-  /**
-   * @param {Item} p
-   * @param {number} index
-   */
-  constructor (p, index) {
-    p.marker = true
-    this.p = p
-    this.index = index
-    this.timestamp = globalSearchMarkerTimestamp++
-  }
-}
-
-/**
- * @param {ArraySearchMarker} marker
- */
-const refreshMarkerTimestamp = marker => { marker.timestamp = globalSearchMarkerTimestamp++ }
-
-/**
- * This is rather complex so this function is the only thing that should overwrite a marker
- *
- * @param {ArraySearchMarker} marker
- * @param {Item} p
- * @param {number} index
- */
-const overwriteMarker = (marker, p, index) => {
-  marker.p.marker = false
-  marker.p = p
-  p.marker = true
-  marker.index = index
-  marker.timestamp = globalSearchMarkerTimestamp++
-}
-
-/**
- * @param {Array<ArraySearchMarker>} searchMarker
- * @param {Item} p
- * @param {number} index
- */
-const markPosition = (searchMarker, p, index) => {
-  if (searchMarker.length >= maxSearchMarker) {
-    // override oldest marker (we don't want to create more objects)
-    const marker = searchMarker.reduce((a, b) => a.timestamp < b.timestamp ? a : b)
-    overwriteMarker(marker, p, index)
-    return marker
-  } else {
-    // create new marker
-    const pm = new ArraySearchMarker(p, index)
-    searchMarker.push(pm)
-    return pm
-  }
-}
 
 /**
  * Search marker help us to find positions in the associative array faster.
@@ -89,82 +29,64 @@ const markPosition = (searchMarker, p, index) => {
  *
  * A maximum of `maxSearchMarker` objects are created.
  *
- * This function always returns a refreshed marker (updated timestamp)
+ * @template T
- *
+ * @param {Transaction} tr
  * @param {AbstractType<any>} yarray
  * @param {number} index
+ * @param {function(ListCursor):T} f
+ * @return T
  */
-export const findMarker = (yarray, index) => {
+export const useSearchMarker = (tr, yarray, index, f) => {
-  if (yarray._start === null || index === 0 || yarray._searchMarker === null) {
+  const searchMarker = yarray._searchMarker
-    return null
+  if (searchMarker === null || yarray._start === null || index < freshSearchMarkerDistance) {
+    return f(new ListCursor(yarray).forward(tr, index, true))
   }
-  const marker = yarray._searchMarker.length === 0 ? null : yarray._searchMarker.reduce((a, b) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b)
+  if (searchMarker.length === 0) {
-  let p = yarray._start
+    const sm = new ListCursor(yarray).forward(tr, index, true)
-  let pindex = 0
+    searchMarker.push(sm)
-  if (marker !== null) {
+    if (sm.nextItem) sm.nextItem.marker = true
-    p = marker.p
-    pindex = marker.index
-    refreshMarkerTimestamp(marker) // we used it, we might need to use it again
   }
-  // iterate to right if possible
+  const sm = searchMarker.reduce(
-  while (p.right !== null && pindex < index) {
+    (a, b, arrayIndex) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b
-    if (!p.deleted && p.countable) {
+  )
-      if (index < pindex + p.length) {
+  const newIsCheaper = math.abs(sm.index - index) >= index
-        break
+  const createFreshMarker = searchMarker.length < maxSearchMarker && (math.abs(sm.index - index) > freshSearchMarkerDistance || newIsCheaper)
-      }
+  const fsm = createFreshMarker ? (newIsCheaper ? new ListCursor(yarray) : sm.clone()) : sm
-      pindex += p.length
+  const prevItem = /** @type {Item} */ (sm.nextItem)
-    }
+  if (createFreshMarker) {
-    p = p.right
+    searchMarker.push(fsm)
   }
-  // iterate to left if necessary (might be that pindex > index)
+  const diff = fsm.index - index
-  while (p.left !== null && pindex > index) {
+  if (diff > 0) {
-    p = p.left
+    fsm.backward(tr, diff)
-    if (!p.deleted && p.countable) {
-      pindex -= p.length
-    }
-  }
-  // we want to make sure that p can't be merged with left, because that would screw up everything
-  // in that cas just return what we have (it is most likely the best marker anyway)
-  // iterate to left until p can't be merged with left
-  while (p.left !== null && p.left.id.client === p.id.client && p.left.id.clock + p.left.length === p.id.clock) {
-    p = p.left
-    if (!p.deleted && p.countable) {
-      pindex -= p.length
-    }
-  }
-
-  // @todo remove!
-  // assure position
-  // {
-  //   let start = yarray._start
-  //   let pos = 0
-  //   while (start !== p) {
-  //     if (!start.deleted && start.countable) {
-  //       pos += start.length
-  //     }
-  //     start = /** @type {Item} */ (start.right)
-  //   }
-  //   if (pos !== pindex) {
-  //     debugger
-  //     throw new Error('Gotcha position fail!')
-  //   }
-  // }
-  // if (marker) {
-  //   if (window.lengthes == null) {
-  //     window.lengthes = []
-  //     window.getLengthes = () => window.lengthes.sort((a, b) => a - b)
-  //   }
-  //   window.lengthes.push(marker.index - pindex)
-  //   console.log('distance', marker.index - pindex, 'len', p && p.parent.length)
-  // }
-  if (marker !== null && math.abs(marker.index - pindex) < /** @type {YText|YArray<any>} */ (p.parent).length / maxSearchMarker) {
-    // adjust existing marker
-    overwriteMarker(marker, p, pindex)
-    return marker
   } else {
-    // create new marker
+    fsm.forward(tr, -diff, true)
-    return markPosition(yarray._searchMarker, p, pindex)
   }
+  const result = f(fsm)
+  if (fsm.reachedEnd) {
+    fsm.reachedEnd = false
+    const nextItem = /** @type {Item} */ (fsm.nextItem)
+    if (nextItem.countable && !nextItem.deleted) {
+      fsm.index -= nextItem.length
+    }
+    fsm.rel = 0
+  }
+  fsm.index -= fsm.rel
+  fsm.rel = 0
+  if (!createFreshMarker) {
+    // reused old marker and we moved to a different position
+    prevItem.marker = false
+  }
+  const fsmItem = fsm.nextItem
+  if (fsmItem) {
+    if (fsmItem.marker) {
+      // already marked, forget current iterator
+      searchMarker.splice(searchMarker.findIndex(m => m === fsm), 1)
+    } else {
+      fsmItem.marker = true
+    }
+  }
+  return result
 }
 
 /**
@@ -172,39 +94,25 @@ export const findMarker = (yarray, index) => {
  *
  * This should be called before doing a deletion!
  *
- * @param {Array<ArraySearchMarker>} searchMarker
+ * @param {Array<ListCursor>} searchMarker
  * @param {number} index
  * @param {number} len If insertion, len is positive. If deletion, len is negative.
+ * @param {ListCursor|null} origSearchMarker Do not update this searchmarker because it is the one we used to manipulate. @todo !=null for improved perf in ytext
  */
-export const updateMarkerChanges = (searchMarker, index, len) => {
+export const updateMarkerChanges = (searchMarker, index, len, origSearchMarker) => {
   for (let i = searchMarker.length - 1; i >= 0; i--) {
-    const m = searchMarker[i]
+    const marker = searchMarker[i]
-    if (len > 0) {
+    if (marker !== origSearchMarker) {
-      /**
+      if (len > 0 && index === marker.index) {
-       * @type {Item|null}
+        // inserting at a marked position deletes the marked position because we can't do a simple transformation
-       */
+        // (we don't know whether to insert directly before or directly after the position)
-      let p = m.p
-      p.marker = false
-      // Ideally we just want to do a simple position comparison, but this will only work if
-      // search markers don't point to deleted items for formats.
-      // Iterate marker to prev undeleted countable position so we know what to do when updating a position
-      while (p && (p.deleted || !p.countable)) {
-        p = p.left
-        if (p && !p.deleted && p.countable) {
-          // adjust position. the loop should break now
-          m.index -= p.length
-        }
-      }
-      if (p === null || p.marker === true) {
-        // remove search marker if updated position is null or if position is already marked
         searchMarker.splice(i, 1)
+        if (marker.nextItem) marker.nextItem.marker = false
         continue
       }
-      m.p = p
+      if (index < marker.index) { // a simple index <= m.index check would actually suffice
-      p.marker = true
+        marker.index = math.max(index, marker.index + len)
-    }
+      }
-    if (index < m.index || (len > 0 && index === m.index)) { // a simple index <= m.index check would actually suffice
-      m.index = math.max(index, m.index + len)
     }
   }
 }
@@ -282,9 +190,16 @@ export class AbstractType {
      */
     this._dEH = createEventHandler()
     /**
-     * @type {null | Array<ArraySearchMarker>}
+     * @type {null | Array<ListCursor>}
      */
     this._searchMarker = null
+    /**
+     * You can store custom stuff here.
+     * This might be useful to associate your application state to this shared type.
+     *
+     * @type {Map<any, any>}
+     */
+    this.meta = new Map()
   }
 
   /**
@@ -454,171 +369,6 @@ export const typeListToArray = type => {
   return cs
 }
 
-/**
- * @param {AbstractType<any>} type
- * @param {Snapshot} snapshot
- * @return {Array<any>}
- *
- * @private
- * @function
- */
-export const typeListToArraySnapshot = (type, snapshot) => {
-  const cs = []
-  let n = type._start
-  while (n !== null) {
-    if (n.countable && isVisible(n, snapshot)) {
-      const c = n.content.getContent()
-      for (let i = 0; i < c.length; i++) {
-        cs.push(c[i])
-      }
-    }
-    n = n.right
-  }
-  return cs
-}
-
-/**
- * Executes a provided function on once on overy element of this YArray.
- *
- * @param {AbstractType<any>} type
- * @param {function(any,number,any):void} f A function to execute on every element of this YArray.
- *
- * @private
- * @function
- */
-export const typeListForEach = (type, f) => {
-  let index = 0
-  let n = type._start
-  while (n !== null) {
-    if (n.countable && !n.deleted) {
-      const c = n.content.getContent()
-      for (let i = 0; i < c.length; i++) {
-        f(c[i], index++, type)
-      }
-    }
-    n = n.right
-  }
-}
-
-/**
- * @template C,R
- * @param {AbstractType<any>} type
- * @param {function(C,number,AbstractType<any>):R} f
- * @return {Array<R>}
- *
- * @private
- * @function
- */
-export const typeListMap = (type, f) => {
-  /**
-   * @type {Array<any>}
-   */
-  const result = []
-  typeListForEach(type, (c, i) => {
-    result.push(f(c, i, type))
-  })
-  return result
-}
-
-/**
- * @param {AbstractType<any>} type
- * @return {IterableIterator<any>}
- *
- * @private
- * @function
- */
-export const typeListCreateIterator = type => {
-  let n = type._start
-  /**
-   * @type {Array<any>|null}
-   */
-  let currentContent = null
-  let currentContentIndex = 0
-  return {
-    [Symbol.iterator] () {
-      return this
-    },
-    next: () => {
-      // find some content
-      if (currentContent === null) {
-        while (n !== null && n.deleted) {
-          n = n.right
-        }
-        // check if we reached the end, no need to check currentContent, because it does not exist
-        if (n === null) {
-          return {
-            done: true,
-            value: undefined
-          }
-        }
-        // we found n, so we can set currentContent
-        currentContent = n.content.getContent()
-        currentContentIndex = 0
-        n = n.right // we used the content of n, now iterate to next
-      }
-      const value = currentContent[currentContentIndex++]
-      // check if we need to empty currentContent
-      if (currentContent.length <= currentContentIndex) {
-        currentContent = null
-      }
-      return {
-        done: false,
-        value
-      }
-    }
-  }
-}
-
-/**
- * Executes a provided function on once on overy element of this YArray.
- * Operates on a snapshotted state of the document.
- *
- * @param {AbstractType<any>} type
- * @param {function(any,number,AbstractType<any>):void} f A function to execute on every element of this YArray.
- * @param {Snapshot} snapshot
- *
- * @private
- * @function
- */
-export const typeListForEachSnapshot = (type, f, snapshot) => {
-  let index = 0
-  let n = type._start
-  while (n !== null) {
-    if (n.countable && isVisible(n, snapshot)) {
-      const c = n.content.getContent()
-      for (let i = 0; i < c.length; i++) {
-        f(c[i], index++, type)
-      }
-    }
-    n = n.right
-  }
-}
-
-/**
- * @param {AbstractType<any>} type
- * @param {number} index
- * @return {any}
- *
- * @private
- * @function
- */
-export const typeListGet = (type, index) => {
-  const marker = findMarker(type, index)
-  let n = type._start
-  if (marker !== null) {
-    n = marker.p
-    index -= marker.index
-  }
-  for (; n !== null; n = n.right) {
-    if (!n.deleted && n.countable) {
-      if (index < n.length) {
-        return n.content.getContent()[index]
-      }
-      index -= n.length
-    }
-  }
-}
-
 /**
  * @param {Transaction} transaction
  * @param {AbstractType<any>} parent
@@ -683,105 +433,6 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem,
   packJsonContent()
 }
 
-const lengthExceeded = error.create('Length exceeded!')
-
-/**
- * @param {Transaction} transaction
- * @param {AbstractType<any>} parent
- * @param {number} index
- * @param {Array<Object<string,any>|Array<any>|number|null|string|Uint8Array>} content
- *
- * @private
- * @function
- */
-export const typeListInsertGenerics = (transaction, parent, index, content) => {
-  if (index > parent._length) {
-    throw lengthExceeded
-  }
-  if (index === 0) {
-    if (parent._searchMarker) {
-      updateMarkerChanges(parent._searchMarker, index, content.length)
-    }
-    return typeListInsertGenericsAfter(transaction, parent, null, content)
-  }
-  const startIndex = index
-  const marker = findMarker(parent, index)
-  let n = parent._start
-  if (marker !== null) {
-    n = marker.p
-    index -= marker.index
-    // we need to iterate one to the left so that the algorithm works
-    if (index === 0) {
-      // @todo refactor this as it actually doesn't consider formats
-      n = n.prev // important! get the left undeleted item so that we can actually decrease index
-      index += (n && n.countable && !n.deleted) ? n.length : 0
-    }
-  }
-  for (; n !== null; n = n.right) {
-    if (!n.deleted && n.countable) {
-      if (index <= n.length) {
-        if (index < n.length) {
-          // insert in-between
-          getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index))
-        }
-        break
-      }
-      index -= n.length
-    }
-  }
-  if (parent._searchMarker) {
-    updateMarkerChanges(parent._searchMarker, startIndex, content.length)
-  }
-  return typeListInsertGenericsAfter(transaction, parent, n, content)
-}
-
-/**
- * @param {Transaction} transaction
- * @param {AbstractType<any>} parent
- * @param {number} index
- * @param {number} length
- *
- * @private
- * @function
- */
-export const typeListDelete = (transaction, parent, index, length) => {
-  if (length === 0) { return }
-  const startIndex = index
-  const startLength = length
-  const marker = findMarker(parent, index)
-  let n = parent._start
-  if (marker !== null) {
-    n = marker.p
-    index -= marker.index
-  }
-  // compute the first item to be deleted
-  for (; n !== null && index > 0; n = n.right) {
-    if (!n.deleted && n.countable) {
-      if (index < n.length) {
-        getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index))
-      }
-      index -= n.length
-    }
-  }
-  // delete all items until done
-  while (length > 0 && n !== null) {
-    if (!n.deleted) {
-      if (length < n.length) {
-        getItemCleanStart(transaction, createID(n.id.client, n.id.clock + length))
-      }
-      n.delete(transaction)
-      length -= n.length
-    }
-    n = n.right
-  }
-  if (length > 0) {
-    throw lengthExceeded
-  }
-  if (parent._searchMarker) {
-    updateMarkerChanges(parent._searchMarker, startIndex, -startLength + length /* in case we remove the above exception */)
-  }
-}
-
 /**
  * @param {Transaction} transaction
  * @param {AbstractType<any>} parent

src/types/YArray.js

@@ -5,19 +5,15 @@
 import {
   YEvent,
   AbstractType,
-  typeListGet,
-  typeListToArray,
-  typeListForEach,
-  typeListCreateIterator,
-  typeListInsertGenerics,
-  typeListDelete,
-  typeListMap,
   YArrayRefID,
   callTypeObservers,
   transact,
-  ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, Transaction, Item // eslint-disable-line
+  ListCursor,
+  useSearchMarker,
+  createRelativePositionFromTypeIndex,
+  UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, Transaction, Item, // eslint-disable-line
+  getMinimalListViewRanges
 } from '../internals.js'
-import { typeListSlice } from './AbstractType.js'
 
 /**
  * Event that describes the changes on a YArray
@@ -49,7 +45,7 @@ export class YArray extends AbstractType {
      */
     this._prelimContent = []
     /**
-     * @type {Array<ArraySearchMarker>}
+     * @type {Array<ListCursor>}
      */
     this._searchMarker = []
   }
@@ -129,12 +125,83 @@ export class YArray extends AbstractType {
    * @param {Array<T>} content The array of content
    */
   insert (index, content) {
+    if (content.length > 0) {
+      if (this.doc !== null) {
+        transact(this.doc, transaction => {
+          useSearchMarker(transaction, this, index, walker =>
+            walker.insertArrayValue(transaction, content)
+          )
+        })
+      } else {
+        /** @type {Array<any>} */ (this._prelimContent).splice(index, 0, ...content)
+      }
+    }
+  }
+
+  /**
+   * Move a single item from $index to $target.
+   *
+   * If the original item is to the left of $target, then the index of the item will decrement.
+   *
+   * ```js
+   * yarray.insert(0, [1, 2, 3])
+   * yarray.move(0, 3) // move "1" to index 3
+   * yarray.toArray() // => [2, 3, 1]
+   * yarray.move(2, 0) // move "1" to index 0
+   * yarray.toArray() // => [1, 2, 3]
+   * ```
+   *
+   * @param {number} index
+   * @param {number} target
+   */
+  move (index, target) {
+    if (index === target || index + 1 === target || index >= this.length) {
+      // It doesn't make sense to move a range into the same range (it's basically a no-op).
+      return
+    }
     if (this.doc !== null) {
       transact(this.doc, transaction => {
-        typeListInsertGenerics(transaction, this, index, content)
+        const start = createRelativePositionFromTypeIndex(this, index, 1)
+        const end = start.clone()
+        end.assoc = -1
+        useSearchMarker(transaction, this, target, walker => {
+          walker.insertMove(transaction, [{ start, end }])
+        })
       })
     } else {
-      /** @type {Array<any>} */ (this._prelimContent).splice(index, 0, ...content)
+      const content = /** @type {Array<any>} */ (this._prelimContent).splice(index, 1)
+      ;/** @type {Array<any>} */ (this._prelimContent).splice(target, 0, ...content)
+    }
+  }
+
+  /**
+   * @experimental
+   *
+   * @param {number} startIndex Inclusive move-start
+   * @param {number} endIndex Inclusive move-end
+   * @param {number} target
+   * @param {number} assocStart >=0 if start should be associated with the right character. See relative-position assoc parameter.
+   * @param {number} assocEnd >= 0 if end should be associated with the right character.
+   */
+  moveRange (startIndex, endIndex, target, assocStart = 1, assocEnd = -1) {
+    if (
+      (startIndex <= target && target <= endIndex) || // It doesn't make sense to move a range into the same range (it's basically a no-op).
+      endIndex - startIndex < 0 // require length of >= 0
+    ) {
+      return
+    }
+    if (this.doc !== null) {
+      transact(this.doc, transaction => {
+        const ranges = useSearchMarker(transaction, this, startIndex, walker =>
+          getMinimalListViewRanges(transaction, walker, endIndex - startIndex + 1)
+        )
+        useSearchMarker(transaction, this, target, walker => {
+          walker.insertMove(transaction, ranges)
+        })
+      })
+    } else {
+      const content = /** @type {Array<any>} */ (this._prelimContent).splice(startIndex, endIndex - startIndex + 1)
+      ;/** @type {Array<any>} */ (this._prelimContent).splice(target, 0, ...content)
     }
   }
 
@@ -165,7 +232,9 @@ export class YArray extends AbstractType {
   delete (index, length = 1) {
     if (this.doc !== null) {
       transact(this.doc, transaction => {
-        typeListDelete(transaction, this, index, length)
+        useSearchMarker(transaction, this, index, walker =>
+          walker.delete(transaction, length)
+        )
       })
     } else {
       /** @type {Array<any>} */ (this._prelimContent).splice(index, length)
@@ -179,7 +248,11 @@ export class YArray extends AbstractType {
    * @return {T}
    */
   get (index) {
-    return typeListGet(this, index)
+    return transact(/** @type {Doc} */ (this.doc), transaction =>
+      useSearchMarker(transaction, this, index, walker =>
+        walker.slice(transaction, 1)[0]
+      )
+    )
   }
 
   /**
@@ -188,7 +261,9 @@ export class YArray extends AbstractType {
    * @return {Array<T>}
    */
   toArray () {
-    return typeListToArray(this)
+    return transact(/** @type {Doc} */ (this.doc), tr =>
+      new ListCursor(this).slice(tr, this.length)
+    )
   }
 
   /**
@@ -199,7 +274,11 @@ export class YArray extends AbstractType {
    * @return {Array<T>}
    */
   slice (start = 0, end = this.length) {
-    return typeListSlice(this, start, end)
+    return transact(/** @type {Doc} */ (this.doc), transaction =>
+      useSearchMarker(transaction, this, start, walker =>
+        walker.slice(transaction, end < 0 ? this.length + end - start : end - start)
+      )
+    )
   }
 
   /**
@@ -221,7 +300,9 @@ export class YArray extends AbstractType {
    *                 callback function
    */
   map (f) {
-    return typeListMap(this, /** @type {any} */ (f))
+    return transact(/** @type {Doc} */ (this.doc), tr =>
+      new ListCursor(this).map(tr, f)
+    )
   }
 
   /**
@@ -230,14 +311,17 @@ export class YArray extends AbstractType {
    * @param {function(T,number,YArray<T>):void} f A function to execute on every element of this YArray.
    */
   forEach (f) {
-    typeListForEach(this, f)
+    return transact(/** @type {Doc} */ (this.doc), tr =>
+      new ListCursor(this).forEach(tr, f)
+    )
   }
 
   /**
    * @return {IterableIterator<T>}
    */
   [Symbol.iterator] () {
-    return typeListCreateIterator(this)
+    // @todo, this could be optimized using a real iterator
+    return this.toArray().values()
   }
 
   /**

src/types/YText.js

@@ -20,14 +20,15 @@ import {
   splitSnapshotAffectedStructs,
   iterateDeletedStructs,
   iterateStructs,
-  findMarker,
   typeMapDelete,
   typeMapSet,
   typeMapGet,
   typeMapGetAll,
   updateMarkerChanges,
   ContentType,
-  ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
+  useSearchMarker,
+  findIndexCleanStart,
+  ListCursor, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as object from 'lib0/object'
@@ -125,10 +126,30 @@ const findNextPosition = (transaction, pos, count) => {
  */
 const findPosition = (transaction, parent, index) => {
   const currentAttributes = new Map()
-  const marker = findMarker(parent, index)
+  if (parent._searchMarker) {
-  if (marker) {
+    return useSearchMarker(transaction, parent, index, listIter => {
-    const pos = new ItemTextListPosition(marker.p.left, marker.p, marker.index, currentAttributes)
+      let left, right
-    return findNextPosition(transaction, pos, index - marker.index)
+      if (listIter.rel > 0) {
+        // must exist because rel > 0
+        const nextItem = /** @type {Item} */ (listIter.nextItem)
+        if (listIter.rel === nextItem.length) {
+          left = nextItem
+          right = left.right
+        } else {
+          const structs = /** @type {Array<Item|GC>} */ (transaction.doc.store.clients.get(nextItem.id.client))
+          const after = /** @type {Item} */ (structs[findIndexCleanStart(transaction, structs, nextItem.id.clock + listIter.rel)])
+          listIter.nextItem = after
+          listIter.rel = 0
+          left = listIter.left
+          right = listIter.right
+        }
+      } else {
+        left = listIter.left
+        right = listIter.right
+      }
+      // @todo this should simply split if .rel > 0
+      return new ItemTextListPosition(left, right, index, currentAttributes)
+    })
   } else {
     const pos = new ItemTextListPosition(null, parent._start, 0, currentAttributes)
     return findNextPosition(transaction, pos, index)
@@ -264,7 +285,7 @@ const insertText = (transaction, parent, currPos, text, attributes) => {
   const content = text.constructor === String ? new ContentString(/** @type {string} */ (text)) : (text instanceof AbstractType ? new ContentType(text) : new ContentEmbed(text))
   let { left, right, index } = currPos
   if (parent._searchMarker) {
-    updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength())
+    updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength(), null)
   }
   right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content)
   right.integrate(transaction, 0)
@@ -469,7 +490,7 @@ const deleteText = (transaction, currPos, length) => {
   }
   const parent = /** @type {AbstractType<any>} */ (/** @type {Item} */ (currPos.left || currPos.right).parent)
   if (parent._searchMarker) {
-    updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length)
+    updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length, null)
   }
   return currPos
 }
@@ -764,7 +785,7 @@ export class YText extends AbstractType {
      */
     this._pending = string !== undefined ? [() => this.insert(0, string)] : []
     /**
-     * @type {Array<ArraySearchMarker>}
+     * @type {Array<ListCursor>}
      */
     this._searchMarker = []
   }

src/types/YXmlElement.js

@@ -7,7 +7,6 @@ import {
   typeMapSet,
   typeMapGet,
   typeMapGetAll,
-  typeListForEach,
   YXmlElementRefID,
   YXmlText, ContentType, AbstractType, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Snapshot, Doc, Item // eslint-disable-line
 } from '../internals.js'
@@ -185,36 +184,6 @@ export class YXmlElement extends YXmlFragment {
     return typeMapGetAll(this)
   }
 
-  /**
-   * Creates a Dom Element that mirrors this YXmlElement.
-   *
-   * @param {Document} [_document=document] The document object (you must define
-   *                                        this when calling this method in
-   *                                        nodejs)
-   * @param {Object<string, any>} [hooks={}] Optional property to customize how hooks
-   *                                             are presented in the DOM
-   * @param {any} [binding] You should not set this property. This is
-   *                               used if DomBinding wants to create a
-   *                               association to the created DOM type.
-   * @return {Node} The {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}
-   *
-   * @public
-   */
-  toDOM (_document = document, hooks = {}, binding) {
-    const dom = _document.createElement(this.nodeName)
-    const attrs = this.getAttributes()
-    for (const key in attrs) {
-      dom.setAttribute(key, attrs[key])
-    }
-    typeListForEach(this, yxml => {
-      dom.appendChild(yxml.toDOM(_document, hooks, binding))
-    })
-    if (binding !== undefined) {
-      binding._createAssociation(dom, this)
-    }
-    return dom
-  }
-
   /**
    * Transform the properties of this type to binary and write it to an
    * BinaryEncoder.

src/types/YXmlFragment.js

@@ -6,18 +6,15 @@ import {
   YXmlEvent,
   YXmlElement,
   AbstractType,
-  typeListMap,
-  typeListForEach,
-  typeListInsertGenerics,
   typeListInsertGenericsAfter,
-  typeListDelete,
   typeListToArray,
   YXmlFragmentRefID,
   callTypeObservers,
   transact,
-  typeListGet,
   typeListSlice,
-  UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line
+  useSearchMarker,
+  UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot, // eslint-disable-line
+  ListCursor
 } from '../internals.js'
 
 import * as error from 'lib0/error'
@@ -256,7 +253,10 @@ export class YXmlFragment extends AbstractType {
    * @return {string} The string representation of all children.
    */
   toString () {
-    return typeListMap(this, xml => xml.toString()).join('')
+    if (this.doc != null) {
+      return transact(this.doc, tr => new ListCursor(this).map(tr, xml => xml.toString()).join(''))
+    }
+    return ''
   }
 
   /**
@@ -266,32 +266,6 @@ export class YXmlFragment extends AbstractType {
     return this.toString()
   }
 
-  /**
-   * Creates a Dom Element that mirrors this YXmlElement.
-   *
-   * @param {Document} [_document=document] The document object (you must define
-   *                                        this when calling this method in
-   *                                        nodejs)
-   * @param {Object<string, any>} [hooks={}] Optional property to customize how hooks
-   *                                             are presented in the DOM
-   * @param {any} [binding] You should not set this property. This is
-   *                               used if DomBinding wants to create a
-   *                               association to the created DOM type.
-   * @return {Node} The {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}
-   *
-   * @public
-   */
-  toDOM (_document = document, hooks = {}, binding) {
-    const fragment = _document.createDocumentFragment()
-    if (binding !== undefined) {
-      binding._createAssociation(fragment, this)
-    }
-    typeListForEach(this, xmlType => {
-      fragment.insertBefore(xmlType.toDOM(_document, hooks, binding), null)
-    })
-    return fragment
-  }
-
   /**
    * Inserts new content at an index.
    *
@@ -304,9 +278,11 @@ export class YXmlFragment extends AbstractType {
    */
   insert (index, content) {
     if (this.doc !== null) {
-      transact(this.doc, transaction => {
+      return transact(this.doc, transaction =>
-        typeListInsertGenerics(transaction, this, index, content)
+        useSearchMarker(transaction, this, index, walker =>
-      })
+          walker.insertArrayValue(transaction, content)
+        )
+      )
     } else {
       // @ts-ignore _prelimContent is defined because this is not yet integrated
       this._prelimContent.splice(index, 0, ...content)
@@ -347,9 +323,11 @@ export class YXmlFragment extends AbstractType {
    */
   delete (index, length = 1) {
     if (this.doc !== null) {
-      transact(this.doc, transaction => {
+      transact(/** @type {Doc} */ (this.doc), transaction =>
-        typeListDelete(transaction, this, index, length)
+        useSearchMarker(transaction, this, index, walker =>
-      })
+          walker.delete(transaction, length)
+        )
+      )
     } else {
       // @ts-ignore _prelimContent is defined because this is not yet integrated
       this._prelimContent.splice(index, length)
@@ -390,7 +368,11 @@ export class YXmlFragment extends AbstractType {
    * @return {YXmlElement|YXmlText}
    */
   get (index) {
-    return typeListGet(this, index)
+    return transact(/** @type {Doc} */ (this.doc), transaction =>
+      useSearchMarker(transaction, this, index, walker =>
+        walker.slice(transaction, 1)[0]
+      )
+    )
   }
 
   /**

src/types/YXmlHook.js

@@ -40,36 +40,6 @@ export class YXmlHook extends YMap {
     return el
   }
 
-  /**
-   * Creates a Dom Element that mirrors this YXmlElement.
-   *
-   * @param {Document} [_document=document] The document object (you must define
-   *                                        this when calling this method in
-   *                                        nodejs)
-   * @param {Object.<string, any>} [hooks] Optional property to customize how hooks
-   *                                             are presented in the DOM
-   * @param {any} [binding] You should not set this property. This is
-   *                               used if DomBinding wants to create a
-   *                               association to the created DOM type
-   * @return {Element} The {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}
-   *
-   * @public
-   */
-  toDOM (_document = document, hooks = {}, binding) {
-    const hook = hooks[this.hookName]
-    let dom
-    if (hook !== undefined) {
-      dom = hook.createDom(this)
-    } else {
-      dom = document.createElement(this.hookName)
-    }
-    dom.setAttribute('data-yjs-hook', this.hookName)
-    if (binding !== undefined) {
-      binding._createAssociation(dom, this)
-    }
-    return dom
-  }
-
   /**
    * Transform the properties of this type to binary and write it to an
    * BinaryEncoder.

src/types/YXmlText.js

@@ -39,29 +39,6 @@ export class YXmlText extends YText {
     return text
   }
 
-  /**
-   * Creates a Dom Element that mirrors this YXmlText.
-   *
-   * @param {Document} [_document=document] The document object (you must define
-   *                                        this when calling this method in
-   *                                        nodejs)
-   * @param {Object<string, any>} [hooks] Optional property to customize how hooks
-   *                                             are presented in the DOM
-   * @param {any} [binding] You should not set this property. This is
-   *                               used if DomBinding wants to create a
-   *                               association to the created DOM type.
-   * @return {Text} The {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}
-   *
-   * @public
-   */
-  toDOM (_document = document, hooks, binding) {
-    const dom = _document.createTextNode(this.toString())
-    if (binding !== undefined) {
-      binding._createAssociation(dom, this)
-    }
-    return dom
-  }
-
   toString () {
     // @ts-ignore
     return this.toDelta().map(delta => {

src/utils/ListCursor.js

@@ -0,0 +1,740 @@
+import * as error from 'lib0/error'
+
+import {
+  getItemCleanStart,
+  createID,
+  getMovedCoords,
+  updateMarkerChanges,
+  getState,
+  ContentAny,
+  ContentBinary,
+  ContentType,
+  ContentDoc,
+  Doc,
+  compareIDs,
+  createRelativePosition,
+  RelativePosition, ID, AbstractContent, ContentMove, Transaction, Item, AbstractType // eslint-disable-line
+} from '../internals.js'
+import { compareRelativePositions } from './RelativePosition.js'
+import * as array from 'lib0/array'
+
+const lengthExceeded = error.create('Length exceeded!')
+
+/**
+ * We keep the moved-stack across several transactions. Local or remote changes can invalidate
+ * "moved coords" on the moved-stack.
+ *
+ * The reason for this is that if assoc < 0, then getMovedCoords will return the target.right item.
+ * While the computed item is on the stack, it is possible that a user inserts something between target
+ * and the item on the stack. Then we expect that the newly inserted item is supposed to be on the new
+ * computed item.
+ *
+ * @param {Transaction} tr
+ * @param {ListCursor} li
+ */
+const popMovedStack = (tr, li) => {
+  let { start, end, move } = li.movedStack.pop() || { start: null, end: null, move: null }
+  if (move) {
+    const moveContent = /** @type {ContentMove} */ (move.content)
+    if (
+      (
+        moveContent.start.assoc < 0 && (
+          (start === null && moveContent.start.item !== null) ||
+          (start !== null && !compareIDs(/** @type {Item} */ (start.left).lastId, moveContent.start.item))
+        )
+      ) || (
+        moveContent.end.assoc < 0 && (
+          (end === null && moveContent.end.item !== null) ||
+          (end !== null && !compareIDs(/** @type {Item} */ (end.left).lastId, moveContent.end.item))
+        )
+      )
+    ) {
+      const coords = getMovedCoords(moveContent, tr, false)
+      start = coords.start
+      end = coords.end
+    }
+  }
+  li.currMove = move
+  li.currMoveStart = start
+  li.currMoveEnd = end
+  li.reachedEnd = false
+}
+
+/**
+ * Structure that helps to iterate through list-like structures. This is a useful abstraction that keeps track of move operations.
+ */
+export class ListCursor {
+  /**
+   * @param {AbstractType<any>} type
+   */
+  constructor (type) {
+    this.type = type
+    /**
+     * Current index-position
+     */
+    this.index = 0
+    /**
+     * Relative position to the current item (if item.content.length > 1)
+     */
+    this.rel = 0
+    /**
+     * This refers to the current right item, unless reachedEnd is true. Then it refers to the left item.
+     *
+     * @public
+     * @type {Item | null}
+     */
+    this.nextItem = type._start
+    this.reachedEnd = type._start === null
+    /**
+     * @type {Item | null}
+     */
+    this.currMove = null
+    /**
+     * @type {Item | null}
+     */
+    this.currMoveStart = null
+    /**
+     * @type {Item | null}
+     */
+    this.currMoveEnd = null
+    /**
+     * @type {Array<{ start: Item | null, end: Item | null, move: Item }>}
+     */
+    this.movedStack = []
+  }
+
+  clone () {
+    const iter = new ListCursor(this.type)
+    iter.index = this.index
+    iter.rel = this.rel
+    iter.nextItem = this.nextItem
+    iter.reachedEnd = this.reachedEnd
+    iter.currMove = this.currMove
+    iter.currMoveStart = this.currMoveStart
+    iter.currMoveEnd = this.currMoveEnd
+    iter.movedStack = this.movedStack.slice()
+    return iter
+  }
+
+  /**
+   * @type {Item | null}
+   */
+  get left () {
+    if (this.reachedEnd) {
+      return this.nextItem
+    } else {
+      return this.nextItem && this.nextItem.left
+    }
+  }
+
+  /**
+   * @type {Item | null}
+   */
+  get right () {
+    if (this.reachedEnd) {
+      return null
+    } else {
+      return this.nextItem
+    }
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {number} index
+   */
+  moveTo (tr, index) {
+    const diff = index - this.index
+    if (diff > 0) {
+      this.forward(tr, diff, true)
+    } else if (diff < 0) {
+      this.backward(tr, -diff)
+    }
+  }
+
+  /**
+   * When using skipUncountables=false within a "useSearchMarker" call, it is recommended
+   * to move the marker to the end. @todo do this after each useSearchMarkerCall
+   *
+   * @param {Transaction} tr
+   * @param {number} len
+   * @param {boolean} skipUncountables Iterate as much as possible iterating over uncountables until we find the next item.
+   */
+  forward (tr, len, skipUncountables) {
+    if (len === 0 && this.nextItem == null) {
+      return this
+    }
+    if (this.index + len > this.type._length || this.nextItem == null) {
+      throw lengthExceeded
+    }
+    let item = /** @type {Item} */ (this.nextItem)
+    this.index += len
+    // eslint-disable-next-line no-unmodified-loop-condition
+    while ((!this.reachedEnd || this.currMove !== null) && (len > 0 || (skipUncountables && len === 0 && item && (!item.countable || item.deleted || item === this.currMoveEnd || (this.reachedEnd && this.currMoveEnd === null) || item.moved !== this.currMove)))) {
+      if (item === this.currMoveEnd || (this.currMoveEnd === null && this.reachedEnd && this.currMove)) {
+        item = /** @type {Item} */ (this.currMove) // we iterate to the right after the current condition
+        popMovedStack(tr, this)
+      } else if (item === null) {
+        error.unexpectedCase() // should never happen
+      } else if (item.countable && !item.deleted && item.moved === this.currMove && len > 0) {
+        len -= item.length
+        if (len < 0) {
+          this.rel = item.length + len
+          len = 0
+          break
+        }
+      } else if (item.content.constructor === ContentMove && item.moved === this.currMove) {
+        if (this.currMove) {
+          this.movedStack.push({ start: this.currMoveStart, end: this.currMoveEnd, move: this.currMove })
+        }
+        const { start, end } = getMovedCoords(item.content, tr, false)
+        this.currMove = item
+        this.currMoveStart = start
+        this.currMoveEnd = end
+        item = start
+        continue
+      }
+      if (this.reachedEnd) {
+        throw error.unexpectedCase
+      }
+      if (item.right) {
+        item = item.right
+      } else {
+        this.reachedEnd = true
+      }
+    }
+    this.index -= len
+    this.nextItem = item
+    return this
+  }
+
+  /**
+   * We prefer to insert content outside of a moved range.
+   * Try to escape the moved range by walking to the left over deleted items.
+   *
+   * @param {Transaction} tr
+   */
+  reduceMoveDepth (tr) {
+    let nextItem = this.nextItem
+    if (nextItem !== null) {
+      while (this.currMove) {
+        if (nextItem === this.currMoveStart) {
+          nextItem = /** @type {Item} */ (this.currMove) // we iterate to the left after the current condition
+          popMovedStack(tr, this)
+          continue
+        }
+        // check if we can iterate to the left while stepping over deleted items until we find an item === this.currMoveStart
+        /**
+         * @type {Item} nextItem
+         */
+        let item = nextItem
+        while (item.deleted && item.moved === this.currMove && item !== this.currMoveStart) {
+          item = /** @type {Item} */ (item.left) // this must exist otherwise we miscalculated the move
+        }
+        if (item === this.currMoveStart) {
+          // we only want to iterate over deleted items if we can escape a move
+          nextItem = item
+        } else {
+          break
+        }
+      }
+      this.nextItem = nextItem
+    }
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {number} len
+   * @return {ListCursor}
+   */
+  backward (tr, len) {
+    if (this.index - len < 0) {
+      throw lengthExceeded
+    }
+    this.index -= len
+    if (this.reachedEnd) {
+      const nextItem = /** @type {Item} */ (this.nextItem)
+      this.rel = nextItem.countable && !nextItem.deleted ? nextItem.length : 0
+      this.reachedEnd = false
+    }
+    if (this.rel >= len) {
+      this.rel -= len
+      return this
+    }
+    let item = this.nextItem
+    if (item && item.content.constructor === ContentMove) {
+      item = item.left
+    } else {
+      len += ((item && item.countable && !item.deleted && item.moved === this.currMove) ? item.length : 0) - this.rel
+    }
+    this.rel = 0
+    while (item && len > 0) {
+      if (item.countable && !item.deleted && item.moved === this.currMove) {
+        len -= item.length
+        if (len < 0) {
+          this.rel = -len
+          len = 0
+        }
+        if (len === 0) {
+          break
+        }
+      } else if (item.content.constructor === ContentMove && item.moved === this.currMove) {
+        if (this.currMove) {
+          this.movedStack.push({ start: this.currMoveStart, end: this.currMoveEnd, move: this.currMove })
+        }
+        const { start, end } = getMovedCoords(item.content, tr, false)
+        this.currMove = item
+        this.currMoveStart = start
+        this.currMoveEnd = end
+        item = start
+        continue
+      }
+      if (item === this.currMoveStart) {
+        item = /** @type {Item} */ (this.currMove) // we iterate to the left after the current condition
+        popMovedStack(tr, this)
+      }
+      item = item.left
+    }
+    this.nextItem = item
+    return this
+  }
+
+  /**
+   * @template {{length: number}} T
+   * @param {Transaction} tr
+   * @param {number} len
+   * @param {T} value the initial content
+   * @param {function(AbstractContent, number, number):T} slice
+   * @param {function(T, T): T} concat
+   */
+  _slice (tr, len, value, slice, concat) {
+    if (this.index + len > this.type._length) {
+      throw lengthExceeded
+    }
+    this.index += len
+    /**
+     * We store nextItem in a variable because this version cannot be null.
+     */
+    let nextItem = /** @type {Item} */ (this.nextItem)
+    while (len > 0 && !this.reachedEnd) {
+      while (nextItem.countable && !this.reachedEnd && len > 0 && nextItem !== this.currMoveEnd) {
+        if (!nextItem.deleted && nextItem.moved === this.currMove) {
+          const slicedContent = slice(nextItem.content, this.rel, len)
+          len -= slicedContent.length
+          value = concat(value, slicedContent)
+          if (this.rel + slicedContent.length === nextItem.length) {
+            this.rel = 0
+          } else {
+            this.rel += slicedContent.length
+            continue // do not iterate to item.right
+          }
+        }
+        if (nextItem.right) {
+          nextItem = nextItem.right
+        } else {
+          this.reachedEnd = true
+        }
+      }
+      if ((!this.reachedEnd || this.currMove !== null) && len > 0) {
+        // always set nextItem before any method call
+        this.nextItem = nextItem
+        this.forward(tr, 0, true)
+        nextItem = this.nextItem
+      }
+    }
+    this.nextItem = nextItem
+    if (len < 0) {
+      this.index -= len
+    }
+    return value
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {number} len
+   */
+  delete (tr, len) {
+    const startLength = len
+    const sm = this.type._searchMarker
+    let item = this.nextItem
+    if (this.index + len > this.type._length) {
+      throw lengthExceeded
+    }
+    while (len > 0) {
+      while (item && !item.deleted && item.countable && !this.reachedEnd && len > 0 && item.moved === this.currMove && item !== this.currMoveEnd) {
+        if (this.rel > 0) {
+          item = getItemCleanStart(tr, createID(item.id.client, item.id.clock + this.rel))
+          this.rel = 0
+        }
+        if (len < item.length) {
+          getItemCleanStart(tr, createID(item.id.client, item.id.clock + len))
+        }
+        len -= item.length
+        item.delete(tr)
+        if (item.right) {
+          item = item.right
+        } else {
+          this.reachedEnd = true
+        }
+      }
+      if (len > 0) {
+        this.nextItem = item
+        this.forward(tr, 0, true)
+        item = this.nextItem
+      }
+    }
+    this.nextItem = item
+    if (sm) {
+      updateMarkerChanges(sm, this.index, -startLength + len, this)
+    }
+  }
+
+  /**
+   * @param {Transaction} tr
+   */
+  _splitRel (tr) {
+    if (this.rel > 0) {
+      /**
+       * @type {ID}
+       */
+      const itemid = /** @type {Item} */ (this.nextItem).id
+      this.nextItem = getItemCleanStart(tr, createID(itemid.client, itemid.clock + this.rel))
+      this.rel = 0
+    }
+  }
+
+  /**
+   * Important: you must update markers after calling this method!
+   *
+   * @param {Transaction} tr
+   * @param {Array<AbstractContent>} content
+   */
+  insertContents (tr, content) {
+    this.reduceMoveDepth(tr)
+    this._splitRel(tr)
+    const parent = this.type
+    const store = tr.doc.store
+    const ownClientId = tr.doc.clientID
+    /**
+     * @type {Item | null}
+     */
+    const right = this.right
+
+    /**
+     * @type {Item | null}
+     */
+    let left = this.left
+    content.forEach(c => {
+      left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, c)
+      left.integrate(tr, 0)
+    })
+    if (right === null) {
+      this.nextItem = left
+      this.reachedEnd = true
+    } else {
+      this.nextItem = right
+    }
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {Array<{ start: RelativePosition, end: RelativePosition }>} ranges
+   */
+  insertMove (tr, ranges) {
+    this.insertContents(tr, ranges.map(range => new ContentMove(range.start, range.end, -1)))
+    // @todo is there a better alrogirthm to update searchmarkers? We could simply remove the markers that are in the updated range.
+    // Also note that searchmarkers are updated in insertContents as well.
+    const sm = this.type._searchMarker
+    if (sm) sm.length = 0 // @todo instead, iterate through sm and delete all marked properties on items
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {Array<Object<string,any>|Array<any>|boolean|number|null|string|Uint8Array>} values
+   */
+  insertArrayValue (tr, values) {
+    this._splitRel(tr)
+    const sm = this.type._searchMarker
+    /**
+     * @type {Array<AbstractContent>}
+     */
+    const contents = []
+    /**
+     * @type {Array<Object|Array<any>|number|null>}
+     */
+    let jsonContent = []
+    const packJsonContent = () => {
+      if (jsonContent.length > 0) {
+        contents.push(new ContentAny(jsonContent))
+        jsonContent = []
+      }
+    }
+    values.forEach(c => {
+      if (c === null) {
+        jsonContent.push(c)
+      } else {
+        switch (c.constructor) {
+          case Number:
+          case Object:
+          case Boolean:
+          case Array:
+          case String:
+            jsonContent.push(c)
+            break
+          default:
+            packJsonContent()
+            switch (c.constructor) {
+              case Uint8Array:
+              case ArrayBuffer:
+                contents.push(new ContentBinary(new Uint8Array(/** @type {Uint8Array} */ (c))))
+                break
+              case Doc:
+                contents.push(new ContentDoc(/** @type {Doc} */ (c)))
+                break
+              default:
+                if (c instanceof AbstractType) {
+                  contents.push(new ContentType(c))
+                } else {
+                  throw new Error('Unexpected content type in insert operation')
+                }
+            }
+        }
+      }
+    })
+    packJsonContent()
+    this.insertContents(tr, contents)
+    this.index += values.length
+    if (sm) {
+      updateMarkerChanges(sm, this.index - values.length, values.length, this)
+    }
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {number} len
+   */
+  slice (tr, len) {
+    return this._slice(tr, len, [], sliceArrayContent, concatArrayContent)
+  }
+
+  /**
+   * @param {Transaction} tr
+   * @param {function(any, number, any):void} f
+   */
+  forEach (tr, f) {
+    for (const val of this.values(tr)) {
+      // decrease index because retrieving value will increase index
+      f(val, this.index - 1, this.type)
+    }
+  }
+
+  /**
+   * @template T
+   * @param {Transaction} tr
+   * @param {function(any, number, any):T} f
+   * @return {Array<T>}
+   */
+  map (tr, f) {
+    const arr = new Array(this.type._length - this.index)
+    let i = 0
+    for (const val of this.values(tr)) {
+      arr[i++] = f(val, this.index - 1, this.type)
+    }
+    return arr
+  }
+
+  /**
+   * @param {Transaction} tr
+   */
+  values (tr) {
+    return {
+      [Symbol.iterator] () {
+        return this
+      },
+      next: () => {
+        if (this.reachedEnd || this.index === this.type._length) {
+          return { done: true }
+        }
+        const [value] = this.slice(tr, 1)
+        return {
+          done: false,
+          value: value
+        }
+      }
+    }
+  }
+}
+
+/**
+ * @param {AbstractContent} itemcontent
+ * @param {number} start
+ * @param {number} len
+ */
+const sliceArrayContent = (itemcontent, start, len) => {
+  const content = itemcontent.getContent()
+  return content.length <= len && start === 0 ? content : content.slice(start, start + len)
+}
+/**
+ * @param {Array<any>} content
+ * @param {Array<any>} added
+ */
+const concatArrayContent = (content, added) => {
+  content.push(...added)
+  return content
+}
+
+/**
+ * Move-ranges must not cross each other.
+ *
+ * This function computes the minimal amount of ranges to move a range of content to
+ * a different place.
+ *
+ * Algorithm:
+ * * Store the current stack in $preStack and $preItem = walker.nextItem
+ * * Iterate forward $len items.
+ * * The current stack is stored is $afterStack and $
+ * * Delete the stack-items that both of them have in common
+ *
+ * @param {Transaction} tr
+ * @param {ListCursor} walker
+ * @param {number} len
+ * @return {Array<{ start: RelativePosition, end: RelativePosition }>}
+ */
+export const getMinimalListViewRanges = (tr, walker, len) => {
+  if (len === 0) return []
+  if (walker.index + len > walker.type._length) {
+    throw lengthExceeded
+  }
+  // stepping outside the current move-range as much as possible
+  walker.reduceMoveDepth(tr)
+
+  /**
+   * @type {Array<{ start: RelativePosition, end: RelativePosition, move: Item | null }>}
+   */
+  const ranges = []
+  // store relevant information for the beginning, before we iterate forward
+  /**
+   * @type {Array<Item>}
+   */
+  const preStack = walker.movedStack.map(si => si.move)
+  const preMove = walker.currMove
+  const preItem = /** @type {Item} */ (walker.nextItem)
+  const preRel = walker.rel
+
+  walker.forward(tr, len, false)
+
+  // store the same information for the end, after we iterate forward
+  /**
+   * @type {Array<Item>}
+   */
+  const afterStack = walker.movedStack.map(si => si.move)
+  const afterMove = walker.currMove
+  /**
+  const nextIsCurrMoveStart = walker.nextItem === walker.currMoveStart
+    const afterItem = /** @type {Item} / (nextIsCurrMoveStart
+    ? walker.currMove
+    : (walker.rel > 0 || walker.reachedEnd)
+        ? walker.nextItem
+        : /** @type {Item} / (walker.nextItem).left
+  ) */
+  const afterItem = /** @type {Item} */ (
+    (walker.rel > 0 || walker.reachedEnd)
+      ? walker.nextItem
+      : /** @type {Item} */ (walker.nextItem).left
+  )
+  /**
+   * afterRel is always > 0
+   */
+  const afterRel = walker.rel > 0
+    ? walker.rel
+    : afterItem.length
+
+  walker.forward(tr, 0, false) // @todo remove once this is done is useSearchMarker
+
+  let start = createRelativePosition(walker.type, createID(preItem.id.client, preItem.id.clock + preRel), 0)
+  let end = createRelativePosition(
+    walker.type,
+    createID(afterItem.id.client, afterItem.id.clock + afterRel - 1),
+    -1
+  )
+
+  if (preMove) {
+    preStack.push(preMove)
+  }
+  if (afterMove) {
+    afterStack.push(afterMove)
+  }
+
+  // remove common stack-items
+  while (preStack.length > 0 && preStack[0] === afterStack[0]) {
+    preStack.shift()
+    afterStack.shift()
+  }
+  const topLevelMove = preStack.length > 0 ? preStack[0].moved : (afterStack.length > 0 ? afterStack[0].moved : null)
+
+  // remove stack-items that are useless for our computation (that wouldn't produce meaningful ranges)
+  // @todo
+
+  while (preStack.length > 0) {
+    const move = /** @type {Item} */ (preStack.pop())
+    ranges.push({
+      start,
+      end: /** @type {ContentMove} */ (move.content).end,
+      move
+    })
+    start = createRelativePosition(walker.type, createID(move.id.client, move.id.clock), -1)
+  }
+
+  const middleMove = { start, end, move: topLevelMove }
+  ranges.push(middleMove)
+
+  while (afterStack.length > 0) {
+    const move = /** @type {Item} */ (afterStack.pop())
+    ranges.push({
+      start: /** @type {ContentMove} */ (move.content).start,
+      end,
+      move
+    })
+    end = createRelativePosition(walker.type, createID(move.id.client, move.id.clock), 0)
+  }
+
+  // Update end of the center move operation
+  // Move ranges must be applied in order
+  middleMove.end = end
+
+  const normalizedRanges = array.flatten(ranges.map(range => {
+    // A subset of a range could be moved by another move with a higher priority.
+    // If that is the case, we need to ignore those moved items.
+    const { start, end } = getMovedCoords(range, tr, false)
+    const move = range.move
+    const ranges = []
+    /**
+     * @type {RelativePosition | null}
+     */
+    let rangeStart = range.start
+    /**
+     * @type {Item}
+     */
+    let item = start
+    while (item !== end) {
+      if (item.moved !== move && rangeStart != null) {
+        ranges.push({ start: rangeStart, end: createRelativePosition(walker.type, createID(item.id.client, item.id.clock), 0) })
+        rangeStart = null
+      }
+      if (item.moved === move && rangeStart === null) {
+        // @todo It might be better to set this to item.left, with assoc -1
+        rangeStart = createRelativePosition(walker.type, createID(item.id.client, item.id.clock), 0)
+      }
+      item = /** @type {Item} */ (item.right)
+    }
+    if (rangeStart != null) {
+      ranges.push({
+        start: rangeStart,
+        end: range.end
+      })
+    }
+    return ranges
+  }))
+
+  // filter out unnecessary ranges
+  return normalizedRanges.filter(range => !compareRelativePositions(range.start, range.end))
+}

src/utils/RelativePosition.js

@@ -9,6 +9,8 @@ import {
   createID,
   ContentType,
   followRedone,
+  transact,
+  useSearchMarker,
   ID, Doc, AbstractType // eslint-disable-line
 } from '../internals.js'
 
@@ -73,6 +75,10 @@ export class RelativePosition {
      */
     this.assoc = assoc
   }
+
+  clone () {
+    return new RelativePosition(this.type, this.tname, this.item, this.assoc)
+  }
 }
 
 /**
@@ -161,7 +167,6 @@ export const createRelativePosition = (type, item, assoc) => {
  * @function
  */
 export const createRelativePositionFromTypeIndex = (type, index, assoc = 0) => {
-  let t = type._start
   if (assoc < 0) {
     // associated to the left character or the beginning of a type, increment index if possible.
     if (index === 0) {
@@ -169,21 +174,17 @@ export const createRelativePositionFromTypeIndex = (type, index, assoc = 0) => {
     }
     index--
   }
-  while (t !== null) {
+  return transact(/** @type {Doc} */ (type.doc), tr =>
-    if (!t.deleted && t.countable) {
+    useSearchMarker(tr, type, index, walker => {
-      if (t.length > index) {
+      if (walker.reachedEnd) {
-        // case 1: found position somewhere in the linked list
+        const item = assoc < 0 ? /** @type {Item} */ (walker.nextItem).lastId : null
-        return createRelativePosition(type, createID(t.id.client, t.id.clock + index), assoc)
+        return createRelativePosition(type, item, assoc)
+      } else {
+        const id = /** @type {Item} */ (walker.nextItem).id
+        return createRelativePosition(type, createID(id.client, id.clock + walker.rel), assoc)
       }
-      index -= t.length
+    })
-    }
+  )
-    if (t.right === null && assoc < 0) {
-      // left-associated position, return last available id
-      return createRelativePosition(type, t.lastId, assoc)
-    }
-    t = t.right
-  }
-  return createRelativePosition(type, null, assoc)
 }
 
 /**
@@ -195,7 +196,7 @@ export const createRelativePositionFromTypeIndex = (type, index, assoc = 0) => {
 export const writeRelativePosition = (encoder, rpos) => {
   const { type, tname, item, assoc } = rpos
   if (item !== null) {
-    encoding.writeVarUint(encoder, 0)
+    encoding.writeUint8(encoder, 0)
     writeID(encoder, item)
   } else if (tname !== null) {
     // case 2: found position at the end of the list and type is stored in y.share
@@ -232,7 +233,7 @@ export const readRelativePosition = decoder => {
   let type = null
   let tname = null
   let itemID = null
-  switch (decoding.readVarUint(decoder)) {
+  switch (decoding.readUint8(decoder)) {
     case 0:
       // case 1: found position somewhere in the linked list
       itemID = readID(decoder)

src/utils/StructStore.js

@@ -199,19 +199,18 @@ export const getItemCleanStart = (transaction, id) => {
  * Expects that id is actually in store. This function throws or is an infinite loop otherwise.
  *
  * @param {Transaction} transaction
- * @param {StructStore} store
  * @param {ID} id
  * @return {Item}
  *
  * @private
  * @function
  */
-export const getItemCleanEnd = (transaction, store, id) => {
+export const getItemCleanEnd = (transaction, id) => {
   /**
    * @type {Array<Item>}
    */
   // @ts-ignore
-  const structs = store.clients.get(id.client)
+  const structs = transaction.doc.store.clients.get(id.client)
   const index = findIndexSS(structs, id.clock)
   const struct = structs[index]
   if (id.clock !== struct.id.clock + struct.length - 1 && struct.constructor !== GC) {

src/utils/Transaction.js

@@ -14,6 +14,7 @@ import {
   UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line
 } from '../internals.js'
 
+import * as encoding from 'lib0/encoding'
 import * as map from 'lib0/map'
 import * as math from 'lib0/math'
 import * as set from 'lib0/set'
@@ -61,6 +62,13 @@ export class Transaction {
      * @type {DeleteSet}
      */
     this.deleteSet = new DeleteSet()
+    /**
+     * These deletes were used to cleanup the document and
+     * should be broadcasted again using a different transaction-origin.
+     *
+     * @type {DeleteSet}
+     */
+    this.cleanupDeletions = new DeleteSet()
     /**
      * Holds the state before the transaction started.
      * @type {Map<Number,Number>}
@@ -114,6 +122,14 @@ export class Transaction {
      * @type {Set<Doc>}
      */
     this.subdocsLoaded = new Set()
+    /**
+     * We store the reference that last moved an item.
+     * This is needed to compute the delta when multiple ContentMove move
+     * the same item.
+     *
+     * @type {Map<Item, Item>}
+     */
+    this.prevMoved = new Map()
   }
 }
 
@@ -132,6 +148,18 @@ export const writeUpdateMessageFromTransaction = (encoder, transaction) => {
   return true
 }
 
+/**
+ * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
+ * @param {Transaction} transaction
+ */
+export const writeCleanupMessageFromTransaction = (encoder, transaction) => {
+  const ds = transaction.cleanupDeletions
+  sortAndMergeDeleteSet(ds)
+  // write structs: 0 structs were created
+  encoding.writeVarUint(encoder.restEncoder, 0)
+  writeDeleteSet(encoder, ds)
+}
+
 /**
  * @param {Transaction} transaction
  *
@@ -336,11 +364,17 @@ const cleanupTransactions = (transactionCleanups, i) => {
       }
       // @todo Merge all the transactions into one and provide send the data as a single update message
       doc.emit('afterTransactionCleanup', [transaction, doc])
+      const needsCleanupEvent = transaction.cleanupDeletions.clients.size > 0
       if (doc._observers.has('update')) {
         const encoder = new UpdateEncoderV1()
         const hasContent = writeUpdateMessageFromTransaction(encoder, transaction)
         if (hasContent) {
           doc.emit('update', [encoder.toUint8Array(), transaction.origin, doc, transaction])
+          if (needsCleanupEvent) {
+            const encoder = new UpdateEncoderV1()
+            writeCleanupMessageFromTransaction(encoder, transaction)
+            doc.emit('update', [encoder.toUint8Array(), 'cleanup', doc, transaction])
+          }
         }
       }
       if (doc._observers.has('updateV2')) {
@@ -348,6 +382,11 @@ const cleanupTransactions = (transactionCleanups, i) => {
         const hasContent = writeUpdateMessageFromTransaction(encoder, transaction)
         if (hasContent) {
           doc.emit('updateV2', [encoder.toUint8Array(), transaction.origin, doc, transaction])
+          if (needsCleanupEvent) {
+            const encoder = new UpdateEncoderV2()
+            writeCleanupMessageFromTransaction(encoder, transaction)
+            doc.emit('updateV2', [encoder.toUint8Array(), 'cleanup', doc, transaction])
+          }
         }
       }
       const { subdocsAdded, subdocsLoaded, subdocsRemoved } = transaction
@@ -377,9 +416,12 @@ const cleanupTransactions = (transactionCleanups, i) => {
 /**
  * Implements the functionality of `y.transact(()=>{..})`
  *
+ * @template T
+ *
  * @param {Doc} doc
- * @param {function(Transaction):void} f
+ * @param {function(Transaction):T} f
  * @param {any} [origin=true]
+ * @return {T}
  *
  * @function
  */
@@ -395,8 +437,9 @@ export const transact = (doc, f, origin = null, local = true) => {
     }
     doc.emit('beforeTransaction', [doc._transaction, doc])
   }
+  let res
   try {
-    f(doc._transaction)
+    res = f(doc._transaction)
   } finally {
     if (initialCall && transactionCleanups[0] === doc._transaction) {
       // The first transaction ended, now process observer calls.
@@ -410,4 +453,12 @@ export const transact = (doc, f, origin = null, local = true) => {
       cleanupTransactions(transactionCleanups, 0)
     }
   }
+  return res
 }
+
+/**
+ * @param {Transaction} tr
+ * @param {AbstractStruct} struct
+ */
+export const addsStruct = (tr, struct) =>
+  struct.id.clock >= (tr.beforeState.get(struct.id.client) || 0)

src/utils/YEvent.js

@@ -1,11 +1,14 @@
 
 import {
   isDeleted,
-  Item, AbstractType, Transaction, AbstractStruct // eslint-disable-line
+  getMovedCoords,
+  ContentMove, Item, AbstractType, Transaction, AbstractStruct // eslint-disable-line
 } from '../internals.js'
 
 import * as set from 'lib0/set'
 import * as array from 'lib0/array'
+import { addsStruct } from './Transaction.js'
+import { ListCursor } from './ListCursor.js'
 
 /**
  * YEvent describes the changes on a YType.
@@ -60,7 +63,7 @@ export class YEvent {
    */
   get path () {
     // @ts-ignore _item is defined because target is integrated
-    return getPathTo(this.currentTarget, this.target)
+    return getPathTo(this.currentTarget, this.target, this.transaction)
   }
 
   /**
@@ -140,11 +143,13 @@ export class YEvent {
    *
    * In contrast to change.deleted, this method also returns true if the struct was added and then deleted.
    *
+   * @todo this can be removed in the next release (prefer function)
+   *
    * @param {AbstractStruct} struct
    * @return {boolean}
    */
   adds (struct) {
-    return struct.id.clock >= (this.transaction.beforeState.get(struct.id.client) || 0)
+    return addsStruct(this.transaction, struct)
   }
 
   /**
@@ -153,62 +158,129 @@ export class YEvent {
   get changes () {
     let changes = this._changes
     if (changes === null) {
-      const target = this.target
+      this.transaction.doc.transact(tr => {
-      const added = set.create()
+        const target = this.target
-      const deleted = set.create()
+        const added = set.create()
-      /**
+        const deleted = set.create()
-       * @type {Array<{insert:Array<any>}|{delete:number}|{retain:number}>}
-       */
-      const delta = []
-      changes = {
-        added,
-        deleted,
-        delta,
-        keys: this.keys
-      }
-      const changed = /** @type Set<string|null> */ (this.transaction.changed.get(target))
-      if (changed.has(null)) {
         /**
-         * @type {any}
+         * @type {Array<{insert:Array<any>}|{delete:number}|{retain:number}>}
          */
-        let lastOp = null
+        const delta = []
-        const packOp = () => {
+        changes = {
-          if (lastOp) {
+          added,
-            delta.push(lastOp)
+          deleted,
-          }
+          delta,
+          keys: this.keys
         }
-        for (let item = target._start; item !== null; item = item.right) {
+        const changed = /** @type Set<string|null> */ (this.transaction.changed.get(target))
-          if (item.deleted) {
+        if (changed.has(null)) {
-            if (this.deletes(item) && !this.adds(item)) {
+          /**
-              if (lastOp === null || lastOp.delete === undefined) {
+           * @type {Array<{ end: Item | null, move: Item | null, isNew: boolean, isDeleted: boolean }>}
-                packOp()
+           */
-                lastOp = { delete: 0 }
+          const movedStack = []
-              }
+          /**
-              lastOp.delete += item.length
+           * @type {Item | null}
-              deleted.add(item)
+           */
-            } // else nop
+          let currMove = null
-          } else {
+          /**
-            if (this.adds(item)) {
+           * @type {boolean}
-              if (lastOp === null || lastOp.insert === undefined) {
+           */
-                packOp()
+          let currMoveIsNew = false
-                lastOp = { insert: [] }
+          /**
-              }
+           * @type {boolean}
-              lastOp.insert = lastOp.insert.concat(item.content.getContent())
+           */
-              added.add(item)
+          let currMoveIsDeleted = false
-            } else {
+          /**
-              if (lastOp === null || lastOp.retain === undefined) {
+           * @type {Item | null}
-                packOp()
+           */
-                lastOp = { retain: 0 }
+          let currMoveEnd = null
-              }
+          /**
-              lastOp.retain += item.length
+           * @type {any}
+           */
+          let lastOp = null
+          const packOp = () => {
+            if (lastOp) {
+              delta.push(lastOp)
             }
           }
+          /**
+           * @param {Item} item
+           */
+          const isMovedByNew = item => {
+            let moved = item.moved
+            while (moved != null) {
+              if (this.adds(moved)) {
+                return true
+              }
+              moved = moved.moved
+            }
+            return false
+          }
+          for (let item = target._start; ;) {
+            if (item === currMoveEnd && currMove) {
+              item = currMove
+              const { end, move, isNew, isDeleted } = movedStack.pop() || { end: null, move: null, isNew: false, isDeleted: false }
+              currMoveIsNew = isNew
+              currMoveIsDeleted = isDeleted
+              currMoveEnd = end
+              currMove = move
+            } else if (item === null) {
+              break
+            } else if (item.content.constructor === ContentMove) {
+              if (item.moved === currMove && (!item.deleted || (this.deletes(item) && !this.adds(item)))) {
+                movedStack.push({ end: currMoveEnd, move: currMove, isNew: currMoveIsNew, isDeleted: currMoveIsDeleted })
+                const { start, end } = getMovedCoords(item.content, tr, true) // We must split items for move-ranges, for single moves no splitting suffices
+                currMove = item
+                currMoveEnd = end
+                currMoveIsNew = this.adds(item) || currMoveIsNew
+                currMoveIsDeleted = item.deleted || currMoveIsDeleted
+                item = start
+                continue // do not move to item.right
+              }
+            } else if (item.moved !== currMove) {
+              if (!currMoveIsNew && item.countable && (!item.deleted || this.deletes(item)) && !this.adds(item) && (item.moved === null || isMovedByNew(item) || currMoveIsDeleted) && (this.transaction.prevMoved.get(item) || null) === currMove) {
+                if (lastOp === null || lastOp.delete === undefined) {
+                  packOp()
+                  lastOp = { delete: 0 }
+                }
+                lastOp.delete += item.length
+              }
+            } else if (item.deleted) {
+              if (!currMoveIsNew && this.deletes(item) && !this.adds(item) && !this.transaction.prevMoved.has(item)) {
+                if (lastOp === null || lastOp.delete === undefined) {
+                  packOp()
+                  lastOp = { delete: 0 }
+                }
+                lastOp.delete += item.length
+                deleted.add(item)
+              }
+            } else {
+              if (currMoveIsNew || this.adds(item) || this.transaction.prevMoved.has(item)) {
+                if (lastOp === null || lastOp.insert === undefined) {
+                  packOp()
+                  lastOp = { insert: [] }
+                }
+                // @todo push items instead (or splice..)
+                lastOp.insert = lastOp.insert.concat(item.content.getContent())
+                if (!currMoveIsNew) {
+                  added.add(item)
+                }
+              } else {
+                if (lastOp === null || lastOp.retain === undefined) {
+                  packOp()
+                  lastOp = { retain: 0 }
+                }
+                lastOp.retain += item.length
+              }
+            }
+            item = /** @type {Item} */ (item).right
+          }
+          if (lastOp !== null && lastOp.retain == null) {
+            packOp()
+          }
         }
-        if (lastOp !== null && lastOp.retain === undefined) {
+        this._changes = changes
-          packOp()
+      })
-        }
-      }
-      this._changes = changes
     }
     return /** @type {any} */ (changes)
   }
@@ -226,12 +298,13 @@ export class YEvent {
  *
  * @param {AbstractType<any>} parent
  * @param {AbstractType<any>} child target
+ * @param {Transaction} tr
  * @return {Array<string|number>} Path to the target
  *
  * @private
  * @function
  */
-const getPathTo = (parent, child) => {
+const getPathTo = (parent, child, tr) => {
   const path = []
   while (child._item !== null && child !== parent) {
     if (child._item.parentSub !== null) {
@@ -239,15 +312,11 @@ const getPathTo = (parent, child) => {
       path.unshift(child._item.parentSub)
     } else {
       // parent is array-ish
-      let i = 0
+      const c = new ListCursor(/** @type {AbstractType<any>} */ (child._item.parent))
-      let c = /** @type {AbstractType<any>} */ (child._item.parent)._start
+      while (c.nextItem != null && !c.reachedEnd && c.nextItem !== child._item) {
-      while (c !== child._item && c !== null) {
+        c.forward(tr, (c.nextItem.countable && !c.nextItem.deleted) ? c.nextItem.length : 0, true)
-        if (!c.deleted) {
-          i++
-        }
-        c = c.right
       }
-      path.unshift(i)
+      path.unshift(c.index)
     }
     child = /** @type {AbstractType<any>} */ (child._item.parent)
   }

src/utils/encoding.js

@@ -193,7 +193,6 @@ export const readClientsStructRefs = (decoder, doc) => {
         }
       }
     }
-    // console.log('time to read: ', performance.now() - start) // @todo remove
   }
   return clientRefs
 }
@@ -389,10 +388,6 @@ export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = n
     const store = doc.store
     // let start = performance.now()
     const ss = readClientsStructRefs(structDecoder, doc)
-    // console.log('time to read structs: ', performance.now() - start) // @todo remove
-    // start = performance.now()
-    // console.log('time to merge: ', performance.now() - start) // @todo remove
-    // start = performance.now()
     const restStructs = integrateStructs(transaction, store, ss)
     const pending = store.pendingStructs
     if (pending) {
@@ -416,8 +411,6 @@ export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = n
     } else {
       store.pendingStructs = restStructs
     }
-    // console.log('time to integrate: ', performance.now() - start) // @todo remove
-    // start = performance.now()
     const dsRest = readAndApplyDeleteSet(structDecoder, transaction, store)
     if (store.pendingDs) {
       // @todo we could make a lower-bound state-vector check as we do above
@@ -437,11 +430,6 @@ export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = n
       // Either dsRest == null && pendingDs == null OR dsRest != null
       store.pendingDs = dsRest
     }
-    // console.log('time to cleanup: ', performance.now() - start) // @todo remove
-    // start = performance.now()
-
-    // console.log('time to resume delete readers: ', performance.now() - start) // @todo remove
-    // start = performance.now()
     if (retry) {
       const update = /** @type {{update: Uint8Array}} */ (store.pendingStructs).update
       store.pendingStructs = null

src/utils/updates.js

@@ -308,6 +308,7 @@ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = U
   // Note: Should handle that some operations cannot be applied yet ()
 
   while (true) {
+    // @todo this incurs an exponential overhead. We could instead only sort the item that changed.
     // Write higher clients first ⇒ sort by clientID & clock and remove decoders without content
     lazyStructDecoders = lazyStructDecoders.filter(dec => dec.curr !== null)
     lazyStructDecoders.sort(

tests/doc.tests.js

@@ -40,6 +40,7 @@ export const testToJSON = tc => {
 
   const arr = doc.getArray('array')
   arr.push(['test1'])
+  t.compare(arr.toJSON(), ['test1'])
 
   const map = doc.getMap('map')
   map.set('k1', 'v1')

tests/encoding.tests.js

@@ -12,6 +12,7 @@ import {
   readContentFormat,
   readContentAny,
   readContentDoc,
+  readContentMove,
   Doc,
   PermanentUserData,
   encodeStateAsUpdate,
@@ -24,7 +25,8 @@ import * as Y from '../src/index.js'
  * @param {t.TestCase} tc
  */
 export const testStructReferences = tc => {
-  t.assert(contentRefs.length === 11)
+  t.assert(contentRefs.length === 12)
+  // contentRefs[0] is reserved for GC
   t.assert(contentRefs[1] === readContentDeleted)
   t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json?
   t.assert(contentRefs[3] === readContentBinary)
@@ -35,6 +37,7 @@ export const testStructReferences = tc => {
   t.assert(contentRefs[8] === readContentAny)
   t.assert(contentRefs[9] === readContentDoc)
   // contentRefs[10] is reserved for Skip structs
+  t.assert(contentRefs[11] === readContentMove)
 }
 
 /**
@@ -74,7 +77,7 @@ export const testDiffStateVectorOfUpdateIsEmpty = tc => {
   /**
    * @type {null | Uint8Array}
    */
-  let sv = /* any */ (null)
+  let sv = /** @type {any} */ (null)
   ydoc.getText().insert(0, 'a')
   ydoc.on('update', update => {
     sv = Y.encodeStateVectorFromUpdate(update)

tests/index.js

@@ -10,6 +10,7 @@ import * as doc from './doc.tests.js'
 import * as snapshot from './snapshot.tests.js'
 import * as updates from './updates.tests.js'
 import * as relativePositions from './relativePositions.tests.js'
+import * as Y from './testHelper.js'
 
 import { runTests } from 'lib0/testing'
 import { isBrowser, isNode } from 'lib0/environment'
@@ -17,6 +18,8 @@ import * as log from 'lib0/logging'
 
 if (isBrowser) {
   log.createVConsole(document.body)
+  // @ts-ignore
+  window.Y = Y
 }
 runTests({
   doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions

tests/undo-redo.tests.js

@@ -1,4 +1,4 @@
-import { init, compare, applyRandomTests, Doc } from './testHelper.js' // eslint-disable-line
+import { init, compare, applyRandomTests, Doc, UndoManager } from './testHelper.js' // eslint-disable-line
 
 import * as Y from '../src/index.js'
 import * as t from 'lib0/testing'
@@ -273,7 +273,7 @@ export const testUndoInEmbed = tc => {
  */
 export const testUndoDeleteFilter = tc => {
   /**
-   * @type {Array<Y.Map<any>>}
+   * @type {Y.Array<any>}
    */
   const array0 = /** @type {any} */ (init(tc, { users: 3 }).array0)
   const undoManager = new Y.UndoManager(array0, { deleteFilter: item => !(item instanceof Y.Item) || (item.content instanceof Y.ContentType && item.content.type._map.size === 0) })

tests/y-array.tests.js

@@ -1,10 +1,71 @@
-import { init, compare, applyRandomTests, Doc } from './testHelper.js' // eslint-disable-line
+import { init, compare, applyRandomTests, Doc, Item } from './testHelper.js' // eslint-disable-line
 
 import * as Y from '../src/index.js'
 import * as t from 'lib0/testing'
 import * as prng from 'lib0/prng'
 import * as math from 'lib0/math'
 
+/**
+ * path should be correct when moving item - see yjs#481
+ *
+ * @param {t.TestCase} tc
+ */
+export const testArrayMovePathIssue481 = tc => {
+  const { users, testConnector, array0, array1 } = init(tc, { users: 2 })
+  array0.observeDeep(events => {
+    events.forEach(event => {
+      if (event.path.length > 0) {
+        /**
+         * @type {any}
+         */
+        let target = event.currentTarget
+        event.path.forEach(p => {
+          target = target.get(p)
+        })
+        t.assert(target === event.target)
+      }
+    })
+  })
+  array0.push([
+    ['a', '1.1'],
+    ['b', '2.2'],
+    ['c', '3.1'],
+    ['d', '4.1'],
+    ['e', '5.1']
+  ].map(e => Y.Array.from(e)))
+  testConnector.flushAllMessages()
+  users[1].transact(() => {
+    array1.get(1).insert(0, ['0'])
+    array1.move(1, 0)
+  })
+  testConnector.flushAllMessages()
+  users[1].transact(() => {
+    array1.get(3).insert(0, ['1'])
+    array1.move(3, 4)
+  })
+  testConnector.flushAllMessages()
+  users[1].transact(() => {
+    array1.get(2).insert(0, ['2'])
+    array1.move(2, array1.length)
+  })
+  testConnector.flushAllMessages()
+}
+
+/**
+ * foreach has correct index - see yjs#485
+ *
+ * @param {t.TestCase} tc
+ */
+export const testArrayIndexIssue485 = tc => {
+  const doc = new Y.Doc()
+  const yarr = doc.getArray()
+  yarr.push([1, 2])
+  yarr.forEach((el, index) => {
+    t.info('index: ' + index)
+    t.assert(yarr.get(index) === el)
+  })
+}
+
 /**
  * @param {t.TestCase} tc
  */
@@ -432,6 +493,192 @@ export const testEventTargetIsSetCorrectlyOnRemote = tc => {
   compare(users)
 }
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testMove = tc => {
+  {
+    // move in uninitialized type
+    const yarr = new Y.Array()
+    yarr.insert(0, [1, 2, 3])
+    yarr.move(1, 0)
+    // @ts-ignore
+    t.compare(yarr._prelimContent, [2, 1, 3])
+  }
+  const { array0, array1, users } = init(tc, { users: 3 })
+  /**
+   * @type {any}
+   */
+  let event0 = null
+  /**
+   * @type {any}
+   */
+  let event1 = null
+  array0.observe(event => {
+    event0 = event
+  })
+  array1.observe(event => {
+    event1 = event
+  })
+  array0.insert(0, [1, 2, 3])
+  array0.move(1, 0)
+  t.compare(array0.toArray(), [2, 1, 3])
+  t.compare(event0.delta, [{ insert: [2] }, { retain: 1 }, { delete: 1 }])
+  Y.applyUpdate(users[1], Y.encodeStateAsUpdate(users[0]))
+  t.compare(array1.toArray(), [2, 1, 3])
+  t.compare(event1.delta, [{ insert: [2, 1, 3] }])
+  array0.move(0, 2)
+  t.compare(array0.toArray(), [1, 2, 3])
+  t.compare(event0.delta, [{ delete: 1 }, { retain: 1 }, { insert: [2] }])
+  compare(users)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testMove2 = tc => {
+  {
+    // move in uninitialized type
+    const yarr = new Y.Array()
+    yarr.insert(0, [1, 2])
+    yarr.move(1, 0)
+    // @ts-ignore
+    t.compare(yarr._prelimContent, [2, 1])
+  }
+  const { array0, array1, users } = init(tc, { users: 3 })
+  /**
+   * @type {any}
+   */
+  let event0 = null
+  /**
+   * @type {any}
+   */
+  let event1 = null
+  array0.observe(event => {
+    event0 = event
+  })
+  array1.observe(event => {
+    event1 = event
+  })
+  array0.insert(0, [1, 2])
+  array0.move(1, 0)
+  t.compare(array0.toArray(), [2, 1])
+  t.compare(event0.delta, [{ insert: [2] }, { retain: 1 }, { delete: 1 }])
+  Y.applyUpdate(users[1], Y.encodeStateAsUpdate(users[0]))
+  t.compare(array1.toArray(), [2, 1])
+  t.compare(event1.delta, [{ insert: [2, 1] }])
+  array0.move(0, 2)
+  t.compare(array0.toArray(), [1, 2])
+  t.compare(event0.delta, [{ delete: 1 }, { retain: 1 }, { insert: [2] }])
+  compare(users)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testMoveSingleItemRemovesPrev = tc => {
+  const ydoc = new Y.Doc()
+  const yarray = ydoc.getArray()
+  yarray.insert(0, [1, 2, 3])
+  yarray.move(0, 3)
+  t.compareArrays(yarray.toArray(), [2, 3, 1])
+  yarray.move(2, 0)
+  t.compareArrays(yarray.toArray(), [1, 2, 3])
+  let item = yarray._start
+  const items = []
+  while (item) {
+    items.push(item)
+    item = item.right
+  }
+  t.assert(items.length === 4)
+  t.assert(items.filter(item => !item.deleted).length === 3)
+}
+
+/**
+ * Check that the searchMarker is reused correctly.
+ *
+ * @param {t.TestCase} tc
+ */
+export const testListWalkerReusesSearchMarker = tc => {
+  const ydoc = new Y.Doc()
+  const yarray = ydoc.getArray()
+  const iterations = 100
+  for (let i = 0; i < iterations; i++) {
+    yarray.insert(0, [i])
+  }
+  /**
+   * @type {any}
+   */
+  let prevSm = null
+  for (let i = 0; i < iterations; i++) {
+    const v = yarray.get(i)
+    t.assert(v === iterations - i - 1)
+    t.assert(yarray._searchMarker.length <= 1)
+    const sm = yarray._searchMarker[0]
+    t.assert(prevSm == null || sm === prevSm)
+    prevSm = sm
+  }
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testMoveDeletions = tc => {
+  const ydoc = new Y.Doc()
+  const yarray = ydoc.getArray()
+  const array = yarray.toArray()
+  /**
+   * @type {any}
+   */
+  let lastDelta = []
+  yarray.observe(event => {
+    lastDelta = event.delta
+    let pos = 0
+    for (let i = 0; i < lastDelta.length; i++) {
+      const d = lastDelta[i]
+      if (d.retain != null) {
+        pos += d.retain
+      } else if (d.insert instanceof Array) {
+        array.splice(pos, 0, ...d.insert)
+        pos += d.insert.length
+      } else if (d.delete != null) {
+        array.splice(pos, d.delete)
+      }
+    }
+  })
+  yarray.insert(0, [1, 2, 3])
+  // @todo should be old-position to new-position. so that below move matches
+  yarray.move(2, 0)
+  t.compare(lastDelta, [{ insert: [3] }, { retain: 2 }, { delete: 1 }])
+  t.compareArrays(yarray.toArray(), [3, 1, 2])
+  t.compareArrays(yarray.toArray(), array)
+  ydoc.transact(tr => {
+    /** @type {Item} */ (yarray._start).delete(tr)
+  })
+  t.compare(lastDelta, [{ delete: 1 }, { retain: 2 }, { insert: [3] }])
+  t.compareArrays(yarray.toArray(), [1, 2, 3])
+  t.compareArrays(yarray.toArray(), array)
+}
+
+/**
+ * @todo
+ * @param {t.TestCase} tc
+ *
+export const testMoveCircles = tc => {
+  const { testConnector, array0, array1 } = init(tc, { users: 3 })
+  array0.insert(0, [1, 2, 3, 4])
+  testConnector.flushAllMessages()
+  array0.moveRange(0, 1, 3)
+  t.compare(array0.toArray(), [3, 1, 2, 4])
+  array1.moveRange(2, 3, 1)
+  t.compare(array1.toArray(), [1, 3, 4, 2])
+  testConnector.flushAllMessages()
+  t.assert(array0.length === 4)
+  t.assert(array0.length === array0.toArray().length)
+  t.compareArrays(array0.toArray(), array1.toArray())
+}
+*/
+
 /**
  * @param {t.TestCase} tc
  */
@@ -458,6 +705,23 @@ const getUniqueNumber = () => _uniqueNumber++
  * @type {Array<function(Doc,prng.PRNG,any):void>}
  */
 const arrayTransactions = [
+  function move (user, gen) {
+    const yarray = user.getArray('array')
+    if (yarray.length === 0) {
+      return
+    }
+    const pos = prng.int32(gen, 0, yarray.length - 1)
+    const len = 1 // prng.int32(gen, 1, math.min(3, yarray.length - pos))
+    const _newPosAdj = prng.int32(gen, 0, yarray.length - len)
+    // make sure that we don't insert in-between the moved range
+    const newPos = _newPosAdj + (_newPosAdj > pos ? len : 0)
+    const oldContent = yarray.toArray()
+    // yarray.moveRange(pos, pos + len - 1, newPos)
+    yarray.move(pos, newPos)
+    const movedValues = oldContent.splice(pos, len)
+    oldContent.splice(pos < newPos ? newPos - len : newPos, 0, ...movedValues)
+    t.compareArrays(yarray.toArray(), oldContent) // we want to make sure that fastSearch markers insert at the correct position
+  },
   function insert (user, gen) {
     const yarray = user.getArray('array')
     const uniqueNumber = getUniqueNumber()
@@ -516,95 +780,156 @@ const arrayTransactions = [
   }
 ]
 
+/**
+ * @param {Y.Doc} user
+ */
+const monitorArrayTestObject = user => {
+  /**
+   * @type {Array<any>}
+   */
+  const arr = []
+  const yarr = user.getArray('array')
+  yarr.observe(event => {
+    let currpos = 0
+    const delta = event.delta
+    for (let i = 0; i < delta.length; i++) {
+      const d = delta[i]
+      if (d.insert != null) {
+        arr.splice(currpos, 0, ...(/** @type {Array<any>} */ (d.insert)))
+        currpos += /** @type {Array<any>} */ (d.insert).length
+      } else if (d.retain != null) {
+        currpos += d.retain
+      } else {
+        arr.splice(currpos, d.delete)
+      }
+    }
+    t.compare(arr, yarr.toArray())
+  })
+  return arr
+}
+
+/**
+ * @param {{ testObjects: Array<Array<any>>, users: Array<Y.Doc> }} cmp
+ */
+const compareTestobjects = cmp => {
+  const arrs = cmp.testObjects
+  for (let i = 0; i < arrs.length; i++) {
+    const type = cmp.users[i].getArray('array')
+    t.compareArrays(arrs[i], type.toArray())
+    t.compareArrays(arrs[i], Array.from(type))
+  }
+}
+
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests6 = tc => {
-  applyRandomTests(tc, arrayTransactions, 6)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 6, monitorArrayTestObject))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGeneratingYarrayTests10 = tc => {
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 10, monitorArrayTestObject))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGeneratingYarrayTests30 = tc => {
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 30, monitorArrayTestObject))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGeneratingYarrayTests35 = tc => {
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 35, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests40 = tc => {
-  applyRandomTests(tc, arrayTransactions, 40)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 40, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests42 = tc => {
-  applyRandomTests(tc, arrayTransactions, 42)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 42, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests43 = tc => {
-  applyRandomTests(tc, arrayTransactions, 43)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 43, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests44 = tc => {
-  applyRandomTests(tc, arrayTransactions, 44)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 44, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests45 = tc => {
-  applyRandomTests(tc, arrayTransactions, 45)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 45, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests46 = tc => {
-  applyRandomTests(tc, arrayTransactions, 46)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 46, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests300 = tc => {
-  applyRandomTests(tc, arrayTransactions, 300)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 300, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests400 = tc => {
-  applyRandomTests(tc, arrayTransactions, 400)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 400, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests500 = tc => {
-  applyRandomTests(tc, arrayTransactions, 500)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 500, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests600 = tc => {
-  applyRandomTests(tc, arrayTransactions, 600)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 600, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests1000 = tc => {
-  applyRandomTests(tc, arrayTransactions, 1000)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 1000, monitorArrayTestObject))
 }
 
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYarrayTests1800 = tc => {
-  applyRandomTests(tc, arrayTransactions, 1800)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 1800, monitorArrayTestObject))
 }
 
 /**
@@ -612,7 +937,7 @@ export const testRepeatGeneratingYarrayTests1800 = tc => {
  */
 export const testRepeatGeneratingYarrayTests3000 = tc => {
   t.skip(!t.production)
-  applyRandomTests(tc, arrayTransactions, 3000)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 3000, monitorArrayTestObject))
 }
 
 /**
@@ -620,7 +945,7 @@ export const testRepeatGeneratingYarrayTests3000 = tc => {
  */
 export const testRepeatGeneratingYarrayTests5000 = tc => {
   t.skip(!t.production)
-  applyRandomTests(tc, arrayTransactions, 5000)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 5000, monitorArrayTestObject))
 }
 
 /**
@@ -628,5 +953,5 @@ export const testRepeatGeneratingYarrayTests5000 = tc => {
  */
 export const testRepeatGeneratingYarrayTests30000 = tc => {
   t.skip(!t.production)
-  applyRandomTests(tc, arrayTransactions, 30000)
+  compareTestobjects(applyRandomTests(tc, arrayTransactions, 30000, monitorArrayTestObject))
 }

tests/y-text.tests.js

@@ -327,6 +327,7 @@ export const testFormattingDeltaUnnecessaryAttributeChange = tc => {
  * @param {t.TestCase} tc
  */
 export const testInsertAndDeleteAtRandomPositions = tc => {
+  // @todo optimize to run at least as fast as previous marker approach
   const N = 100000
   const { text0 } = init(tc, { users: 1 })
   const gen = tc.prng
@@ -552,8 +553,6 @@ export const testSearchMarkerBug1 = tc => {
 }
 
 /**
- * Reported in https://github.com/yjs/yjs/pull/32
- *
  * @param {t.TestCase} tc
  */
 export const testFormattingBug = async tc => {
@@ -563,7 +562,6 @@ export const testFormattingBug = async tc => {
   text1.insert(0, '\n\n\n')
   text1.format(0, 3, { url: 'http://example.com' })
   ydoc1.getText().format(1, 1, { url: 'http://docs.yjs.dev' })
-  ydoc2.getText().format(1, 1, { url: 'http://docs.yjs.dev' })
   Y.applyUpdate(ydoc2, Y.encodeStateAsUpdate(ydoc1))
   const text2 = ydoc2.getText()
   const expectedResult = [