Move content and list iteration abstraction

src/internals.js

@@ -38,6 +38,7 @@ export * from './structs/ContentFormat.js'
 export * from './structs/ContentJSON.js'
 export * from './structs/ContentAny.js'
 export * from './structs/ContentString.js'
+export * from './structs/ContentMove.js'
 export * from './structs/ContentType.js'
 export * from './structs/Item.js'
 export * from './structs/Skip.js'

src/structs/Item.js

@@ -21,6 +21,7 @@ import {
   createID,
   readContentFormat,
   readContentType,
+  readContentMove,
   addChangedTypeToTransaction,
   UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line
 } from '../internals.js'
@@ -281,11 +282,14 @@ export class Item extends AbstractStruct {
      */
     this.parentSub = parentSub
     /**
-     * If this type's effect is reundone this type refers to the type that undid
+     * If this type is deleted: If this type's effect is reundone this type refers to the type-id that undid
      * this operation.
-     * @type {ID | null}
+     *
+     * If this item is not deleted: This property is reused by the moved prop. In this case this property refers to an Item.
+     *
+     * @type {ID | Item | null}
      */
-    this.redone = null
+    this._ref = null
     /**
      * @type {AbstractContent}
      */
@@ -295,11 +299,57 @@ export class Item extends AbstractStruct {
      * bit2: countable
      * bit3: deleted
      * bit4: mark - mark node as fast-search-marker
+     * bit5: moved - whether this item has been moved. The moved item is then referred to on the "redone" prop.
      * @type {number} byte
      */
     this.info = this.content.isCountable() ? binary.BIT2 : 0
   }
 
+  /**
+   * If this type's effect is reundone this type refers to the type-id that undid
+   * this operation.
+   *
+   * @return {ID | null}
+   */
+  get redone () {
+    return /** @type {ID | null} */ (this._ref)
+  }
+
+  /**
+   * @param {ID | null} id
+   */
+  set redone (id) {
+    this._ref = id
+  }
+
+  /**
+   * If this item has been moved, the moved property will referr to the item that moved this content.
+   *
+   * @param {Item | null} item
+   */
+  set movedBy (item) {
+    this._ref = item
+    if (item != null) {
+      this.info |= binary.BIT5
+    } else if (this.moved) {
+      this.info ^= binary.BIT5
+    }
+  }
+
+  /**
+   * @return {Item | null}
+   */
+  get movedBy () {
+    return this.moved ? /** @type {Item} */ (this._ref) : null
+  }
+
+  /**
+   * @return {boolean}
+   */
+  get moved () {
+    return (this.info & binary.BIT5) > 0
+  }
+
   /**
    * This is used to mark the item as an indexed fast-search marker
    *
@@ -371,7 +421,7 @@ export class Item extends AbstractStruct {
     // We have all missing ids, now find the items
 
     if (this.origin) {
-      this.left = getItemCleanEnd(transaction, store, this.origin)
+      this.left = getItemCleanEnd(transaction, this.origin)
       this.origin = this.left.lastId
     }
     if (this.rightOrigin) {
@@ -409,7 +459,7 @@ export class Item extends AbstractStruct {
   integrate (transaction, offset) {
     if (offset > 0) {
       this.id.clock += offset
-      this.left = getItemCleanEnd(transaction, transaction.doc.store, createID(this.id.client, this.id.clock - 1))
+      this.left = getItemCleanEnd(transaction, createID(this.id.client, this.id.clock - 1))
       this.origin = this.left.lastId
       this.content = this.content.splice(offset)
       this.length -= offset
@@ -567,8 +617,8 @@ export class Item extends AbstractStruct {
       this.id.client === right.id.client &&
       this.id.clock + this.length === right.id.clock &&
       this.deleted === right.deleted &&
-      this.redone === null &&
-      right.redone === null &&
+      this._ref === right._ref &&
+      (!this.deleted || this.redone === null) &&
       this.content.constructor === right.content.constructor &&
       this.content.mergeWith(right.content)
     ) {
@@ -613,7 +663,7 @@ export class Item extends AbstractStruct {
       this.markDeleted()
       addToDeleteSet(transaction.deleteSet, this.id.client, this.id.clock, this.length)
       addChangedTypeToTransaction(transaction, parent, this.parentSub)
-      this.content.delete(transaction)
+      this.content.delete(transaction, this)
     }
   }
 
@@ -710,7 +760,8 @@ export const contentRefs = [
   readContentType, // 7
   readContentAny, // 8
   readContentDoc, // 9
-  () => { error.unexpectedCase() } // 10 - Skip is not ItemContent
+  () => { error.unexpectedCase() }, // 10 - Skip is not ItemContent
+  readContentMove // 11
 ]
 
 /**
@@ -777,8 +828,9 @@ export class AbstractContent {
 
   /**
    * @param {Transaction} transaction
+   * @param {Item} item
    */
-  delete (transaction) {
+  delete (transaction, item) {
     throw error.methodUnimplemented()
   }
 

src/types/AbstractType.js

@@ -11,6 +11,8 @@ import {
   ContentAny,
   ContentBinary,
   getItemCleanStart,
+  ContentMove,
+  getMovedCoords,
   ContentDoc, YText, YArray, UpdateEncoderV1, UpdateEncoderV2, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
 } from '../internals.js'
 
@@ -395,6 +397,131 @@ export class AbstractType {
   toJSON () {}
 }
 
+export class ListPosition {
+  /**
+   * @param {AbstractType<any>} type
+   * @param {Transaction} tr
+   */
+  constructor (type, tr) {
+    this.type = type
+    /**
+     * Current index-position
+     */
+    this.index = 0
+    /**
+     * Relative position to the current item (if item.content.length > 1)
+     */
+    this.rel = 0
+    /**
+     * This refers to the current right item, unless reachedEnd is true. Then it refers to the left item.
+     */
+    this.item = type._start
+    this.reachedEnd = type._start === null
+    /**
+     * @type {Item | null}
+     */
+    this.currMove = null
+    /**
+     * @type {Item | null}
+     */
+    this.currMoveEnd = null
+    /**
+     * @type {Array<{ end: Item | null, move: Item }>}
+     */
+    this.movedStack = []
+    this.tr = tr
+  }
+
+  /**
+   * @param {number} len
+   */
+  forward (len) {
+    let item = this.item
+    this.index += len
+    if (this.rel) {
+      len += this.rel
+      this.rel = 0
+    }
+    while (item && !this.reachedEnd && (len > 0 || (len === 0 && (!item.countable || item.deleted)))) {
+      if (item.countable && !item.deleted && item.movedBy === this.currMove) {
+        len -= item.length
+        if (len <= 0) {
+          this.rel = -len
+          break
+        }
+      } else if (item.content.constructor === ContentMove) {
+        if (this.currMove) {
+          this.movedStack.push({ end: this.currMoveEnd, move: this.currMove })
+        }
+        const { start, end } = getMovedCoords(item.content, this.tr)
+        this.currMove = item
+        this.currMoveEnd = end
+        this.item = start
+        continue
+      }
+      if (item === this.currMoveEnd) {
+        this.item = this.currMove // we iterate to the right after the current condition
+        const { end, move } = this.movedStack.pop() || { end: null, move: null }
+        this.currMove = move
+        this.currMoveEnd = end
+      }
+      if (item.right) {
+        item = item.right
+      } else {
+        this.reachedEnd = true
+      }
+    }
+    this.index -= len
+    this.item = item
+  }
+
+  /**
+   * @param {number} len
+   */
+  slice (len) {
+    const result = []
+    while (len > 0 && !this.reachedEnd) {
+      while (this.item && this.item.countable && !this.reachedEnd && len > 0) {
+        if (!this.item.deleted) {
+          const content = this.item.content.getContent()
+          const slicedContent = content.length <= len || this.rel > 0 ? content : content.slice(this.rel, len)
+          len -= slicedContent.length
+          result.push(...slicedContent)
+          if (content.length !== slicedContent.length) {
+            if (this.rel + slicedContent.length === content.length) {
+              this.rel = 0
+            } else {
+              this.rel += slicedContent.length
+              continue // do not iterate to item.right
+            }
+          }
+        }
+        if (this.item.right) {
+          this.item = this.item.right
+        } else {
+          this.reachedEnd = true
+        }
+      }
+      if (this.item && !this.reachedEnd && len > 0) {
+        this.forward(0)
+      }
+    }
+    return result
+  }
+
+  [Symbol.iterator] () {
+    return this
+  }
+
+  next () {
+    const [value] = this.slice(1)
+    return {
+      done: value == null,
+      value: value
+    }
+  }
+}
+
 /**
  * @param {AbstractType<any>} type
  * @param {number} start

src/types/YArray.js

@@ -6,7 +6,6 @@ import {
   YEvent,
   AbstractType,
   typeListGet,
-  typeListToArray,
   typeListForEach,
   typeListCreateIterator,
   typeListInsertGenerics,
@@ -15,6 +14,7 @@ import {
   YArrayRefID,
   callTypeObservers,
   transact,
+  ListPosition,
   ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, Transaction, Item // eslint-disable-line
 } from '../internals.js'
 import { typeListSlice } from './AbstractType.js'
@@ -188,7 +188,9 @@ export class YArray extends AbstractType {
    * @return {Array<T>}
    */
   toArray () {
-    return typeListToArray(this)
+    return transact(/** @type {Doc} */ (this.doc), tr =>
+      new ListPosition(this, tr).slice(this.length)
+    )
   }
 
   /**

src/utils/StructStore.js

@@ -199,19 +199,18 @@ export const getItemCleanStart = (transaction, id) => {
  * Expects that id is actually in store. This function throws or is an infinite loop otherwise.
  *
  * @param {Transaction} transaction
- * @param {StructStore} store
  * @param {ID} id
  * @return {Item}
  *
  * @private
  * @function
  */
-export const getItemCleanEnd = (transaction, store, id) => {
+export const getItemCleanEnd = (transaction, id) => {
   /**
    * @type {Array<Item>}
    */
   // @ts-ignore
-  const structs = store.clients.get(id.client)
+  const structs = transaction.doc.store.clients.get(id.client)
   const index = findIndexSS(structs, id.clock)
   const struct = structs[index]
   if (id.clock !== struct.id.clock + struct.length - 1 && struct.constructor !== GC) {

src/utils/Transaction.js

@@ -377,9 +377,12 @@ const cleanupTransactions = (transactionCleanups, i) => {
 /**
  * Implements the functionality of `y.transact(()=>{..})`
  *
+ * @template T
+ *
  * @param {Doc} doc
- * @param {function(Transaction):void} f
+ * @param {function(Transaction):T} f
  * @param {any} [origin=true]
+ * @return {T}
  *
  * @function
  */
@@ -395,8 +398,9 @@ export const transact = (doc, f, origin = null, local = true) => {
     }
     doc.emit('beforeTransaction', [doc._transaction, doc])
   }
+  let res
   try {
-    f(doc._transaction)
+    res = f(doc._transaction)
   } finally {
     if (initialCall && transactionCleanups[0] === doc._transaction) {
       // The first transaction ended, now process observer calls.
@@ -410,4 +414,5 @@ export const transact = (doc, f, origin = null, local = true) => {
       cleanupTransactions(transactionCleanups, 0)
     }
   }
+  return res
 }

tests/encoding.tests.js

@@ -12,6 +12,7 @@ import {
   readContentFormat,
   readContentAny,
   readContentDoc,
+  readContentMove,
   Doc,
   PermanentUserData,
   encodeStateAsUpdate,
@@ -24,7 +25,8 @@ import * as Y from '../src/index.js'
  * @param {t.TestCase} tc
  */
 export const testStructReferences = tc => {
-  t.assert(contentRefs.length === 11)
+  t.assert(contentRefs.length === 12)
+  // contentRefs[0] is reserved for GC
   t.assert(contentRefs[1] === readContentDeleted)
   t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json?
   t.assert(contentRefs[3] === readContentBinary)
@@ -35,6 +37,7 @@ export const testStructReferences = tc => {
   t.assert(contentRefs[8] === readContentAny)
   t.assert(contentRefs[9] === readContentDoc)
   // contentRefs[10] is reserved for Skip structs
+  t.assert(contentRefs[11] === readContentMove)
 }
 
 /**