diff --git a/package-lock.json b/package-lock.json index 6d8ad924..7a733470 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.5.22", + "version": "13.6.0-2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index d4a69b1f..aa070c27 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.5.22", + "version": "13.6.0-2", "description": "Shared Editing Library", "main": "./dist/yjs.cjs", "module": "./dist/yjs.mjs", @@ -19,7 +19,7 @@ "lint": "markdownlint README.md && standard && tsc", "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true", "serve-docs": "npm run docs && http-server ./docs/", - "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repetition-time 1000 && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs", + "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs", "debug": "concurrently 'http-server -o test.html' 'npm run watch'", "trace-deopt": "clear && rollup -c && node --trace-deopt dist/test.cjs", "trace-opt": "clear && rollup -c && node --trace-opt dist/test.cjs", diff --git a/src/internals.js b/src/internals.js index bc386f0a..e84e7ad2 100644 --- a/src/internals.js +++ b/src/internals.js @@ -8,6 +8,7 @@ export * from './utils/encoding.js' export * from './utils/EventHandler.js' export * from './utils/ID.js' export * from './utils/isParentOf.js' +export * from './utils/ListIterator.js' export * from './utils/logging.js' export * from './utils/PermanentUserData.js' export * from './utils/RelativePosition.js' @@ -38,6 +39,7 @@ export * from './structs/ContentFormat.js' export * from './structs/ContentJSON.js' export * from './structs/ContentAny.js' export * from './structs/ContentString.js' +export * from './structs/ContentMove.js' export * from './structs/ContentType.js' export * from './structs/Item.js' export * from './structs/Skip.js' diff --git a/src/structs/ContentMove.js b/src/structs/ContentMove.js new file mode 100644 index 00000000..a196ac02 --- /dev/null +++ b/src/structs/ContentMove.js @@ -0,0 +1,286 @@ + +import * as error from 'lib0/error' +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import * as math from 'lib0/math' +import { + AbstractType, ContentType, RelativePosition, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Transaction, Item, StructStore, getItem, getItemCleanStart, getItemCleanEnd // eslint-disable-line +} from '../internals.js' +import { decodeRelativePosition, encodeRelativePosition } from 'yjs' + +/** + * @param {ContentMove} moved + * @param {Transaction} tr + * @return {{ start: Item, end: Item | null }} $start (inclusive) is the beginning and $end (exclusive) is the end of the moved area + */ +export const getMovedCoords = (moved, tr) => { + let start // this (inclusive) is the beginning of the moved area + let end // this (exclusive) is the first item after start that is not part of the moved area + if (moved.start.item) { + if (moved.start.assoc < 0) { + start = getItemCleanEnd(tr, moved.start.item) + start = start.right + } else { + start = getItemCleanStart(tr, moved.start.item) + } + } else if (moved.start.tname != null) { + start = tr.doc.get(moved.start.tname)._start + } else if (moved.start.type) { + start = /** @type {ContentType} */ (getItem(tr.doc.store, moved.start.type).content).type._start + } else { + error.unexpectedCase() + } + if (moved.end.item) { + if (moved.end.assoc < 0) { + end = getItemCleanEnd(tr, moved.end.item) + end = end.right + } else { + end = getItemCleanStart(tr, moved.end.item) + } + } else { + end = null + } + return { start: /** @type {Item} */ (start), end } +} + +/** + * @todo remove this if not needed + * + * @param {ContentMove} moved + * @param {Item} movedItem + * @param {Transaction} tr + * @param {function(Item):void} cb + */ +export const iterateMoved = (moved, movedItem, tr, cb) => { + /** + * @type {{ start: Item | null, end: Item | null }} + */ + let { start, end } = getMovedCoords(moved, tr) + while (start !== end && start != null) { + if (!start.deleted) { + if (start.moved === movedItem) { + if (start.content.constructor === ContentMove) { + iterateMoved(start.content, start, tr, cb) + } else { + cb(start) + } + } + } + start = start.right + } +} + +/** + * @param {ContentMove} moved + * @param {Item} movedItem + * @param {Set} trackedMovedItems + * @param {Transaction} tr + * @return {boolean} true if there is a loop + */ +export const findMoveLoop = (moved, movedItem, trackedMovedItems, tr) => { + if (trackedMovedItems.has(movedItem)) { + return true + } + trackedMovedItems.add(movedItem) + /** + * @type {{ start: Item | null, end: Item | null }} + */ + let { start, end } = getMovedCoords(moved, tr) + while (start !== end && start != null) { + if (start.deleted && start.moved === movedItem && start.content.constructor === ContentMove) { + if (findMoveLoop(start.content, start, trackedMovedItems, tr)) { + return true + } + } + start = start.right + } + return false +} + +/** + * @private + */ +export class ContentMove { + /** + * @param {RelativePosition} start + * @param {RelativePosition} end + * @param {number} priority if we want to move content that is already moved, we need to assign a higher priority to this move operation. + */ + constructor (start, end, priority) { + this.start = start + this.end = end + this.priority = priority + /** + * We store which Items+ContentMove we override. Once we delete + * this ContentMove, we need to re-integrate the overridden items. + * + * This representation can be improved if we ever run into memory issues because of too many overrides. + * Ideally, we should probably just re-iterate the document and re-integrate all moved items. + * This is fast enough and reduces memory footprint significantly. + * + * @type {Set} + */ + this.overrides = new Set() + } + + /** + * @return {number} + */ + getLength () { + return 1 + } + + /** + * @return {Array} + */ + getContent () { + return [null] + } + + /** + * @return {boolean} + */ + isCountable () { + return false + } + + /** + * @return {ContentMove} + */ + copy () { + return new ContentMove(this.start, this.end, this.priority) + } + + /** + * @param {number} offset + * @return {ContentMove} + */ + splice (offset) { + return this + } + + /** + * @param {ContentMove} right + * @return {boolean} + */ + mergeWith (right) { + return false + } + + /** + * @param {Transaction} transaction + * @param {Item} item + */ + integrate (transaction, item) { + /** @type {AbstractType} */ (item.parent)._searchMarker = [] + /** + * @type {{ start: Item | null, end: Item | null }} + */ + let { start, end } = getMovedCoords(this, transaction) + let maxPriority = 0 + // If this ContentMove was created locally, we set prio = -1. This indicates + // that we want to set prio to the current prio-maximum of the moved range. + const adaptPriority = this.priority < 0 + while (start !== end && start != null) { + if (!start.deleted) { + const currMoved = start.moved + const nextPrio = currMoved ? /** @type {ContentMove} */ (currMoved.content).priority : -1 + if (currMoved === null || adaptPriority || nextPrio < this.priority || currMoved.id.client < item.id.client || (currMoved.id.client === item.id.client && currMoved.id.clock < item.id.clock)) { + if (currMoved !== null) { + this.overrides.add(currMoved) + } + maxPriority = math.max(maxPriority, nextPrio) + // was already moved + if (start.moved && !transaction.prevMoved.has(start)) { + // we need to know which item previously moved an item + transaction.prevMoved.set(start, start.moved) + } + start.moved = item + } else { + /** @type {ContentMove} */ (currMoved.content).overrides.add(item) + } + } + start = start.right + } + if (adaptPriority) { + this.priority = maxPriority + 1 + } + } + + /** + * @param {Transaction} transaction + * @param {Item} item + */ + delete (transaction, item) { + /** + * @type {{ start: Item | null, end: Item | null }} + */ + let { start, end } = getMovedCoords(this, transaction) + while (start !== end && start != null) { + if (start.moved === item) { + start.moved = null + } + start = start.right + } + /** + * @param {Item} reIntegrateItem + */ + const reIntegrate = reIntegrateItem => { + const content = /** @type {ContentMove} */ (reIntegrateItem.content) + if (reIntegrateItem.deleted) { + // potentially we can integrate the items that reIntegrateItem overrides + content.overrides.forEach(reIntegrate) + } else { + content.integrate(transaction, reIntegrateItem) + } + } + this.overrides.forEach(reIntegrate) + } + + /** + * @param {StructStore} store + */ + gc (store) {} + + /** + * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder + * @param {number} offset + */ + write (encoder, offset) { + const isCollapsed = this.isCollapsed() + encoding.writeUint8(encoder.restEncoder, isCollapsed ? 1 : 0) + encoder.writeBuf(encodeRelativePosition(this.start)) + if (!isCollapsed) { + encoder.writeBuf(encodeRelativePosition(this.end)) + } + encoding.writeVarUint(encoder.restEncoder, this.priority) + } + + /** + * @return {number} + */ + getRef () { + return 11 + } + + isCollapsed () { + return this.start.item === this.end.item && this.start.item !== null + } +} + +/** + * @private + * @todo use binary encoding option for start & end relpos's + * + * @param {UpdateDecoderV1 | UpdateDecoderV2} decoder + * @return {ContentMove} + */ +export const readContentMove = decoder => { + const isCollapsed = decoding.readUint8(decoder.restDecoder) === 1 + const start = decodeRelativePosition(decoder.readBuf()) + const end = isCollapsed ? start.clone() : decodeRelativePosition(decoder.readBuf()) + if (isCollapsed) { + end.assoc = -1 + } + return new ContentMove(start, end, decoding.readVarUint(decoder.restDecoder)) +} diff --git a/src/structs/Item.js b/src/structs/Item.js index 656f5e5b..44e9aeef 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -21,12 +21,14 @@ import { createID, readContentFormat, readContentType, + readContentMove, addChangedTypeToTransaction, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line } from '../internals.js' import * as error from 'lib0/error' import * as binary from 'lib0/binary' +import { ContentMove } from './ContentMove.js' /** * @todo This should return several items @@ -116,6 +118,12 @@ export const splitItem = (transaction, leftItem, diff) => { /** @type {AbstractType} */ (rightItem.parent)._map.set(rightItem.parentSub, rightItem) } leftItem.length = diff + if (leftItem.moved) { + const m = transaction.prevMoved.get(leftItem) + if (m) { + transaction.prevMoved.set(rightItem, m) + } + } return rightItem } @@ -281,11 +289,18 @@ export class Item extends AbstractStruct { */ this.parentSub = parentSub /** - * If this type's effect is reundone this type refers to the type that undid + * If this type's effect is reundone this type refers to the type-id that undid * this operation. + * * @type {ID | null} */ this.redone = null + /** + * This property is reused by the moved prop. In this case this property refers to an Item. + * + * @type {Item | null} + */ + this.moved = null /** * @type {AbstractContent} */ @@ -367,11 +382,21 @@ export class Item extends AbstractStruct { if (this.parent && this.parent.constructor === ID && this.id.client !== this.parent.client && this.parent.clock >= getState(store, this.parent.client)) { return this.parent.client } + if (this.content.constructor === ContentMove) { + const c = /** @type {ContentMove} */ (this.content) + const start = c.start.item + const end = c.isCollapsed() ? null : c.end.item + if (start && start.clock >= getState(store, start.client)) { + return start.client + } + if (end && end.clock >= getState(store, end.client)) { + return end.client + } + } // We have all missing ids, now find the items - if (this.origin) { - this.left = getItemCleanEnd(transaction, store, this.origin) + this.left = getItemCleanEnd(transaction, this.origin) this.origin = this.left.lastId } if (this.rightOrigin) { @@ -399,6 +424,7 @@ export class Item extends AbstractStruct { this.parent = /** @type {ContentType} */ (parentItem.content).type } } + return null } @@ -409,7 +435,7 @@ export class Item extends AbstractStruct { integrate (transaction, offset) { if (offset > 0) { this.id.clock += offset - this.left = getItemCleanEnd(transaction, transaction.doc.store, createID(this.id.client, this.id.clock - 1)) + this.left = getItemCleanEnd(transaction, createID(this.id.client, this.id.clock - 1)) this.origin = this.left.lastId this.content = this.content.splice(offset) this.length -= offset @@ -569,21 +595,22 @@ export class Item extends AbstractStruct { this.deleted === right.deleted && this.redone === null && right.redone === null && + this.moved === right.moved && this.content.constructor === right.content.constructor && this.content.mergeWith(right.content) ) { - const searchMarker = /** @type {AbstractType} */ (this.parent)._searchMarker - if (searchMarker) { - searchMarker.forEach(marker => { - if (marker.p === right) { - // right is going to be "forgotten" so we need to update the marker - marker.p = this - // adjust marker index - if (!this.deleted && this.countable) { - marker.index -= this.length + if (right.marker) { + // Right will be "forgotten", so we delete all + // search markers that reference right. + const searchMarker = /** @type {AbstractType} */ (this.parent)._searchMarker + if (searchMarker) { + for (let i = searchMarker.length - 1; i >= 0; i--) { + if (searchMarker[i].nextItem === right) { + // @todo do something more efficient than splicing.. + searchMarker.splice(i, 1) } } - }) + } } if (right.keep) { this.keep = true @@ -613,7 +640,7 @@ export class Item extends AbstractStruct { this.markDeleted() addToDeleteSet(transaction.deleteSet, this.id.client, this.id.clock, this.length) addChangedTypeToTransaction(transaction, parent, this.parentSub) - this.content.delete(transaction) + this.content.delete(transaction, this) } } @@ -625,6 +652,7 @@ export class Item extends AbstractStruct { if (!this.deleted) { throw error.unexpectedCase() } + this.moved = null this.content.gc(store) if (parentGCd) { replaceStruct(store, this, new GC(this.id, this.length)) @@ -710,7 +738,8 @@ export const contentRefs = [ readContentType, // 7 readContentAny, // 8 readContentDoc, // 9 - () => { error.unexpectedCase() } // 10 - Skip is not ItemContent + () => { error.unexpectedCase() }, // 10 - Skip is not ItemContent + readContentMove // 11 ] /** @@ -777,8 +806,9 @@ export class AbstractContent { /** * @param {Transaction} transaction + * @param {Item} item */ - delete (transaction) { + delete (transaction, item) { throw error.methodUnimplemented() } diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 43c6127d..9e1a877c 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -10,7 +10,7 @@ import { createID, ContentAny, ContentBinary, - getItemCleanStart, + ListIterator, ContentDoc, YText, YArray, UpdateEncoderV1, UpdateEncoderV2, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line } from '../internals.js' @@ -21,67 +21,6 @@ import * as math from 'lib0/math' const maxSearchMarker = 80 -/** - * A unique timestamp that identifies each marker. - * - * Time is relative,.. this is more like an ever-increasing clock. - * - * @type {number} - */ -let globalSearchMarkerTimestamp = 0 - -export class ArraySearchMarker { - /** - * @param {Item} p - * @param {number} index - */ - constructor (p, index) { - p.marker = true - this.p = p - this.index = index - this.timestamp = globalSearchMarkerTimestamp++ - } -} - -/** - * @param {ArraySearchMarker} marker - */ -const refreshMarkerTimestamp = marker => { marker.timestamp = globalSearchMarkerTimestamp++ } - -/** - * This is rather complex so this function is the only thing that should overwrite a marker - * - * @param {ArraySearchMarker} marker - * @param {Item} p - * @param {number} index - */ -const overwriteMarker = (marker, p, index) => { - marker.p.marker = false - marker.p = p - p.marker = true - marker.index = index - marker.timestamp = globalSearchMarkerTimestamp++ -} - -/** - * @param {Array} searchMarker - * @param {Item} p - * @param {number} index - */ -const markPosition = (searchMarker, p, index) => { - if (searchMarker.length >= maxSearchMarker) { - // override oldest marker (we don't want to create more objects) - const marker = searchMarker.reduce((a, b) => a.timestamp < b.timestamp ? a : b) - overwriteMarker(marker, p, index) - return marker - } else { - // create new marker - const pm = new ArraySearchMarker(p, index) - searchMarker.push(pm) - return pm - } -} - /** * Search marker help us to find positions in the associative array faster. * @@ -89,82 +28,69 @@ const markPosition = (searchMarker, p, index) => { * * A maximum of `maxSearchMarker` objects are created. * - * This function always returns a refreshed marker (updated timestamp) - * + * @template T + * @param {Transaction} tr * @param {AbstractType} yarray * @param {number} index + * @param {function(ListIterator):T} f */ -export const findMarker = (yarray, index) => { - if (yarray._start === null || index === 0 || yarray._searchMarker === null) { - return null +export const useSearchMarker = (tr, yarray, index, f) => { + const searchMarker = yarray._searchMarker + if (searchMarker === null || yarray._start === null || index < 5) { + return f(new ListIterator(yarray).forward(tr, index)) } - const marker = yarray._searchMarker.length === 0 ? null : yarray._searchMarker.reduce((a, b) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b) - let p = yarray._start - let pindex = 0 - if (marker !== null) { - p = marker.p - pindex = marker.index - refreshMarkerTimestamp(marker) // we used it, we might need to use it again + if (searchMarker.length === 0) { + const sm = new ListIterator(yarray).forward(tr, index) + searchMarker.push(sm) + if (sm.nextItem) sm.nextItem.marker = true } - // iterate to right if possible - while (p.right !== null && pindex < index) { - if (!p.deleted && p.countable) { - if (index < pindex + p.length) { - break - } - pindex += p.length - } - p = p.right + const sm = searchMarker.reduce( + (a, b, arrayIndex) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b + ) + const newIsCheaper = math.abs(sm.index - index) > index + const createFreshMarker = searchMarker.length < maxSearchMarker && (math.abs(sm.index - index) > 5 || newIsCheaper) + const fsm = createFreshMarker ? (newIsCheaper ? new ListIterator(yarray) : sm.clone()) : sm + const prevItem = /** @type {Item} */ (sm.nextItem) + if (createFreshMarker) { + searchMarker.push(fsm) } - // iterate to left if necessary (might be that pindex > index) - while (p.left !== null && pindex > index) { - p = p.left - if (!p.deleted && p.countable) { - pindex -= p.length - } - } - // we want to make sure that p can't be merged with left, because that would screw up everything - // in that cas just return what we have (it is most likely the best marker anyway) - // iterate to left until p can't be merged with left - while (p.left !== null && p.left.id.client === p.id.client && p.left.id.clock + p.left.length === p.id.clock) { - p = p.left - if (!p.deleted && p.countable) { - pindex -= p.length - } - } - - // @todo remove! - // assure position - // { - // let start = yarray._start - // let pos = 0 - // while (start !== p) { - // if (!start.deleted && start.countable) { - // pos += start.length - // } - // start = /** @type {Item} */ (start.right) - // } - // if (pos !== pindex) { - // debugger - // throw new Error('Gotcha position fail!') - // } - // } - // if (marker) { - // if (window.lengthes == null) { - // window.lengthes = [] - // window.getLengthes = () => window.lengthes.sort((a, b) => a - b) - // } - // window.lengthes.push(marker.index - pindex) - // console.log('distance', marker.index - pindex, 'len', p && p.parent.length) - // } - if (marker !== null && math.abs(marker.index - pindex) < /** @type {YText|YArray} */ (p.parent).length / maxSearchMarker) { - // adjust existing marker - overwriteMarker(marker, p, pindex) - return marker + const diff = fsm.index - index + if (diff > 0) { + fsm.backward(tr, diff) } else { - // create new marker - return markPosition(yarray._searchMarker, p, pindex) + fsm.forward(tr, -diff) } + // @todo remove this tests + /* + const otherTesting = new ListIterator(yarray) + otherTesting.forward(tr, index) + if (otherTesting.nextItem !== fsm.nextItem || otherTesting.index !== fsm.index || otherTesting.reachedEnd !== fsm.reachedEnd) { + throw new Error('udtirane') + } + */ + const result = f(fsm) + if (fsm.reachedEnd) { + fsm.reachedEnd = false + const nextItem = /** @type {Item} */ (fsm.nextItem) + if (nextItem.countable && !nextItem.deleted) { + fsm.index -= nextItem.length + } + fsm.rel = 0 + } + if (!createFreshMarker) { + // reused old marker and we moved to a different position + prevItem.marker = false + } + const fsmItem = fsm.nextItem + if (fsmItem) { + if (fsmItem.marker) { + // already marked, forget current iterator + searchMarker.splice(searchMarker.findIndex(m => m === fsm), 1) + } else { + fsmItem.marker = true + } + } + return result } /** @@ -172,39 +98,25 @@ export const findMarker = (yarray, index) => { * * This should be called before doing a deletion! * - * @param {Array} searchMarker + * @param {Array} searchMarker * @param {number} index * @param {number} len If insertion, len is positive. If deletion, len is negative. + * @param {ListIterator|null} origSearchMarker Do not update this searchmarker because it is the one we used to manipulate. @todo !=null for improved perf in ytext */ -export const updateMarkerChanges = (searchMarker, index, len) => { +export const updateMarkerChanges = (searchMarker, index, len, origSearchMarker) => { for (let i = searchMarker.length - 1; i >= 0; i--) { - const m = searchMarker[i] - if (len > 0) { - /** - * @type {Item|null} - */ - let p = m.p - p.marker = false - // Ideally we just want to do a simple position comparison, but this will only work if - // search markers don't point to deleted items for formats. - // Iterate marker to prev undeleted countable position so we know what to do when updating a position - while (p && (p.deleted || !p.countable)) { - p = p.left - if (p && !p.deleted && p.countable) { - // adjust position. the loop should break now - m.index -= p.length - } - } - if (p === null || p.marker === true) { - // remove search marker if updated position is null or if position is already marked + const marker = searchMarker[i] + if (marker !== origSearchMarker) { + if (len > 0 && index === marker.index) { + // inserting at a marked position deletes the marked position because we can't do a simple transformation + // (we don't know whether to insert directly before or directly after the position) searchMarker.splice(i, 1) + if (marker.nextItem) marker.nextItem.marker = false continue } - m.p = p - p.marker = true - } - if (index < m.index || (len > 0 && index === m.index)) { // a simple index <= m.index check would actually suffice - m.index = math.max(index, m.index + len) + if (index < marker.index) { // a simple index <= m.index check would actually suffice + marker.index = math.max(index, marker.index + len) + } } } } @@ -282,9 +194,16 @@ export class AbstractType { */ this._dEH = createEventHandler() /** - * @type {null | Array} + * @type {null | Array} */ this._searchMarker = null + /** + * You can store custom stuff here. + * This might be useful to associate your application state to this shared type. + * + * @type {Map} + */ + this.meta = new Map() } /** @@ -594,31 +513,6 @@ export const typeListForEachSnapshot = (type, f, snapshot) => { } } -/** - * @param {AbstractType} type - * @param {number} index - * @return {any} - * - * @private - * @function - */ -export const typeListGet = (type, index) => { - const marker = findMarker(type, index) - let n = type._start - if (marker !== null) { - n = marker.p - index -= marker.index - } - for (; n !== null; n = n.right) { - if (!n.deleted && n.countable) { - if (index < n.length) { - return n.content.getContent()[index] - } - index -= n.length - } - } -} - /** * @param {Transaction} transaction * @param {AbstractType} parent @@ -683,105 +577,6 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem, packJsonContent() } -const lengthExceeded = error.create('Length exceeded!') - -/** - * @param {Transaction} transaction - * @param {AbstractType} parent - * @param {number} index - * @param {Array|Array|number|null|string|Uint8Array>} content - * - * @private - * @function - */ -export const typeListInsertGenerics = (transaction, parent, index, content) => { - if (index > parent._length) { - throw lengthExceeded - } - if (index === 0) { - if (parent._searchMarker) { - updateMarkerChanges(parent._searchMarker, index, content.length) - } - return typeListInsertGenericsAfter(transaction, parent, null, content) - } - const startIndex = index - const marker = findMarker(parent, index) - let n = parent._start - if (marker !== null) { - n = marker.p - index -= marker.index - // we need to iterate one to the left so that the algorithm works - if (index === 0) { - // @todo refactor this as it actually doesn't consider formats - n = n.prev // important! get the left undeleted item so that we can actually decrease index - index += (n && n.countable && !n.deleted) ? n.length : 0 - } - } - for (; n !== null; n = n.right) { - if (!n.deleted && n.countable) { - if (index <= n.length) { - if (index < n.length) { - // insert in-between - getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index)) - } - break - } - index -= n.length - } - } - if (parent._searchMarker) { - updateMarkerChanges(parent._searchMarker, startIndex, content.length) - } - return typeListInsertGenericsAfter(transaction, parent, n, content) -} - -/** - * @param {Transaction} transaction - * @param {AbstractType} parent - * @param {number} index - * @param {number} length - * - * @private - * @function - */ -export const typeListDelete = (transaction, parent, index, length) => { - if (length === 0) { return } - const startIndex = index - const startLength = length - const marker = findMarker(parent, index) - let n = parent._start - if (marker !== null) { - n = marker.p - index -= marker.index - } - // compute the first item to be deleted - for (; n !== null && index > 0; n = n.right) { - if (!n.deleted && n.countable) { - if (index < n.length) { - getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index)) - } - index -= n.length - } - } - // delete all items until done - while (length > 0 && n !== null) { - if (!n.deleted) { - if (length < n.length) { - getItemCleanStart(transaction, createID(n.id.client, n.id.clock + length)) - } - n.delete(transaction) - length -= n.length - } - n = n.right - } - if (length > 0) { - throw lengthExceeded - } - if (parent._searchMarker) { - updateMarkerChanges(parent._searchMarker, startIndex, -startLength + length /* in case we remove the above exception */) - } -} - /** * @param {Transaction} transaction * @param {AbstractType} parent diff --git a/src/types/YArray.js b/src/types/YArray.js index 7c210fa6..e04f5793 100644 --- a/src/types/YArray.js +++ b/src/types/YArray.js @@ -5,19 +5,14 @@ import { YEvent, AbstractType, - typeListGet, - typeListToArray, - typeListForEach, - typeListCreateIterator, - typeListInsertGenerics, - typeListDelete, - typeListMap, YArrayRefID, callTypeObservers, transact, - ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, Transaction, Item // eslint-disable-line + ListIterator, + useSearchMarker, + createRelativePositionFromTypeIndex, + UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, Transaction, Item // eslint-disable-line } from '../internals.js' -import { typeListSlice } from './AbstractType.js' /** * Event that describes the changes on a YArray @@ -49,7 +44,7 @@ export class YArray extends AbstractType { */ this._prelimContent = [] /** - * @type {Array} + * @type {Array} */ this._searchMarker = [] } @@ -129,12 +124,70 @@ export class YArray extends AbstractType { * @param {Array} content The array of content */ insert (index, content) { + if (content.length > 0) { + if (this.doc !== null) { + transact(this.doc, transaction => { + useSearchMarker(transaction, this, index, walker => + walker.insertArrayValue(transaction, content) + ) + }) + } else { + /** @type {Array} */ (this._prelimContent).splice(index, 0, ...content) + } + } + } + + /** + * Move a single item from $index to $target. + * + * @todo make sure that collapsed moves are removed (i.e. when moving the same item twice) + * + * @param {number} index + * @param {number} target + */ + move (index, target) { + if (index === target || index + 1 === target || index >= this.length) { + // It doesn't make sense to move a range into the same range (it's basically a no-op). + return + } if (this.doc !== null) { transact(this.doc, transaction => { - typeListInsertGenerics(transaction, this, index, content) + const left = createRelativePositionFromTypeIndex(this, index, 1) + const right = left.clone() + right.assoc = -1 + useSearchMarker(transaction, this, target, walker => { + walker.insertMove(transaction, left, right) + }) }) } else { - /** @type {Array} */ (this._prelimContent).splice(index, 0, ...content) + const content = /** @type {Array} */ (this._prelimContent).splice(index, 1) + ;/** @type {Array} */ (this._prelimContent).splice(target, 0, ...content) + } + } + + /** + * @param {number} start Inclusive move-start + * @param {number} end Inclusive move-end + * @param {number} target + * @param {number} assocStart >=0 if start should be associated with the right character. See relative-position assoc parameter. + * @param {number} assocEnd >= 0 if end should be associated with the right character. + */ + moveRange (start, end, target, assocStart = 1, assocEnd = -1) { + if (start <= target && target <= end) { + // It doesn't make sense to move a range into the same range (it's basically a no-op). + return + } + if (this.doc !== null) { + transact(this.doc, transaction => { + const left = createRelativePositionFromTypeIndex(this, start, assocStart) + const right = createRelativePositionFromTypeIndex(this, end + 1, assocEnd) + useSearchMarker(transaction, this, target, walker => { + walker.insertMove(transaction, left, right) + }) + }) + } else { + const content = /** @type {Array} */ (this._prelimContent).splice(start, end - start + 1) + ;/** @type {Array} */ (this._prelimContent).splice(target, 0, ...content) } } @@ -165,7 +218,9 @@ export class YArray extends AbstractType { delete (index, length = 1) { if (this.doc !== null) { transact(this.doc, transaction => { - typeListDelete(transaction, this, index, length) + useSearchMarker(transaction, this, index, walker => + walker.delete(transaction, length) + ) }) } else { /** @type {Array} */ (this._prelimContent).splice(index, length) @@ -179,7 +234,11 @@ export class YArray extends AbstractType { * @return {T} */ get (index) { - return typeListGet(this, index) + return transact(/** @type {Doc} */ (this.doc), transaction => + useSearchMarker(transaction, this, index, walker => + walker.slice(transaction, 1)[0] + ) + ) } /** @@ -188,7 +247,9 @@ export class YArray extends AbstractType { * @return {Array} */ toArray () { - return typeListToArray(this) + return transact(/** @type {Doc} */ (this.doc), tr => + new ListIterator(this).slice(tr, this.length) + ) } /** @@ -199,7 +260,11 @@ export class YArray extends AbstractType { * @return {Array} */ slice (start = 0, end = this.length) { - return typeListSlice(this, start, end) + return transact(/** @type {Doc} */ (this.doc), transaction => + useSearchMarker(transaction, this, start, walker => + walker.slice(transaction, end < 0 ? this.length + end - start : end - start) + ) + ) } /** @@ -221,7 +286,9 @@ export class YArray extends AbstractType { * callback function */ map (f) { - return typeListMap(this, /** @type {any} */ (f)) + return transact(/** @type {Doc} */ (this.doc), tr => + new ListIterator(this).map(tr, f) + ) } /** @@ -230,14 +297,16 @@ export class YArray extends AbstractType { * @param {function(T,number,YArray):void} f A function to execute on every element of this YArray. */ forEach (f) { - typeListForEach(this, f) + return transact(/** @type {Doc} */ (this.doc), tr => + new ListIterator(this).forEach(tr, f) + ) } /** * @return {IterableIterator} */ [Symbol.iterator] () { - return typeListCreateIterator(this) + return this.toArray().values() } /** diff --git a/src/types/YText.js b/src/types/YText.js index 328795e0..e327a640 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -20,14 +20,15 @@ import { splitSnapshotAffectedStructs, iterateDeletedStructs, iterateStructs, - findMarker, typeMapDelete, typeMapSet, typeMapGet, typeMapGetAll, updateMarkerChanges, ContentType, - ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line + useSearchMarker, + findIndexCleanStart, + ListIterator, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line } from '../internals.js' import * as object from 'lib0/object' @@ -125,10 +126,30 @@ const findNextPosition = (transaction, pos, count) => { */ const findPosition = (transaction, parent, index) => { const currentAttributes = new Map() - const marker = findMarker(parent, index) - if (marker) { - const pos = new ItemTextListPosition(marker.p.left, marker.p, marker.index, currentAttributes) - return findNextPosition(transaction, pos, index - marker.index) + if (parent._searchMarker) { + return useSearchMarker(transaction, parent, index, listIter => { + let left, right + if (listIter.rel > 0) { + // must exist because rel > 0 + const nextItem = /** @type {Item} */ (listIter.nextItem) + if (listIter.rel === nextItem.length) { + left = nextItem + right = left.right + } else { + const structs = /** @type {Array} */ (transaction.doc.store.clients.get(nextItem.id.client)) + const after = /** @type {Item} */ (structs[findIndexCleanStart(transaction, structs, nextItem.id.clock + listIter.rel)]) + listIter.nextItem = after + listIter.rel = 0 + left = listIter.left + right = listIter.right + } + } else { + left = listIter.left + right = listIter.right + } + // @todo this should simply split if .rel > 0 + return new ItemTextListPosition(left, right, index, currentAttributes) + }) } else { const pos = new ItemTextListPosition(null, parent._start, 0, currentAttributes) return findNextPosition(transaction, pos, index) @@ -264,7 +285,7 @@ const insertText = (transaction, parent, currPos, text, attributes) => { const content = text.constructor === String ? new ContentString(/** @type {string} */ (text)) : (text instanceof AbstractType ? new ContentType(text) : new ContentEmbed(text)) let { left, right, index } = currPos if (parent._searchMarker) { - updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength()) + updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength(), null) } right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content) right.integrate(transaction, 0) @@ -469,7 +490,7 @@ const deleteText = (transaction, currPos, length) => { } const parent = /** @type {AbstractType} */ (/** @type {Item} */ (currPos.left || currPos.right).parent) if (parent._searchMarker) { - updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length) + updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length, null) } return currPos } @@ -764,7 +785,7 @@ export class YText extends AbstractType { */ this._pending = string !== undefined ? [() => this.insert(0, string)] : [] /** - * @type {Array} + * @type {Array} */ this._searchMarker = [] } diff --git a/src/types/YXmlFragment.js b/src/types/YXmlFragment.js index c18b9ca0..9f957475 100644 --- a/src/types/YXmlFragment.js +++ b/src/types/YXmlFragment.js @@ -8,15 +8,13 @@ import { AbstractType, typeListMap, typeListForEach, - typeListInsertGenerics, typeListInsertGenericsAfter, - typeListDelete, typeListToArray, YXmlFragmentRefID, callTypeObservers, transact, - typeListGet, typeListSlice, + useSearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line } from '../internals.js' @@ -304,9 +302,11 @@ export class YXmlFragment extends AbstractType { */ insert (index, content) { if (this.doc !== null) { - transact(this.doc, transaction => { - typeListInsertGenerics(transaction, this, index, content) - }) + return transact(/** @type {Doc} */ (this.doc), transaction => + useSearchMarker(transaction, this, index, walker => + walker.insertArrayValue(transaction, content) + ) + ) } else { // @ts-ignore _prelimContent is defined because this is not yet integrated this._prelimContent.splice(index, 0, ...content) @@ -347,9 +347,11 @@ export class YXmlFragment extends AbstractType { */ delete (index, length = 1) { if (this.doc !== null) { - transact(this.doc, transaction => { - typeListDelete(transaction, this, index, length) - }) + transact(/** @type {Doc} */ (this.doc), transaction => + useSearchMarker(transaction, this, index, walker => + walker.delete(transaction, length) + ) + ) } else { // @ts-ignore _prelimContent is defined because this is not yet integrated this._prelimContent.splice(index, length) @@ -390,7 +392,11 @@ export class YXmlFragment extends AbstractType { * @return {YXmlElement|YXmlText} */ get (index) { - return typeListGet(this, index) + return transact(/** @type {Doc} */ (this.doc), transaction => + useSearchMarker(transaction, this, index, walker => + walker.slice(transaction, 1)[0] + ) + ) } /** diff --git a/src/utils/ListIterator.js b/src/utils/ListIterator.js new file mode 100644 index 00000000..4404f23c --- /dev/null +++ b/src/utils/ListIterator.js @@ -0,0 +1,510 @@ +import * as error from 'lib0/error' + +import { + getItemCleanStart, + createID, + getMovedCoords, + updateMarkerChanges, + getState, + ContentAny, + ContentBinary, + ContentType, + ContentDoc, + Doc, + RelativePosition, ID, AbstractContent, ContentMove, Transaction, Item, AbstractType // eslint-disable-line +} from '../internals.js' + +const lengthExceeded = error.create('Length exceeded!') + +/** + * @todo rename to walker? + * @todo check that inserting character one after another always reuses ListIterators + */ +export class ListIterator { + /** + * @param {AbstractType} type + */ + constructor (type) { + this.type = type + /** + * Current index-position + */ + this.index = 0 + /** + * Relative position to the current item (if item.content.length > 1) + */ + this.rel = 0 + /** + * This refers to the current right item, unless reachedEnd is true. Then it refers to the left item. + * + * @public + * @type {Item | null} + */ + this.nextItem = type._start + this.reachedEnd = type._start === null + /** + * @type {Item | null} + */ + this.currMove = null + /** + * @type {Item | null} + */ + this.currMoveStart = null + /** + * @type {Item | null} + */ + this.currMoveEnd = null + /** + * @type {Array<{ start: Item | null, end: Item | null, move: Item }>} + */ + this.movedStack = [] + } + + clone () { + const iter = new ListIterator(this.type) + iter.index = this.index + iter.rel = this.rel + iter.nextItem = this.nextItem + iter.reachedEnd = this.reachedEnd + iter.currMove = this.currMove + iter.currMoveStart = this.currMoveStart + iter.currMoveEnd = this.currMoveEnd + iter.movedStack = this.movedStack.slice() + return iter + } + + /** + * @type {Item | null} + */ + get left () { + if (this.reachedEnd) { + return this.nextItem + } else { + return this.nextItem && this.nextItem.left + } + } + + /** + * @type {Item | null} + */ + get right () { + if (this.reachedEnd) { + return null + } else { + return this.nextItem + } + } + + /** + * @param {Transaction} tr + * @param {number} index + */ + moveTo (tr, index) { + const diff = index - this.index + if (diff > 0) { + this.forward(tr, diff) + } else if (diff < 0) { + this.backward(tr, -diff) + } + } + + /** + * @param {Transaction} tr + * @param {number} len + */ + forward (tr, len) { + if (this.index + len > this.type._length) { + throw lengthExceeded + } + let item = this.nextItem + this.index += len + if (this.rel) { + len += this.rel + this.rel = 0 + } + while ((!this.reachedEnd || this.currMove !== null) && (len > 0 || (len === 0 && item && (!item.countable || item.deleted || item === this.currMoveEnd || (this.reachedEnd && this.currMoveEnd === null) || item.moved !== this.currMove)))) { + if (item === this.currMoveEnd || (this.currMoveEnd === null && this.reachedEnd && this.currMove)) { + item = /** @type {Item} */ (this.currMove) // we iterate to the right after the current condition + const { start, end, move } = this.movedStack.pop() || { start: null, end: null, move: null } + this.currMove = move + this.currMoveStart = start + this.currMoveEnd = end + this.reachedEnd = false + } else if (item === null) { + break + } else if (item.countable && !item.deleted && item.moved === this.currMove && len > 0) { + len -= item.length + if (len < 0) { + this.rel = item.length + len + len = 0 + break + } + } else if (item.content.constructor === ContentMove && item.moved === this.currMove) { + if (this.currMove) { + this.movedStack.push({ start: this.currMoveStart, end: this.currMoveEnd, move: this.currMove }) + } + const { start, end } = getMovedCoords(item.content, tr) + this.currMove = item + this.currMoveStart = start + this.currMoveEnd = end + item = start + continue + } + if (item.right) { + item = item.right + } else { + this.reachedEnd = true + } + } + this.index -= len + this.nextItem = item + return this + } + + /** + * @param {Transaction} tr + */ + reduceMoves (tr) { + let item = this.nextItem + if (item !== null) { + while (item === this.currMoveStart) { + item = /** @type {Item} */ (this.currMove) // we iterate to the left after the current condition + const { start, end, move } = this.movedStack.pop() || { start: null, end: null, move: null } + this.currMove = move + this.currMoveStart = start + this.currMoveEnd = end + } + this.nextItem = item + } + } + + /** + * @param {Transaction} tr + * @param {number} len + * @return {ListIterator} + */ + backward (tr, len) { + if (this.index - len < 0) { + throw lengthExceeded + } + this.index -= len + if (this.reachedEnd) { + const nextItem = /** @type {Item} */ (this.nextItem) + this.rel = nextItem.countable && !nextItem.deleted ? nextItem.length : 0 + this.reachedEnd = false + } + if (this.rel >= len) { + this.rel -= len + return this + } + let item = this.nextItem && this.nextItem.left + if (this.rel) { + len -= this.rel + this.rel = 0 + } + while (item && len > 0) { + if (item.countable && !item.deleted && item.moved === this.currMove) { + len -= item.length + if (len < 0) { + this.rel = -len + len = 0 + } + if (len === 0) { + break + } + } else if (item.content.constructor === ContentMove && item.moved === this.currMove) { + if (this.currMove) { + this.movedStack.push({ start: this.currMoveStart, end: this.currMoveEnd, move: this.currMove }) + } + const { start, end } = getMovedCoords(item.content, tr) + this.currMove = item + this.currMoveStart = start + this.currMoveEnd = end + item = start + continue + } + if (item === this.currMoveStart) { + item = /** @type {Item} */ (this.currMove) // we iterate to the left after the current condition + const { start, end, move } = this.movedStack.pop() || { start: null, end: null, move: null } + this.currMove = move + this.currMoveStart = start + this.currMoveEnd = end + } + item = item.left + } + this.nextItem = item + return this + } + + /** + * @template {{length: number}} T + * @param {Transaction} tr + * @param {number} len + * @param {T} value the initial content + * @param {function(AbstractContent, number, number):T} slice + * @param {function(T, T): T} concat + */ + _slice (tr, len, value, slice, concat) { + this.index += len + while (len > 0 && !this.reachedEnd) { + while (this.nextItem && this.nextItem.countable && !this.reachedEnd && len > 0 && this.nextItem !== this.currMoveEnd) { + if (!this.nextItem.deleted && this.nextItem.moved === this.currMove) { + const item = this.nextItem + const slicedContent = slice(item.content, this.rel, len) + len -= slicedContent.length + value = concat(value, slicedContent) + if (item.length !== slicedContent.length) { + if (this.rel + slicedContent.length === item.length) { + this.rel = 0 + } else { + this.rel += slicedContent.length + continue // do not iterate to item.right + } + } + } + if (this.nextItem.right) { + this.nextItem = this.nextItem.right + } else { + this.reachedEnd = true + } + } + if (this.nextItem && (!this.reachedEnd || this.currMove !== null) && len > 0) { + this.forward(tr, 0) + } + } + if (len < 0) { + this.index -= len + } + return value + } + + /** + * @param {Transaction} tr + * @param {number} len + */ + delete (tr, len) { + const startLength = len + const sm = this.type._searchMarker + let item = this.nextItem + while (len > 0) { + while (item && !item.deleted && item.countable && !this.reachedEnd && len > 0 && item.moved === this.currMove && item !== this.currMoveEnd) { + if (this.rel > 0) { + item = getItemCleanStart(tr, createID(item.id.client, item.id.clock + this.rel)) + this.rel = 0 + } + if (len < item.length) { + getItemCleanStart(tr, createID(item.id.client, item.id.clock + len)) + } + len -= item.length + item.delete(tr) + if (item.right) { + item = item.right + } else { + this.reachedEnd = true + } + } + if (len > 0) { + this.nextItem = item + this.forward(tr, 0) + item = this.nextItem + } + } + this.nextItem = item + if (sm) { + updateMarkerChanges(sm, this.index, -startLength + len, this) + } + } + + /** + * @param {Transaction} tr + */ + _splitRel (tr) { + if (this.rel > 0) { + /** + * @type {ID} + */ + const itemid = /** @type {Item} */ (this.nextItem).id + this.nextItem = getItemCleanStart(tr, createID(itemid.client, itemid.clock + this.rel)) + this.rel = 0 + } + } + + /** + * Important: you must update markers after calling this method! + * + * @param {Transaction} tr + * @param {Array} content + */ + insertContents (tr, content) { + this.reduceMoves(tr) + this._splitRel(tr) + const parent = this.type + const store = tr.doc.store + const ownClientId = tr.doc.clientID + /** + * @type {Item | null} + */ + const right = this.right + + /** + * @type {Item | null} + */ + let left = this.left + content.forEach(c => { + left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, c) + left.integrate(tr, 0) + }) + if (right === null) { + this.nextItem = left + this.reachedEnd = true + } else { + this.nextItem = right + } + } + + /** + * @param {Transaction} tr + * @param {RelativePosition} start + * @param {RelativePosition} end + */ + insertMove (tr, start, end) { + this.insertContents(tr, [new ContentMove(start, end, -1)]) // @todo adjust priority + // @todo is there a better alrogirthm to update searchmarkers? We could simply remove the markers that are in the updated range. + // Also note that searchmarkers are updated in insertContents as well. + const sm = this.type._searchMarker + if (sm) sm.length = 0 + } + + /** + * @param {Transaction} tr + * @param {Array|Array|boolean|number|null|string|Uint8Array>} values + */ + insertArrayValue (tr, values) { + this._splitRel(tr) + const sm = this.type._searchMarker + /** + * @type {Array} + */ + const contents = [] + /** + * @type {Array|number|null>} + */ + let jsonContent = [] + const packJsonContent = () => { + if (jsonContent.length > 0) { + contents.push(new ContentAny(jsonContent)) + jsonContent = [] + } + } + values.forEach(c => { + if (c === null) { + jsonContent.push(c) + } else { + switch (c.constructor) { + case Number: + case Object: + case Boolean: + case Array: + case String: + jsonContent.push(c) + break + default: + packJsonContent() + switch (c.constructor) { + case Uint8Array: + case ArrayBuffer: + contents.push(new ContentBinary(new Uint8Array(/** @type {Uint8Array} */ (c)))) + break + case Doc: + contents.push(new ContentDoc(/** @type {Doc} */ (c))) + break + default: + if (c instanceof AbstractType) { + contents.push(new ContentType(c)) + } else { + throw new Error('Unexpected content type in insert operation') + } + } + } + } + }) + packJsonContent() + this.insertContents(tr, contents) + this.index += values.length + if (sm) { + updateMarkerChanges(sm, this.index - values.length, values.length, this) + } + } + + /** + * @param {Transaction} tr + * @param {number} len + */ + slice (tr, len) { + return this._slice(tr, len, [], sliceArrayContent, concatArrayContent) + } + + /** + * @param {Transaction} tr + * @param {function(any, number, any):void} f + */ + forEach (tr, f) { + for (const val of this.values(tr)) { + f(val, this.index, this.type) + } + } + + /** + * @template T + * @param {Transaction} tr + * @param {function(any, number, any):T} f + * @return {Array} + */ + map (tr, f) { + const arr = new Array(this.type._length - this.index) + let i = 0 + for (const val of this.values(tr)) { + arr[i++] = f(val, this.index, this.type) + } + return arr + } + + /** + * @param {Transaction} tr + */ + values (tr) { + return { + [Symbol.iterator] () { + return this + }, + next: () => { + if (this.reachedEnd || this.index === this.type._length) { + return { done: true } + } + const [value] = this.slice(tr, 1) + return { + done: false, + value: value + } + } + } + } +} + +/** + * @param {AbstractContent} itemcontent + * @param {number} start + * @param {number} len + */ +const sliceArrayContent = (itemcontent, start, len) => { + const content = itemcontent.getContent() + return content.length <= len && start === 0 ? content : content.slice(start, start + len) +} +/** + * @param {Array} content + * @param {Array} added + */ +const concatArrayContent = (content, added) => { + content.push(...added) + return content +} diff --git a/src/utils/RelativePosition.js b/src/utils/RelativePosition.js index 614c0bc5..ea3e33e9 100644 --- a/src/utils/RelativePosition.js +++ b/src/utils/RelativePosition.js @@ -9,6 +9,8 @@ import { createID, ContentType, followRedone, + transact, + useSearchMarker, ID, Doc, AbstractType // eslint-disable-line } from '../internals.js' @@ -73,6 +75,10 @@ export class RelativePosition { */ this.assoc = assoc } + + clone () { + return new RelativePosition(this.type, this.tname, this.item, this.assoc) + } } /** @@ -161,7 +167,6 @@ export const createRelativePosition = (type, item, assoc) => { * @function */ export const createRelativePositionFromTypeIndex = (type, index, assoc = 0) => { - let t = type._start if (assoc < 0) { // associated to the left character or the beginning of a type, increment index if possible. if (index === 0) { @@ -169,21 +174,17 @@ export const createRelativePositionFromTypeIndex = (type, index, assoc = 0) => { } index-- } - while (t !== null) { - if (!t.deleted && t.countable) { - if (t.length > index) { - // case 1: found position somewhere in the linked list - return createRelativePosition(type, createID(t.id.client, t.id.clock + index), assoc) + return transact(/** @type {Doc} */ (type.doc), tr => + useSearchMarker(tr, type, index, walker => { + if (walker.reachedEnd) { + const item = assoc < 0 ? /** @type {Item} */ (walker.nextItem).lastId : null + return createRelativePosition(type, item, assoc) + } else { + const id = /** @type {Item} */ (walker.nextItem).id + return createRelativePosition(type, createID(id.client, id.clock + walker.rel), assoc) } - index -= t.length - } - if (t.right === null && assoc < 0) { - // left-associated position, return last available id - return createRelativePosition(type, t.lastId, assoc) - } - t = t.right - } - return createRelativePosition(type, null, assoc) + }) + ) } /** diff --git a/src/utils/StructStore.js b/src/utils/StructStore.js index 7a2e256c..079d3bc3 100644 --- a/src/utils/StructStore.js +++ b/src/utils/StructStore.js @@ -199,19 +199,18 @@ export const getItemCleanStart = (transaction, id) => { * Expects that id is actually in store. This function throws or is an infinite loop otherwise. * * @param {Transaction} transaction - * @param {StructStore} store * @param {ID} id * @return {Item} * * @private * @function */ -export const getItemCleanEnd = (transaction, store, id) => { +export const getItemCleanEnd = (transaction, id) => { /** * @type {Array} */ // @ts-ignore - const structs = store.clients.get(id.client) + const structs = transaction.doc.store.clients.get(id.client) const index = findIndexSS(structs, id.clock) const struct = structs[index] if (id.clock !== struct.id.clock + struct.length - 1 && struct.constructor !== GC) { diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index a9ab6afa..6476fa30 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -114,6 +114,14 @@ export class Transaction { * @type {Set} */ this.subdocsLoaded = new Set() + /** + * We store the reference that last moved an item. + * This is needed to compute the delta when multiple ContentMove move + * the same item. + * + * @type {Map} + */ + this.prevMoved = new Map() } } @@ -377,9 +385,12 @@ const cleanupTransactions = (transactionCleanups, i) => { /** * Implements the functionality of `y.transact(()=>{..})` * + * @template T + * * @param {Doc} doc - * @param {function(Transaction):void} f + * @param {function(Transaction):T} f * @param {any} [origin=true] + * @return {T} * * @function */ @@ -395,8 +406,9 @@ export const transact = (doc, f, origin = null, local = true) => { } doc.emit('beforeTransaction', [doc._transaction, doc]) } + let res try { - f(doc._transaction) + res = f(doc._transaction) } finally { if (initialCall && transactionCleanups[0] === doc._transaction) { // The first transaction ended, now process observer calls. @@ -410,4 +422,5 @@ export const transact = (doc, f, origin = null, local = true) => { cleanupTransactions(transactionCleanups, 0) } } + return res } diff --git a/src/utils/YEvent.js b/src/utils/YEvent.js index 3501b587..e583fed3 100644 --- a/src/utils/YEvent.js +++ b/src/utils/YEvent.js @@ -1,7 +1,8 @@ import { isDeleted, - Item, AbstractType, Transaction, AbstractStruct // eslint-disable-line + getMovedCoords, + ContentMove, Item, AbstractType, Transaction, AbstractStruct // eslint-disable-line } from '../internals.js' import * as set from 'lib0/set' @@ -153,62 +154,107 @@ export class YEvent { get changes () { let changes = this._changes if (changes === null) { - const target = this.target - const added = set.create() - const deleted = set.create() - /** - * @type {Array<{insert:Array}|{delete:number}|{retain:number}>} - */ - const delta = [] - changes = { - added, - deleted, - delta, - keys: this.keys - } - const changed = /** @type Set */ (this.transaction.changed.get(target)) - if (changed.has(null)) { + this.transaction.doc.transact(tr => { + const target = this.target + const added = set.create() + const deleted = set.create() /** - * @type {any} + * @type {Array<{insert:Array}|{delete:number}|{retain:number}>} */ - let lastOp = null - const packOp = () => { - if (lastOp) { - delta.push(lastOp) - } + const delta = [] + changes = { + added, + deleted, + delta, + keys: this.keys } - for (let item = target._start; item !== null; item = item.right) { - if (item.deleted) { - if (this.deletes(item) && !this.adds(item)) { - if (lastOp === null || lastOp.delete === undefined) { - packOp() - lastOp = { delete: 0 } - } - lastOp.delete += item.length - deleted.add(item) - } // else nop - } else { - if (this.adds(item)) { - if (lastOp === null || lastOp.insert === undefined) { - packOp() - lastOp = { insert: [] } - } - lastOp.insert = lastOp.insert.concat(item.content.getContent()) - added.add(item) - } else { - if (lastOp === null || lastOp.retain === undefined) { - packOp() - lastOp = { retain: 0 } - } - lastOp.retain += item.length + const changed = /** @type Set */ (this.transaction.changed.get(target)) + if (changed.has(null)) { + /** + * @type {Array<{ end: Item | null, move: Item | null, isNew : boolean }>} + */ + const movedStack = [] + /** + * @type {Item | null} + */ + let currMove = null + /** + * @type {boolean} + */ + let currMoveIsNew = false + /** + * @type {Item | null} + */ + let currMoveEnd = null + /** + * @type {any} + */ + let lastOp = null + const packOp = () => { + if (lastOp) { + delta.push(lastOp) } } + for (let item = target._start; ;) { + if (item === currMoveEnd && currMove) { + item = currMove + const { end, move, isNew } = movedStack.pop() || { end: null, move: null, isNew: false } + currMoveIsNew = isNew + currMoveEnd = end + currMove = move + } else if (item === null) { + break + } else if (item.content.constructor === ContentMove) { + if (item.moved === currMove) { + movedStack.push({ end: currMoveEnd, move: currMove, isNew: currMoveIsNew }) + const { start, end } = getMovedCoords(item.content, tr) + currMove = item + currMoveEnd = end + currMoveIsNew = this.adds(item) + item = start + continue // do not move to item.right + } + } else if (item.moved !== currMove) { + if (!currMoveIsNew && item.countable && item.moved && !this.adds(item) && this.adds(item.moved) && (this.transaction.prevMoved.get(item) || null) === currMove) { + if (lastOp === null || lastOp.delete === undefined) { + packOp() + lastOp = { delete: 0 } + } + lastOp.delete += item.length + } + } else if (item.deleted) { + if (!currMoveIsNew && this.deletes(item) && !this.adds(item)) { + if (lastOp === null || lastOp.delete === undefined) { + packOp() + lastOp = { delete: 0 } + } + lastOp.delete += item.length + deleted.add(item) + } + } else { + if (currMoveIsNew || this.adds(item)) { + if (lastOp === null || lastOp.insert === undefined) { + packOp() + lastOp = { insert: [] } + } + lastOp.insert = lastOp.insert.concat(item.content.getContent()) + added.add(item) + } else { + if (lastOp === null || lastOp.retain === undefined) { + packOp() + lastOp = { retain: 0 } + } + lastOp.retain += item.length + } + } + item = /** @type {Item} */ (item).right + } + if (lastOp !== null && lastOp.retain === undefined) { + packOp() + } } - if (lastOp !== null && lastOp.retain === undefined) { - packOp() - } - } - this._changes = changes + this._changes = changes + }) } return /** @type {any} */ (changes) } diff --git a/src/utils/updates.js b/src/utils/updates.js index fa6add00..0a9072c5 100644 --- a/src/utils/updates.js +++ b/src/utils/updates.js @@ -308,6 +308,7 @@ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = U // Note: Should handle that some operations cannot be applied yet () while (true) { + // @todo this incurs an exponential overhead. We could instead only sort the item that changed. // Write higher clients first ⇒ sort by clientID & clock and remove decoders without content lazyStructDecoders = lazyStructDecoders.filter(dec => dec.curr !== null) lazyStructDecoders.sort( diff --git a/tests/doc.tests.js b/tests/doc.tests.js index 994ecaeb..c167019e 100644 --- a/tests/doc.tests.js +++ b/tests/doc.tests.js @@ -40,6 +40,7 @@ export const testToJSON = tc => { const arr = doc.getArray('array') arr.push(['test1']) + t.compare(arr.toJSON(), ['test1']) const map = doc.getMap('map') map.set('k1', 'v1') diff --git a/tests/encoding.tests.js b/tests/encoding.tests.js index 6a191009..73c396aa 100644 --- a/tests/encoding.tests.js +++ b/tests/encoding.tests.js @@ -12,6 +12,7 @@ import { readContentFormat, readContentAny, readContentDoc, + readContentMove, Doc, PermanentUserData, encodeStateAsUpdate, @@ -24,7 +25,8 @@ import * as Y from '../src/index.js' * @param {t.TestCase} tc */ export const testStructReferences = tc => { - t.assert(contentRefs.length === 11) + t.assert(contentRefs.length === 12) + // contentRefs[0] is reserved for GC t.assert(contentRefs[1] === readContentDeleted) t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json? t.assert(contentRefs[3] === readContentBinary) @@ -35,6 +37,7 @@ export const testStructReferences = tc => { t.assert(contentRefs[8] === readContentAny) t.assert(contentRefs[9] === readContentDoc) // contentRefs[10] is reserved for Skip structs + t.assert(contentRefs[11] === readContentMove) } /** diff --git a/tests/testHelper.js b/tests/testHelper.js index 4df1d11e..9963e16e 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -373,6 +373,33 @@ export const compare = users => { t.compare(Y.encodeStateVector(users[i]), Y.encodeStateVector(users[i + 1])) compareDS(Y.createDeleteSetFromStructStore(users[i].store), Y.createDeleteSetFromStructStore(users[i + 1].store)) compareStructStores(users[i].store, users[i + 1].store) + // @todo + // test list-iterator + // console.log('dutiraneduiaentdr', users[0].getArray('array')._searchMarker) + /* + { + const user = users[0] + user.transact(tr => { + const type = user.getArray('array') + Y.useSearchMarker(tr, type, type.length, walker => { + for (let i = type.length; i >= 0; i--) { + const otherWalker = new Y.ListIterator(type) + otherWalker.forward(tr, walker.index) + otherWalker.forward(tr, 0) + walker.forward(tr, 0) + t.assert(walker.index === i) + t.assert(walker.left === otherWalker.left) + t.assert(walker.right === otherWalker.right) + t.assert(walker.nextItem === otherWalker.nextItem) + t.assert(walker.reachedEnd === otherWalker.reachedEnd) + if (i > 0) { + walker.backward(tr, 1) + } + } + }) + }) + } + */ } users.map(u => u.destroy()) } diff --git a/tests/undo-redo.tests.js b/tests/undo-redo.tests.js index 991225ca..1bf25c82 100644 --- a/tests/undo-redo.tests.js +++ b/tests/undo-redo.tests.js @@ -1,4 +1,4 @@ -import { init, compare, applyRandomTests, Doc } from './testHelper.js' // eslint-disable-line +import { init, compare, applyRandomTests, Doc, UndoManager } from './testHelper.js' // eslint-disable-line import * as Y from '../src/index.js' import * as t from 'lib0/testing' diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index 8cdaac5b..0d0e84fa 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -1,4 +1,4 @@ -import { init, compare, applyRandomTests, Doc } from './testHelper.js' // eslint-disable-line +import { init, compare, applyRandomTests, Doc, AbstractType, TestConnector } from './testHelper.js' // eslint-disable-line import * as Y from '../src/index.js' import * as t from 'lib0/testing' @@ -432,6 +432,86 @@ export const testEventTargetIsSetCorrectlyOnRemote = tc => { compare(users) } +/** + * @param {t.TestCase} tc + */ +export const testMove = tc => { + { + // move in uninitialized type + const yarr = new Y.Array() + yarr.insert(0, [1, 2, 3]) + yarr.move(1, 0) + // @ts-ignore + t.compare(yarr._prelimContent, [2, 1, 3]) + } + const { array0, array1, users } = init(tc, { users: 3 }) + /** + * @type {any} + */ + let event0 = null + /** + * @type {any} + */ + let event1 = null + array0.observe(event => { + event0 = event + }) + array1.observe(event => { + event1 = event + }) + array0.insert(0, [1, 2, 3]) + array0.move(1, 0) + t.compare(array0.toArray(), [2, 1, 3]) + t.compare(event0.delta, [{ insert: [2] }, { retain: 1 }, { delete: 1 }]) + Y.applyUpdate(users[1], Y.encodeStateAsUpdate(users[0])) + t.compare(array1.toArray(), [2, 1, 3]) + t.compare(event1.delta, [{ insert: [2, 1, 3] }]) + array0.move(0, 2) + t.compare(array0.toArray(), [1, 2, 3]) + t.compare(event0.delta, [{ delete: 1 }, { retain: 1 }, { insert: [2] }]) + compare(users) +} + +/** + * @param {t.TestCase} tc + */ +export const testMove2 = tc => { + { + // move in uninitialized type + const yarr = new Y.Array() + yarr.insert(0, [1, 2]) + yarr.move(1, 0) + // @ts-ignore + t.compare(yarr._prelimContent, [2, 1]) + } + const { array0, array1, users } = init(tc, { users: 3 }) + /** + * @type {any} + */ + let event0 = null + /** + * @type {any} + */ + let event1 = null + array0.observe(event => { + event0 = event + }) + array1.observe(event => { + event1 = event + }) + array0.insert(0, [1, 2]) + array0.move(1, 0) + t.compare(array0.toArray(), [2, 1]) + t.compare(event0.delta, [{ insert: [2] }, { retain: 1 }, { delete: 1 }]) + Y.applyUpdate(users[1], Y.encodeStateAsUpdate(users[0])) + t.compare(array1.toArray(), [2, 1]) + t.compare(event1.delta, [{ insert: [2, 1] }]) + array0.move(0, 2) + t.compare(array0.toArray(), [1, 2]) + t.compare(event0.delta, [{ delete: 1 }, { retain: 1 }, { insert: [2] }]) + compare(users) +} + /** * @param {t.TestCase} tc */ @@ -456,8 +536,23 @@ const getUniqueNumber = () => _uniqueNumber++ /** * @type {Array} + * + * @todo to replace content to a separate data structure so we know that insert & returns work as expected!!! */ const arrayTransactions = [ + function move (user, gen) { + const yarray = user.getArray('array') + if (yarray.length === 0) { + return + } + const pos = prng.int32(gen, 0, yarray.length - 1) + const newPos = prng.int32(gen, 0, yarray.length) + const oldContent = yarray.toArray() + yarray.move(pos, newPos) + const [x] = oldContent.splice(pos, 1) + oldContent.splice(pos < newPos ? newPos - 1 : newPos, 0, x) + t.compareArrays(yarray.toArray(), oldContent) // we want to make sure that fastSearch markers insert at the correct position + }, function insert (user, gen) { const yarray = user.getArray('array') const uniqueNumber = getUniqueNumber() @@ -516,11 +611,49 @@ const arrayTransactions = [ } ] +/** + * @param {Y.Doc} user + */ +const monitorArrayTestObject = user => { + /** + * @type {Array} + */ + const arr = [] + const yarr = user.getArray('array') + yarr.observe(event => { + let currpos = 0 + const delta = event.delta + for (let i = 0; i < delta.length; i++) { + const d = delta[i] + if (d.insert != null) { + arr.splice(currpos, 0, ...(/** @type {Array} */ (d.insert))) + currpos += /** @type {Array} */ (d.insert).length + } else if (d.retain != null) { + currpos += d.retain + } else { + arr.splice(currpos, d.delete) + } + } + }) + return arr +} + +/** + * @param {{ testObjects: Array>, users: Array }} cmp + */ +const compareTestobjects = cmp => { + const arrs = cmp.testObjects + for (let i = 0; i < arrs.length; i++) { + const type = cmp.users[i].getArray('array') + t.compareArrays(arrs[i], type.toArray()) + } +} + /** * @param {t.TestCase} tc */ export const testRepeatGeneratingYarrayTests6 = tc => { - applyRandomTests(tc, arrayTransactions, 6) + compareTestobjects(applyRandomTests(tc, arrayTransactions, 7, monitorArrayTestObject)) } /** diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index 2a555237..f066782c 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -327,6 +327,7 @@ export const testFormattingDeltaUnnecessaryAttributeChange = tc => { * @param {t.TestCase} tc */ export const testInsertAndDeleteAtRandomPositions = tc => { + // @todo optimize to run at least as fast as previous marker approach const N = 100000 const { text0 } = init(tc, { users: 1 }) const gen = tc.prng @@ -552,8 +553,6 @@ export const testSearchMarkerBug1 = tc => { } /** - * Reported in https://github.com/yjs/yjs/pull/32 - * * @param {t.TestCase} tc */ export const testFormattingBug = async tc => { @@ -563,7 +562,6 @@ export const testFormattingBug = async tc => { text1.insert(0, '\n\n\n') text1.format(0, 3, { url: 'http://example.com' }) ydoc1.getText().format(1, 1, { url: 'http://docs.yjs.dev' }) - ydoc2.getText().format(1, 1, { url: 'http://docs.yjs.dev' }) Y.applyUpdate(ydoc2, Y.encodeStateAsUpdate(ydoc1)) const text2 = ydoc2.getText() const expectedResult = [