diff --git a/src/index.js b/src/index.js index 2c02b1e9..66fe70da 100644 --- a/src/index.js +++ b/src/index.js @@ -69,5 +69,6 @@ export { PermanentUserData, // @TODO experimental tryGc, transact, - AbstractConnector + AbstractConnector, + logType } from './internals.js' diff --git a/src/internals.js b/src/internals.js index d9ee9035..02e66640 100644 --- a/src/internals.js +++ b/src/internals.js @@ -8,6 +8,7 @@ export * from './utils/encoding.js' export * from './utils/EventHandler.js' export * from './utils/ID.js' export * from './utils/isParentOf.js' +export * from './utils/logging.js' export * from './utils/PermanentUserData.js' export * from './utils/RelativePosition.js' export * from './utils/Snapshot.js' diff --git a/src/structs/ContentFormat.js b/src/structs/ContentFormat.js index 7cfc85d7..43716408 100644 --- a/src/structs/ContentFormat.js +++ b/src/structs/ContentFormat.js @@ -1,6 +1,6 @@ import { - AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line + AbstractType, AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line } from '../internals.js' import * as error from 'lib0/error.js' @@ -66,7 +66,11 @@ export class ContentFormat { * @param {Transaction} transaction * @param {Item} item */ - integrate (transaction, item) {} + integrate (transaction, item) { + // @todo searchmarker are currently unsupported for rich text documents + /** @type {AbstractType} */ (item.parent)._searchMarker = null + } + /** * @param {Transaction} transaction */ diff --git a/src/structs/Item.js b/src/structs/Item.js index 6eaffeb8..c5e167c6 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -285,9 +285,31 @@ export class Item extends AbstractStruct { * @type {AbstractContent} */ this.content = content + /** + * bit1: keep + * bit2: countable + * bit3: deleted + * bit4: mark - mark node as fast-search-marker + * @type {number} byte + */ this.info = this.content.isCountable() ? binary.BIT2 : 0 } + /** + * This is used to mark the item as an indexed fast-search marker + * + * @type {boolean} + */ + set marker (isMarked) { + if (((this.info & binary.BIT4) > 0) !== isMarked) { + this.info ^= binary.BIT4 + } + } + + get marker () { + return (this.info & binary.BIT4) > 0 + } + /** * If true, do not garbage collect this Item. */ diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 5281372a..1f414fec 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -17,6 +17,196 @@ import { import * as map from 'lib0/map.js' import * as iterator from 'lib0/iterator.js' import * as error from 'lib0/error.js' +import * as math from 'lib0/math.js' + +const maxSearchMarker = 60 + +/** + * A unique timestamp that identifies each marker. + * + * Time is relative,.. this is more like an ever-increasing clock. + * + * @type {number} + */ +let globalSearchMarkerTimestamp = 0 + +export class ArraySearchMarker { + /** + * @param {Item} p + * @param {number} index + */ + constructor (p, index) { + p.marker = true + this.p = p + this.index = index + this.timestamp = globalSearchMarkerTimestamp++ + } +} + +/** + * @param {ArraySearchMarker} marker + */ +const refreshMarkerTimestamp = marker => { marker.timestamp = globalSearchMarkerTimestamp++ } + +/** + * This is rather complex so this function is the only thing that should overwrite a marker + * + * @param {ArraySearchMarker} marker + * @param {Item} p + * @param {number} index + */ +const overwriteMarker = (marker, p, index) => { + marker.p.marker = false + marker.p = p + p.marker = true + marker.index = index + marker.timestamp = globalSearchMarkerTimestamp++ +} + +/** + * @param {Array} searchMarker + * @param {Item} p + * @param {number} index + */ +const markPosition = (searchMarker, p, index) => { + if (searchMarker.length >= maxSearchMarker) { + // override oldest marker (we don't want to create more objects) + const marker = searchMarker.reduce((a, b) => a.timestamp < b.timestamp ? a : b) + overwriteMarker(marker, p, index) + return marker + } else { + // create new marker + const pm = new ArraySearchMarker(p, index) + searchMarker.push(pm) + return pm + } +} + +/** + * Search marker help us to find positions in the associative array faster. + * + * They speed up the process of finding a position without much bookkeeping. + * + * A maximum of `maxSearchMarker` objects are created. + * + * This function always returns a refreshed marker (updated timestamp) + * + * @param {AbstractType} yarray + * @param {number} index + */ +export const findMarker = (yarray, index) => { + if (yarray._start === null || index === 0 || yarray._searchMarker === null) { + return null + } + const marker = yarray._searchMarker.length === 0 ? null : yarray._searchMarker.reduce((a, b) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b) + let p = yarray._start + let pindex = 0 + if (marker !== null) { + p = marker.p + pindex = marker.index + refreshMarkerTimestamp(marker) // we used it, we might need to use it again + } + // iterate to right if possible + while (p.right !== null && pindex < index) { + if (!p.deleted && p.countable) { + if (index < pindex + p.length) { + break + } + pindex += p.length + } + p = p.right + } + // iterate to left if necessary (might be that pindex > index) + while (p.left !== null && pindex > index) { + p = p.left + if (!p.deleted && p.countable) { + pindex -= p.length + } + } + // we want to make sure that p can't be merged with left, because that would screw up everything + // in that cas just return what we have (it is most likely the best marker anyway) + // iterate to left until p can't be merged with left + while (p.left !== null && p.left.id.client === p.id.client && p.left.id.clock + p.left.length === p.id.clock) { + p = p.left + if (!p.deleted && p.countable) { + pindex -= p.length + } + } + + // @todo remove! + // assure position + // { + // let start = yarray._start + // let pos = 0 + // while (start !== p) { + // if (!start.deleted && start.countable) { + // pos += start.length + // } + // start = /** @type {Item} */ (start.right) + // } + // if (pos !== pindex) { + // debugger + // throw new Error('Gotcha position fail!') + // } + // } + // if (marker) { + // if (window.lengthes == null) { + // window.lengthes = [] + // } + // window.lengthes.push(marker.index - pindex) + // console.log('distance', marker.index - pindex, 'len', p && p.parent.length) + // } + if (marker !== null && math.abs(marker.index - pindex) < 30) { + // adjust existing marker + overwriteMarker(marker, p, pindex) + return marker + } else { + // create new marker + return markPosition(yarray._searchMarker, p, pindex) + } +} + +/** + * Update markers when a change happened. + * + * This should be called before doing a deletion! + * + * @param {Array} searchMarker + * @param {number} index + * @param {number} len If insertion, len is positive. If deletion, len is negative. + */ +export const updateMarkerChanges = (searchMarker, index, len) => { + for (let i = searchMarker.length - 1; i >= 0; i--) { + const m = searchMarker[i] + if (len > 0) { + /** + * @type {Item|null} + */ + let p = m.p + p.marker = false + // Ideally we just want to do a simple position comparison, but this will only work if + // search markers don't point to deleted items for formats. + // Iterate marker to prev undeleted countable position so we know what to do when updating a position + while (p && (p.deleted || !p.countable)) { + p = p.left + if (p && !p.deleted && p.countable) { + // adjust position. the loop should break now + m.index -= p.length + } + } + if (p === null || p.marker === true) { + // remove search marker if updated position is null or if position is already marked + searchMarker.splice(i, 1) + continue + } + m.p = p + p.marker = true + } + if (index < m.index || (len > 0 && index === m.index)) { // a simple index <= m.index check would actually suffice + m.index = math.max(index, m.index + len) + } + } +} /** * Accumulate all (list) children of a type and return them as an Array. @@ -90,6 +280,10 @@ export class AbstractType { * @type {EventHandler,Transaction>} */ this._dEH = createEventHandler() + /** + * @type {null | Array} + */ + this._searchMarker = null } /** @@ -137,7 +331,11 @@ export class AbstractType { * @param {Transaction} transaction * @param {Set} parentSubs Keys changed on this type. `null` if list was modified. */ - _callObserver (transaction, parentSubs) { /* skip if no type is specified */ } + _callObserver (transaction, parentSubs) { + if (!transaction.local && this._searchMarker) { + this._searchMarker.length = 0 + } + } /** * Observe all events that are created on this type. @@ -353,7 +551,13 @@ export const typeListForEachSnapshot = (type, f, snapshot) => { * @function */ export const typeListGet = (type, index) => { - for (let n = type._start; n !== null; n = n.right) { + const marker = findMarker(type, index) + let n = type._start + if (marker !== null) { + n = marker.p + index -= marker.index + } + for (; n !== null; n = n.right) { if (!n.deleted && n.countable) { if (index < n.length) { return n.content.getContent()[index] @@ -430,9 +634,24 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem, */ export const typeListInsertGenerics = (transaction, parent, index, content) => { if (index === 0) { + if (parent._searchMarker) { + updateMarkerChanges(parent._searchMarker, index, content.length) + } return typeListInsertGenericsAfter(transaction, parent, null, content) } + const startIndex = index + const marker = findMarker(parent, index) let n = parent._start + if (marker !== null) { + n = marker.p + index -= marker.index + // we need to iterate one to the left so that the algorithm works + if (index === 0) { + // @todo refactor this as it actually doesn't consider formats + n = n.prev // important! get the left undeleted item so that we can actually decrease index + index += (n && n.countable && !n.deleted) ? n.length : 0 + } + } for (; n !== null; n = n.right) { if (!n.deleted && n.countable) { if (index <= n.length) { @@ -445,6 +664,9 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => { index -= n.length } } + if (parent._searchMarker) { + updateMarkerChanges(parent._searchMarker, startIndex, content.length) + } return typeListInsertGenericsAfter(transaction, parent, n, content) } @@ -459,7 +681,14 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => { */ export const typeListDelete = (transaction, parent, index, length) => { if (length === 0) { return } + const startIndex = index + const startLength = length + const marker = findMarker(parent, index) let n = parent._start + if (marker !== null) { + n = marker.p + index -= marker.index + } // compute the first item to be deleted for (; n !== null && index > 0; n = n.right) { if (!n.deleted && n.countable) { @@ -483,6 +712,9 @@ export const typeListDelete = (transaction, parent, index, length) => { if (length > 0) { throw error.create('array length exceeded') } + if (parent._searchMarker) { + updateMarkerChanges(parent._searchMarker, startIndex, -startLength + length /* in case we remove the above exception */) + } } /** diff --git a/src/types/YArray.js b/src/types/YArray.js index b6bd9252..b0f78fdb 100644 --- a/src/types/YArray.js +++ b/src/types/YArray.js @@ -15,7 +15,7 @@ import { YArrayRefID, callTypeObservers, transact, - AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line + ArraySearchMarker, AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line } from '../internals.js' /** @@ -47,6 +47,10 @@ export class YArray extends AbstractType { * @private */ this._prelimContent = [] + /** + * @type {Array} + */ + this._searchMarker = [] } /** @@ -80,6 +84,7 @@ export class YArray extends AbstractType { * @param {Set} parentSubs Keys changed on this type. `null` if list was modified. */ _callObserver (transaction, parentSubs) { + super._callObserver(transaction, parentSubs) callTypeObservers(this, transaction, new YArrayEvent(this, transaction)) } diff --git a/src/types/YText.js b/src/types/YText.js index 12d57bce..51eb02de 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -20,11 +20,14 @@ import { splitSnapshotAffectedStructs, iterateDeletedStructs, iterateStructs, - AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line + findMarker, + updateMarkerChanges, + ArraySearchMarker, AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line } from '../internals.js' import * as object from 'lib0/object.js' import * as map from 'lib0/map.js' +import * as error from 'lib0/error.js' /** * @param {any} a @@ -33,75 +36,79 @@ import * as map from 'lib0/map.js' */ const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b)) -export class ItemListPosition { - /** - * @param {Item|null} left - * @param {Item|null} right - */ - constructor (left, right) { - this.left = left - this.right = right - } -} - -export class ItemTextListPosition extends ItemListPosition { +export class ItemTextListPosition { /** * @param {Item|null} left * @param {Item|null} right + * @param {number} index * @param {Map} currentAttributes */ - constructor (left, right, currentAttributes) { - super(left, right) + constructor (left, right, index, currentAttributes) { + this.left = left + this.right = right + this.index = index this.currentAttributes = currentAttributes } -} -export class ItemInsertionResult extends ItemListPosition { /** - * @param {Item|null} left - * @param {Item|null} right - * @param {Map} negatedAttributes + * Only call this if you know that this.right is defined */ - constructor (left, right, negatedAttributes) { - super(left, right) - this.negatedAttributes = negatedAttributes + forward () { + if (this.right === null) { + error.unexpectedCase() + } + switch (this.right.content.constructor) { + case ContentEmbed: + case ContentString: + if (!this.right.deleted) { + this.index += this.right.length + } + break + case ContentFormat: + if (!this.right.deleted) { + updateCurrentAttributes(this.currentAttributes, /** @type {ContentFormat} */ (this.right.content)) + } + break + } + this.left = this.right + this.right = this.right.right } } /** * @param {Transaction} transaction - * @param {Map} currentAttributes - * @param {Item|null} left - * @param {Item|null} right - * @param {number} count + * @param {ItemTextListPosition} pos + * @param {number} count steps to move forward * @return {ItemTextListPosition} * * @private * @function */ -const findNextPosition = (transaction, currentAttributes, left, right, count) => { - while (right !== null && count > 0) { - switch (right.content.constructor) { +const findNextPosition = (transaction, pos, count) => { + while (pos.right !== null && count > 0) { + switch (pos.right.content.constructor) { case ContentEmbed: case ContentString: - if (!right.deleted) { - if (count < right.length) { + if (!pos.right.deleted) { + if (count < pos.right.length) { // split right - getItemCleanStart(transaction, createID(right.id.client, right.id.clock + count)) + getItemCleanStart(transaction, createID(pos.right.id.client, pos.right.id.clock + count)) } - count -= right.length + pos.index += pos.right.length + count -= pos.right.length } break case ContentFormat: - if (!right.deleted) { - updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content)) + if (!pos.right.deleted) { + updateCurrentAttributes(pos.currentAttributes, /** @type {ContentFormat} */ (pos.right.content)) } break } - left = right - right = right.right + pos.left = pos.right + pos.right = pos.right.right + // pos.forward() - we don't forward because that would halve the performance because we already do the checks above } - return new ItemTextListPosition(left, right, currentAttributes) + return pos } /** @@ -115,8 +122,14 @@ const findNextPosition = (transaction, currentAttributes, left, right, count) => */ const findPosition = (transaction, parent, index) => { const currentAttributes = new Map() - const right = parent._start - return findNextPosition(transaction, currentAttributes, null, right, index) + const marker = findMarker(parent, index) + if (marker) { + const pos = new ItemTextListPosition(marker.p.left, marker.p, marker.index, currentAttributes) + return findNextPosition(transaction, pos, index - marker.index) + } else { + const pos = new ItemTextListPosition(null, parent._start, 0, currentAttributes) + return findNextPosition(transaction, pos, index) + } } /** @@ -124,37 +137,35 @@ const findPosition = (transaction, parent, index) => { * * @param {Transaction} transaction * @param {AbstractType} parent - * @param {ItemListPosition} currPos + * @param {ItemTextListPosition} currPos * @param {Map} negatedAttributes * * @private * @function */ const insertNegatedAttributes = (transaction, parent, currPos, negatedAttributes) => { - let { left, right } = currPos // check if we really need to remove attributes while ( - right !== null && ( - right.deleted === true || ( - right.content.constructor === ContentFormat && - equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key), /** @type {ContentFormat} */ (right.content).value) + currPos.right !== null && ( + currPos.right.deleted === true || ( + currPos.right.content.constructor === ContentFormat && + equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (currPos.right.content).key), /** @type {ContentFormat} */ (currPos.right.content).value) ) ) ) { - if (!right.deleted) { - negatedAttributes.delete(/** @type {ContentFormat} */ (right.content).key) + if (!currPos.right.deleted) { + negatedAttributes.delete(/** @type {ContentFormat} */ (currPos.right.content).key) } - left = right - right = right.right + currPos.forward() } const doc = transaction.doc const ownClientId = doc.clientID + let left = currPos.left + const right = currPos.right negatedAttributes.forEach((val, key) => { left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val)) left.integrate(transaction, 0) }) - currPos.left = left - currPos.right = right } /** @@ -174,59 +185,51 @@ const updateCurrentAttributes = (currentAttributes, format) => { } /** - * @param {ItemListPosition} currPos - * @param {Map} currentAttributes + * @param {ItemTextListPosition} currPos * @param {Object} attributes * * @private * @function */ -const minimizeAttributeChanges = (currPos, currentAttributes, attributes) => { +const minimizeAttributeChanges = (currPos, attributes) => { // go right while attributes[right.key] === right.value (or right is deleted) - let { left, right } = currPos while (true) { - if (right === null) { + if (currPos.right === null) { break - } else if (right.deleted) { - // continue - } else if (right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (right.content)).key] || null, /** @type {ContentFormat} */ (right.content).value)) { - // found a format, update currentAttributes and continue - updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content)) + } else if (currPos.right.deleted || (currPos.right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (currPos.right.content)).key] || null, /** @type {ContentFormat} */ (currPos.right.content).value))) { + // } else { break } - left = right - right = right.right + currPos.forward() } - currPos.left = left - currPos.right = right } /** * @param {Transaction} transaction * @param {AbstractType} parent - * @param {ItemListPosition} currPos - * @param {Map} currentAttributes + * @param {ItemTextListPosition} currPos * @param {Object} attributes * @return {Map} * * @private * @function **/ -const insertAttributes = (transaction, parent, currPos, currentAttributes, attributes) => { +const insertAttributes = (transaction, parent, currPos, attributes) => { const doc = transaction.doc const ownClientId = doc.clientID const negatedAttributes = new Map() // insert format-start items for (const key in attributes) { const val = attributes[key] - const currentVal = currentAttributes.get(key) || null + const currentVal = currPos.currentAttributes.get(key) || null if (!equalAttrs(currentVal, val)) { // save negated attribute (set null if currentVal undefined) negatedAttributes.set(key, currentVal) const { left, right } = currPos - currPos.left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val)) - currPos.left.integrate(transaction, 0) + currPos.right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val)) + currPos.right.integrate(transaction, 0) + currPos.forward() } } return negatedAttributes @@ -235,56 +238,59 @@ const insertAttributes = (transaction, parent, currPos, currentAttributes, attri /** * @param {Transaction} transaction * @param {AbstractType} parent - * @param {ItemListPosition} currPos - * @param {Map} currentAttributes + * @param {ItemTextListPosition} currPos * @param {string|object} text * @param {Object} attributes * * @private * @function **/ -const insertText = (transaction, parent, currPos, currentAttributes, text, attributes) => { - currentAttributes.forEach((val, key) => { +const insertText = (transaction, parent, currPos, text, attributes) => { + currPos.currentAttributes.forEach((val, key) => { if (attributes[key] === undefined) { attributes[key] = null } }) const doc = transaction.doc const ownClientId = doc.clientID - minimizeAttributeChanges(currPos, currentAttributes, attributes) - const negatedAttributes = insertAttributes(transaction, parent, currPos, currentAttributes, attributes) + minimizeAttributeChanges(currPos, attributes) + const negatedAttributes = insertAttributes(transaction, parent, currPos, attributes) // insert content const content = text.constructor === String ? new ContentString(/** @type {string} */ (text)) : new ContentEmbed(text) - const { left, right } = currPos - currPos.left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content) - currPos.left.integrate(transaction, 0) - return insertNegatedAttributes(transaction, parent, currPos, negatedAttributes) + let { left, right, index } = currPos + if (parent._searchMarker) { + updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength()) + } + right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content) + right.integrate(transaction, 0) + currPos.right = right + currPos.index = index + currPos.forward() + insertNegatedAttributes(transaction, parent, currPos, negatedAttributes) } /** * @param {Transaction} transaction * @param {AbstractType} parent - * @param {ItemListPosition} currPos - * @param {Map} currentAttributes + * @param {ItemTextListPosition} currPos * @param {number} length * @param {Object} attributes * * @private * @function */ -const formatText = (transaction, parent, currPos, currentAttributes, length, attributes) => { +const formatText = (transaction, parent, currPos, length, attributes) => { const doc = transaction.doc const ownClientId = doc.clientID - minimizeAttributeChanges(currPos, currentAttributes, attributes) - const negatedAttributes = insertAttributes(transaction, parent, currPos, currentAttributes, attributes) - let { left, right } = currPos + minimizeAttributeChanges(currPos, attributes) + const negatedAttributes = insertAttributes(transaction, parent, currPos, attributes) // iterate until first non-format or null is found // delete all formats with attributes[format.key] != null - while (length > 0 && right !== null) { - if (!right.deleted) { - switch (right.content.constructor) { + while (length > 0 && currPos.right !== null) { + if (!currPos.right.deleted) { + switch (currPos.right.content.constructor) { case ContentFormat: { - const { key, value } = /** @type {ContentFormat} */ (right.content) + const { key, value } = /** @type {ContentFormat} */ (currPos.right.content) const attr = attributes[key] if (attr !== undefined) { if (equalAttrs(attr, value)) { @@ -292,22 +298,20 @@ const formatText = (transaction, parent, currPos, currentAttributes, length, att } else { negatedAttributes.set(key, value) } - right.delete(transaction) + currPos.right.delete(transaction) } - updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content)) break } case ContentEmbed: case ContentString: - if (length < right.length) { - getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length)) + if (length < currPos.right.length) { + getItemCleanStart(transaction, createID(currPos.right.id.client, currPos.right.id.clock + length)) } - length -= right.length + length -= currPos.right.length break } } - left = right - right = right.right + currPos.forward() } // Quill just assumes that the editor starts with a newline and that it always // ends with a newline. We only insert that newline when a new newline is @@ -317,11 +321,10 @@ const formatText = (transaction, parent, currPos, currentAttributes, length, att for (; length > 0; length--) { newlines += '\n' } - left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentString(newlines)) - left.integrate(transaction, 0) + currPos.right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), currPos.left, currPos.left && currPos.left.lastId, currPos.right, currPos.right && currPos.right.id, parent, null, new ContentString(newlines)) + currPos.right.integrate(transaction, 0) + currPos.forward() } - currPos.left = left - currPos.right = right insertNegatedAttributes(transaction, parent, currPos, negatedAttributes) } @@ -431,42 +434,39 @@ export const cleanupYTextFormatting = type => { /** * @param {Transaction} transaction - * @param {ItemListPosition} currPos - * @param {Map} currentAttributes + * @param {ItemTextListPosition} currPos * @param {number} length - * @return {ItemListPosition} + * @return {ItemTextListPosition} * * @private * @function */ -const deleteText = (transaction, currPos, currentAttributes, length) => { - const startAttrs = map.copy(currentAttributes) +const deleteText = (transaction, currPos, length) => { + const startLength = length + const startAttrs = map.copy(currPos.currentAttributes) const start = currPos.right - let { left, right } = currPos - while (length > 0 && right !== null) { - if (right.deleted === false) { - switch (right.content.constructor) { - case ContentFormat: - updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content)) - break + while (length > 0 && currPos.right !== null) { + if (currPos.right.deleted === false) { + switch (currPos.right.content.constructor) { case ContentEmbed: case ContentString: - if (length < right.length) { - getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length)) + if (length < currPos.right.length) { + getItemCleanStart(transaction, createID(currPos.right.id.client, currPos.right.id.clock + length)) } - length -= right.length - right.delete(transaction) + length -= currPos.right.length + currPos.right.delete(transaction) break } } - left = right - right = right.right + currPos.forward() } if (start) { - cleanupFormattingGap(transaction, start, right, startAttrs, map.copy(currentAttributes)) + cleanupFormattingGap(transaction, start, currPos.right, startAttrs, map.copy(currPos.currentAttributes)) + } + const parent = /** @type {AbstractType} */ (/** @type {Item} */ (currPos.left || currPos.right).parent) + if (parent._searchMarker) { + updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length) } - currPos.left = left - currPos.right = right return currPos } @@ -729,6 +729,10 @@ export class YText extends AbstractType { * @type {Array?} */ this._pending = string !== undefined ? [() => this.insert(0, string)] : [] + /** + * @type {Array} + */ + this._searchMarker = [] } /** @@ -765,6 +769,7 @@ export class YText extends AbstractType { * @param {Set} parentSubs Keys changed on this type. `null` if list was modified. */ _callObserver (transaction, parentSubs) { + super._callObserver(transaction, parentSubs) const event = new YTextEvent(this, transaction) const doc = transaction.doc // If a remote change happened, we try to cleanup potential formatting duplicates. @@ -778,7 +783,7 @@ export class YText extends AbstractType { } iterateStructs(transaction, /** @type {Array} */ (doc.store.clients.get(client)), clock, afterClock, item => { // @ts-ignore - if (!item.deleted && item.content.constructor === ContentFormat) { + if (item.content.constructor === ContentFormat) { foundFormattingItem = true } }) @@ -786,7 +791,17 @@ export class YText extends AbstractType { break } } - transact(doc, t => { + if (!foundFormattingItem) { + iterateDeletedStructs(transaction, transaction.deleteSet, item => { + if (item instanceof GC || foundFormattingItem) { + return + } + if (item.parent === this && item.content.constructor === ContentFormat) { + foundFormattingItem = true + } + }) + } + transact(doc, (t) => { if (foundFormattingItem) { // If a formatting item was inserted, we simply clean the whole type. // We need to compute currentAttributes for the current position anyway. @@ -795,7 +810,7 @@ export class YText extends AbstractType { // If no formatting attribute was inserted, we can make due with contextless // formatting cleanups. // Contextless: it is not necessary to compute currentAttributes for the affected position. - iterateDeletedStructs(t, transaction.deleteSet, item => { + iterateDeletedStructs(t, t.deleteSet, item => { if (item instanceof GC) { return } @@ -852,11 +867,7 @@ export class YText extends AbstractType { applyDelta (delta, { sanitize = true } = {}) { if (this.doc !== null) { transact(this.doc, transaction => { - /** - * @type {ItemListPosition} - */ - const currPos = new ItemListPosition(null, this._start) - const currentAttributes = new Map() + const currPos = new ItemTextListPosition(null, this._start, 0, new Map()) for (let i = 0; i < delta.length; i++) { const op = delta[i] if (op.insert !== undefined) { @@ -867,12 +878,12 @@ export class YText extends AbstractType { // paragraphs, but nothing bad will happen. const ins = (!sanitize && typeof op.insert === 'string' && i === delta.length - 1 && currPos.right === null && op.insert.slice(-1) === '\n') ? op.insert.slice(0, -1) : op.insert if (typeof ins !== 'string' || ins.length > 0) { - insertText(transaction, this, currPos, currentAttributes, ins, op.attributes || {}) + insertText(transaction, this, currPos, ins, op.attributes || {}) } } else if (op.retain !== undefined) { - formatText(transaction, this, currPos, currentAttributes, op.retain, op.attributes || {}) + formatText(transaction, this, currPos, op.retain, op.attributes || {}) } else if (op.delete !== undefined) { - deleteText(transaction, currPos, currentAttributes, op.delete) + deleteText(transaction, currPos, op.delete) } } }) @@ -1004,13 +1015,13 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, this, index) + const pos = findPosition(transaction, this, index) if (!attributes) { attributes = {} // @ts-ignore - currentAttributes.forEach((v, k) => { attributes[k] = v }) + pos.currentAttributes.forEach((v, k) => { attributes[k] = v }) } - insertText(transaction, this, new ItemListPosition(left, right), currentAttributes, text, attributes) + insertText(transaction, this, pos, text, attributes) }) } else { /** @type {Array} */ (this._pending).push(() => this.insert(index, text, attributes)) @@ -1034,8 +1045,8 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, this, index) - insertText(transaction, this, new ItemListPosition(left, right), currentAttributes, embed, attributes) + const pos = findPosition(transaction, this, index) + insertText(transaction, this, pos, embed, attributes) }) } else { /** @type {Array} */ (this._pending).push(() => this.insertEmbed(index, embed, attributes)) @@ -1057,8 +1068,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, this, index) - deleteText(transaction, new ItemListPosition(left, right), currentAttributes, length) + deleteText(transaction, findPosition(transaction, this, index), length) }) } else { /** @type {Array} */ (this._pending).push(() => this.delete(index, length)) @@ -1082,11 +1092,11 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, this, index) - if (right === null) { + const pos = findPosition(transaction, this, index) + if (pos.right === null) { return } - formatText(transaction, this, new ItemListPosition(left, right), currentAttributes, length, attributes) + formatText(transaction, this, pos, length, attributes) }) } else { /** @type {Array} */ (this._pending).push(() => this.format(index, length, attributes)) diff --git a/src/utils/logging.js b/src/utils/logging.js new file mode 100644 index 00000000..37709763 --- /dev/null +++ b/src/utils/logging.js @@ -0,0 +1,22 @@ + +import { + AbstractType // eslint-disable-line +} from '../internals.js' + +/** + * Convenient helper to log type information. + * + * Do not use in productive systems as the output can be immense! + * + * @param {AbstractType} type + */ +export const logType = type => { + const res = [] + let n = type._start + while (n) { + res.push(n) + n = n.right + } + console.log('Children: ', res) + console.log('Children content: ', res.filter(m => !m.deleted).map(m => m.content)) +} diff --git a/tests/testHelper.js b/tests/testHelper.js index 5a5e0eb2..2409af79 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -1,13 +1,3 @@ -import * as Y from '../src/index.js' - -import { - createDeleteSetFromStructStore, - getStateVector, - Item, - useV1Encoding, - useV2Encoding, - DeleteItem, DeleteSet, StructStore, Doc // eslint-disable-line -} from '../src/internals.js' import * as t from 'lib0/testing.js' import * as prng from 'lib0/prng.js' @@ -15,8 +5,14 @@ import * as encoding from 'lib0/encoding.js' import * as decoding from 'lib0/decoding.js' import * as syncProtocol from 'y-protocols/sync.js' import * as object from 'lib0/object.js' +import * as Y from '../src/internals.js' export * from '../src/internals.js' +if (typeof window !== 'undefined') { + // @ts-ignore + window.Y = Y // eslint-disable-line +} + /** * @param {TestYInstance} y // publish message created by `y` to all other online clients * @param {Uint8Array} m @@ -31,7 +27,7 @@ const broadcastMessage = (y, m) => { } } -export class TestYInstance extends Doc { +export class TestYInstance extends Y.Doc { /** * @param {TestConnector} testConnector * @param {number} clientID @@ -232,7 +228,7 @@ export class TestConnector { * @param {t.TestCase} tc * @param {{users?:number}} conf * @param {InitTestObjectCallback} [initTestObject] - * @return {{testObjects:Array,testConnector:TestConnector,users:Array,array0:Y.Array,array1:Y.Array,array2:Y.Array,map0:Y.Map,map1:Y.Map,map2:Y.Map,map3:Y.Map,text0:Y.Text,text1:Y.Text,text2:Y.Text,xml0:Y.XmlElement,xml1:Y.XmlElement,xml2:Y.XmlElement}} + * @return {{testObjects:Array,testConnector:TestConnector,users:Array,array0:Y.YArray,array1:Y.YArray,array2:Y.YArray,map0:Y.YMap,map1:Y.YMap,map2:Y.YMap,map3:Y.YMap,text0:Y.YText,text1:Y.YText,text2:Y.YText,xml0:Y.YXmlElement,xml1:Y.YXmlElement,xml2:Y.YXmlElement}} */ export const init = (tc, { users = 5 } = {}, initTestObject) => { /** @@ -244,9 +240,9 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => { const gen = tc.prng // choose an encoding approach at random if (prng.bool(gen)) { - useV2Encoding() + Y.useV2Encoding() } else { - useV1Encoding() + Y.useV1Encoding() } const testConnector = new TestConnector(gen) @@ -255,14 +251,14 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => { const y = testConnector.createY(i) y.clientID = i result.users.push(y) - result['array' + i] = y.get('array', Y.Array) - result['map' + i] = y.get('map', Y.Map) - result['xml' + i] = y.get('xml', Y.XmlElement) - result['text' + i] = y.get('text', Y.Text) + result['array' + i] = y.getArray('array') + result['map' + i] = y.getMap('map') + result['xml' + i] = y.get('xml', Y.YXmlElement) + result['text' + i] = y.getText('text') } testConnector.syncAll() result.testObjects = result.users.map(initTestObject || (() => null)) - useV1Encoding() + Y.useV1Encoding() return /** @type {any} */ (result) } @@ -280,7 +276,7 @@ export const compare = users => { while (users[0].tc.flushAllMessages()) {} const userArrayValues = users.map(u => u.getArray('array').toJSON()) const userMapValues = users.map(u => u.getMap('map').toJSON()) - const userXmlValues = users.map(u => u.get('xml', Y.XmlElement).toString()) + const userXmlValues = users.map(u => u.get('xml', Y.YXmlElement).toString()) const userTextValues = users.map(u => u.getText('text').toDelta()) for (const u of users) { t.assert(u.store.pendingDeleteReaders.length === 0) @@ -309,23 +305,23 @@ export const compare = users => { t.compare(userXmlValues[i], userXmlValues[i + 1]) t.compare(userTextValues[i].map(/** @param {any} a */ a => typeof a.insert === 'string' ? a.insert : ' ').join('').length, users[i].getText('text').length) t.compare(userTextValues[i], userTextValues[i + 1]) - t.compare(getStateVector(users[i].store), getStateVector(users[i + 1].store)) - compareDS(createDeleteSetFromStructStore(users[i].store), createDeleteSetFromStructStore(users[i + 1].store)) + t.compare(Y.getStateVector(users[i].store), Y.getStateVector(users[i + 1].store)) + compareDS(Y.createDeleteSetFromStructStore(users[i].store), Y.createDeleteSetFromStructStore(users[i + 1].store)) compareStructStores(users[i].store, users[i + 1].store) } users.map(u => u.destroy()) } /** - * @param {Item?} a - * @param {Item?} b + * @param {Y.Item?} a + * @param {Y.Item?} b * @return {boolean} */ export const compareItemIDs = (a, b) => a === b || (a !== null && b != null && Y.compareIDs(a.id, b.id)) /** - * @param {StructStore} ss1 - * @param {StructStore} ss2 + * @param {Y.StructStore} ss1 + * @param {Y.StructStore} ss2 */ export const compareStructStores = (ss1, ss2) => { t.assert(ss1.clients.size === ss2.clients.size) @@ -345,9 +341,9 @@ export const compareStructStores = (ss1, ss2) => { ) { t.fail('Structs dont match') } - if (s1 instanceof Item) { + if (s1 instanceof Y.Item) { if ( - !(s2 instanceof Item) || + !(s2 instanceof Y.Item) || !((s1.left === null && s2.left === null) || (s1.left !== null && s2.left !== null && Y.compareIDs(s1.left.lastId, s2.left.lastId))) || !compareItemIDs(s1.right, s2.right) || !Y.compareIDs(s1.origin, s2.origin) || @@ -367,13 +363,13 @@ export const compareStructStores = (ss1, ss2) => { } /** - * @param {DeleteSet} ds1 - * @param {DeleteSet} ds2 + * @param {Y.DeleteSet} ds1 + * @param {Y.DeleteSet} ds2 */ export const compareDS = (ds1, ds2) => { t.assert(ds1.clients.size === ds2.clients.size) ds1.clients.forEach((deleteItems1, client) => { - const deleteItems2 = /** @type {Array} */ (ds2.clients.get(client)) + const deleteItems2 = /** @type {Array} */ (ds2.clients.get(client)) t.assert(deleteItems2 !== undefined && deleteItems1.length === deleteItems2.length) for (let i = 0; i < deleteItems1.length; i++) { const di1 = deleteItems1[i] diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index e06a8a60..724b50fc 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -352,7 +352,10 @@ const arrayTransactions = [ content.push(uniqueNumber) } var pos = prng.int32(gen, 0, yarray.length) + const oldContent = yarray.toArray() yarray.insert(pos, content) + oldContent.splice(pos, 0, ...content) + t.compareArrays(yarray.toArray(), oldContent) // we want to make sure that fastSearch markers insert at the correct position }, function insertTypeArray (user, gen) { const yarray = user.getArray('array') @@ -384,7 +387,10 @@ const arrayTransactions = [ type.delete(somePos, delLength) } } else { + const oldContent = yarray.toArray() yarray.delete(somePos, delLength) + oldContent.splice(somePos, delLength) + t.compareArrays(yarray.toArray(), oldContent) } } } @@ -393,8 +399,8 @@ const arrayTransactions = [ /** * @param {t.TestCase} tc */ -export const testRepeatGeneratingYarrayTests4 = tc => { - applyRandomTests(tc, arrayTransactions, 4) +export const testRepeatGeneratingYarrayTests6 = tc => { + applyRandomTests(tc, arrayTransactions, 6) } /** diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index 6d5a3db6..396e2a13 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -205,6 +205,50 @@ export const testFormattingRemovedInMidText = tc => { t.assert(Y.getTypeChildren(text0).length === 3) } +/** + * @param {t.TestCase} tc + */ +export const testInsertAndDeleteAtRandomPositions = tc => { + const N = 10000 + const { text0 } = init(tc, { users: 1 }) + const gen = tc.prng + + // create initial content + // let expectedResult = init + text0.insert(0, prng.word(gen, N / 2, N / 2)) + + // apply changes + for (let i = 0; i < N; i++) { + const pos = prng.uint32(gen, 0, text0.length) + if (prng.bool(gen)) { + const len = prng.uint32(gen, 1, 5) + const word = prng.word(gen, 0, len) + text0.insert(pos, word) + // expectedResult = expectedResult.slice(0, pos) + word + expectedResult.slice(pos) + } else { + const len = prng.uint32(gen, 0, math.min(3, text0.length - pos)) + text0.delete(pos, len) + // expectedResult = expectedResult.slice(0, pos) + expectedResult.slice(pos + len) + } + } + // t.compareStrings(text0.toString(), expectedResult) + t.describe('final length', '' + text0.length) +} + +/** + * @param {t.TestCase} tc + */ +export const testAppendChars = tc => { + const N = 10000 + const { text0 } = init(tc, { users: 1 }) + + // apply changes + for (let i = 0; i < N; i++) { + text0.insert(text0.length, 'a') + } + t.assert(text0.length === N) +} + const id = Y.createID(0, 0) const c = new Y.ContentString('a') @@ -281,6 +325,102 @@ export const testLargeFragmentedDocument = tc => { let charCounter = 0 +/** + * Random tests for pure text operations without formatting. + * + * @type Array + */ +const textChanges = [ + /** + * @param {Y.Doc} y + * @param {prng.PRNG} gen + */ + (y, gen) => { // insert text + const ytext = y.getText('text') + const insertPos = prng.int32(gen, 0, ytext.length) + const text = charCounter++ + prng.word(gen) + const prevText = ytext.toString() + ytext.insert(insertPos, text) + t.compareStrings(ytext.toString(), prevText.slice(0, insertPos) + text + prevText.slice(insertPos)) + }, + /** + * @param {Y.Doc} y + * @param {prng.PRNG} gen + */ + (y, gen) => { // delete text + const ytext = y.getText('text') + const contentLen = ytext.toString().length + const insertPos = prng.int32(gen, 0, contentLen) + const overwrite = math.min(prng.int32(gen, 0, contentLen - insertPos), 2) + const prevText = ytext.toString() + ytext.delete(insertPos, overwrite) + t.compareStrings(ytext.toString(), prevText.slice(0, insertPos) + prevText.slice(insertPos + overwrite)) + } +] + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges5 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 5)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges30 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 30)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges40 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 40)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges50 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 50)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges70 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 70)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges90 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 90)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatGenerateTextChanges300 = tc => { + const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 300)) + const cleanups = Y.cleanupYTextFormatting(users[0].getText('text')) + t.assert(cleanups === 0) +} + const marks = [ { bold: true }, { italic: true }, @@ -293,6 +433,8 @@ const marksChoices = [ ] /** + * Random tests for all features of y-text (formatting, embeds, ..). + * * @type Array */ const qChanges = [ @@ -302,7 +444,7 @@ const qChanges = [ */ (y, gen) => { // insert text const ytext = y.getText('text') - const insertPos = prng.int32(gen, 0, ytext.toString().length) + const insertPos = prng.int32(gen, 0, ytext.length) const attrs = prng.oneOf(gen, marksChoices) const text = charCounter++ + prng.word(gen) ytext.insert(insertPos, text, attrs) @@ -313,7 +455,7 @@ const qChanges = [ */ (y, gen) => { // insert embed const ytext = y.getText('text') - const insertPos = prng.int32(gen, 0, ytext.toString().length) + const insertPos = prng.int32(gen, 0, ytext.length) ytext.insertEmbed(insertPos, { image: 'https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png' }) }, /**