yonote/yjs
23
0
Fork 0
Code Pull Requests 1 Releases Activity

implement search-marker prototype (limited usage for now)

Browse Source
This commit is contained in:
Kevin Jahns 2020-07-15 22:03:02 +02:00
parent 6e8167fe51
commit 6e3b708599
11 changed files with 628 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/index.js
View File

@ -69,5 +69,6 @@ export {
PermanentUserData, // @TODO experimental
tryGc,
transact,
AbstractConnector
AbstractConnector,
logType
} from './internals.js'

1
src/internals.js
View File

@ -8,6 +8,7 @@ export * from './utils/encoding.js'
export * from './utils/EventHandler.js'
export * from './utils/ID.js'
export * from './utils/isParentOf.js'
export * from './utils/logging.js'
export * from './utils/PermanentUserData.js'
export * from './utils/RelativePosition.js'
export * from './utils/Snapshot.js'

8
src/structs/ContentFormat.js
View File

@ -1,6 +1,6 @@
import {
AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line
AbstractType, AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line
} from '../internals.js'
import * as error from 'lib0/error.js'
@ -66,7 +66,11 @@ export class ContentFormat {
* @param {Transaction} transaction
* @param {Item} item
*/
integrate (transaction, item) {}
integrate (transaction, item) {
// @todo searchmarker are currently unsupported for rich text documents
/** @type {AbstractType<any>} */ (item.parent)._searchMarker = null
}
/**
* @param {Transaction} transaction
*/

22
src/structs/Item.js
View File

@ -285,9 +285,31 @@ export class Item extends AbstractStruct {
* @type {AbstractContent}
*/
this.content = content
/**
* bit1: keep
* bit2: countable
* bit3: deleted
* bit4: mark - mark node as fast-search-marker
* @type {number} byte
*/
this.info = this.content.isCountable() ? binary.BIT2 : 0
}
/**
* This is used to mark the item as an indexed fast-search marker
*
* @type {boolean}
*/
set marker (isMarked) {
if (((this.info & binary.BIT4) > 0) !== isMarked) {
this.info ^= binary.BIT4
}
}
get marker () {
return (this.info & binary.BIT4) > 0
}
/**
* If true, do not garbage collect this Item.
*/

236
src/types/AbstractType.js
View File

@ -17,6 +17,196 @@ import {
import * as map from 'lib0/map.js'
import * as iterator from 'lib0/iterator.js'
import * as error from 'lib0/error.js'
import * as math from 'lib0/math.js'
const maxSearchMarker = 60
/**
* A unique timestamp that identifies each marker.
*
* Time is relative,.. this is more like an ever-increasing clock.
*
* @type {number}
*/
let globalSearchMarkerTimestamp = 0
export class ArraySearchMarker {
/**
* @param {Item} p
* @param {number} index
*/
constructor (p, index) {
p.marker = true
this.p = p
this.index = index
this.timestamp = globalSearchMarkerTimestamp++
}
}
/**
* @param {ArraySearchMarker} marker
*/
const refreshMarkerTimestamp = marker => { marker.timestamp = globalSearchMarkerTimestamp++ }
/**
* This is rather complex so this function is the only thing that should overwrite a marker
*
* @param {ArraySearchMarker} marker
* @param {Item} p
* @param {number} index
*/
const overwriteMarker = (marker, p, index) => {
marker.p.marker = false
marker.p = p
p.marker = true
marker.index = index
marker.timestamp = globalSearchMarkerTimestamp++
}
/**
* @param {Array<ArraySearchMarker>} searchMarker
* @param {Item} p
* @param {number} index
*/
const markPosition = (searchMarker, p, index) => {
if (searchMarker.length >= maxSearchMarker) {
// override oldest marker (we don't want to create more objects)
const marker = searchMarker.reduce((a, b) => a.timestamp < b.timestamp ? a : b)
overwriteMarker(marker, p, index)
return marker
} else {
// create new marker
const pm = new ArraySearchMarker(p, index)
searchMarker.push(pm)
return pm
}
}
/**
* Search marker help us to find positions in the associative array faster.
*
* They speed up the process of finding a position without much bookkeeping.
*
* A maximum of `maxSearchMarker` objects are created.
*
* This function always returns a refreshed marker (updated timestamp)
*
* @param {AbstractType<any>} yarray
* @param {number} index
*/
export const findMarker = (yarray, index) => {
if (yarray._start === null || index === 0 || yarray._searchMarker === null) {
return null
}
const marker = yarray._searchMarker.length === 0 ? null : yarray._searchMarker.reduce((a, b) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b)
let p = yarray._start
let pindex = 0
if (marker !== null) {
p = marker.p
pindex = marker.index
refreshMarkerTimestamp(marker) // we used it, we might need to use it again
}
// iterate to right if possible
while (p.right !== null && pindex < index) {
if (!p.deleted && p.countable) {
if (index < pindex + p.length) {
break
}
pindex += p.length
}
p = p.right
}
// iterate to left if necessary (might be that pindex > index)
while (p.left !== null && pindex > index) {
p = p.left
if (!p.deleted && p.countable) {
pindex -= p.length
}
}
// we want to make sure that p can't be merged with left, because that would screw up everything
// in that cas just return what we have (it is most likely the best marker anyway)
// iterate to left until p can't be merged with left
while (p.left !== null && p.left.id.client === p.id.client && p.left.id.clock + p.left.length === p.id.clock) {
p = p.left
if (!p.deleted && p.countable) {
pindex -= p.length
}
}
// @todo remove!
// assure position
// {
// let start = yarray._start
// let pos = 0
// while (start !== p) {
// if (!start.deleted && start.countable) {
// pos += start.length
// }
// start = /** @type {Item} */ (start.right)
// }
// if (pos !== pindex) {
// debugger
// throw new Error('Gotcha position fail!')
// }
// }
// if (marker) {
// if (window.lengthes == null) {
// window.lengthes = []
// }
// window.lengthes.push(marker.index - pindex)
// console.log('distance', marker.index - pindex, 'len', p && p.parent.length)
// }
if (marker !== null && math.abs(marker.index - pindex) < 30) {
// adjust existing marker
overwriteMarker(marker, p, pindex)
return marker
} else {
// create new marker
return markPosition(yarray._searchMarker, p, pindex)
}
}
/**
* Update markers when a change happened.
*
* This should be called before doing a deletion!
*
* @param {Array<ArraySearchMarker>} searchMarker
* @param {number} index
* @param {number} len If insertion, len is positive. If deletion, len is negative.
*/
export const updateMarkerChanges = (searchMarker, index, len) => {
for (let i = searchMarker.length - 1; i >= 0; i--) {
const m = searchMarker[i]
if (len > 0) {
/**
* @type {Item|null}
*/
let p = m.p
p.marker = false
// Ideally we just want to do a simple position comparison, but this will only work if
// search markers don't point to deleted items for formats.
// Iterate marker to prev undeleted countable position so we know what to do when updating a position
while (p && (p.deleted || !p.countable)) {
p = p.left
if (p && !p.deleted && p.countable) {
// adjust position. the loop should break now
m.index -= p.length
}
}
if (p === null || p.marker === true) {
// remove search marker if updated position is null or if position is already marked
searchMarker.splice(i, 1)
continue
}
m.p = p
p.marker = true
}
if (index < m.index || (len > 0 && index === m.index)) { // a simple index <= m.index check would actually suffice
m.index = math.max(index, m.index + len)
}
}
}
/**
* Accumulate all (list) children of a type and return them as an Array.
@ -90,6 +280,10 @@ export class AbstractType {
* @type {EventHandler<Array<YEvent>,Transaction>}
*/
this._dEH = createEventHandler()
/**
* @type {null | Array<ArraySearchMarker>}
*/
this._searchMarker = null
}
/**
@ -137,7 +331,11 @@ export class AbstractType {
* @param {Transaction} transaction
* @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
*/
_callObserver (transaction, parentSubs) { /* skip if no type is specified */ }
_callObserver (transaction, parentSubs) {
if (!transaction.local && this._searchMarker) {
this._searchMarker.length = 0
}
}
/**
* Observe all events that are created on this type.
@ -353,7 +551,13 @@ export const typeListForEachSnapshot = (type, f, snapshot) => {
* @function
*/
export const typeListGet = (type, index) => {
for (let n = type._start; n !== null; n = n.right) {
const marker = findMarker(type, index)
let n = type._start
if (marker !== null) {
n = marker.p
index -= marker.index
}
for (; n !== null; n = n.right) {
if (!n.deleted && n.countable) {
if (index < n.length) {
return n.content.getContent()[index]
@ -430,9 +634,24 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem,
*/
export const typeListInsertGenerics = (transaction, parent, index, content) => {
if (index === 0) {
if (parent._searchMarker) {
updateMarkerChanges(parent._searchMarker, index, content.length)
}
return typeListInsertGenericsAfter(transaction, parent, null, content)
}
const startIndex = index
const marker = findMarker(parent, index)
let n = parent._start
if (marker !== null) {
n = marker.p
index -= marker.index
// we need to iterate one to the left so that the algorithm works
if (index === 0) {
// @todo refactor this as it actually doesn't consider formats
n = n.prev // important! get the left undeleted item so that we can actually decrease index
index += (n && n.countable && !n.deleted) ? n.length : 0
}
}
for (; n !== null; n = n.right) {
if (!n.deleted && n.countable) {
if (index <= n.length) {
@ -445,6 +664,9 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => {
index -= n.length
}
}
if (parent._searchMarker) {
updateMarkerChanges(parent._searchMarker, startIndex, content.length)
}
return typeListInsertGenericsAfter(transaction, parent, n, content)
}
@ -459,7 +681,14 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => {
*/
export const typeListDelete = (transaction, parent, index, length) => {
if (length === 0) { return }
const startIndex = index
const startLength = length
const marker = findMarker(parent, index)
let n = parent._start
if (marker !== null) {
n = marker.p
index -= marker.index
}
// compute the first item to be deleted
for (; n !== null && index > 0; n = n.right) {
if (!n.deleted && n.countable) {
@ -483,6 +712,9 @@ export const typeListDelete = (transaction, parent, index, length) => {
if (length > 0) {
throw error.create('array length exceeded')
}
if (parent._searchMarker) {
updateMarkerChanges(parent._searchMarker, startIndex, -startLength + length /* in case we remove the above exception */)
}
}
/**

7
src/types/YArray.js
View File

@ -15,7 +15,7 @@ import {
YArrayRefID,
callTypeObservers,
transact,
AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line
ArraySearchMarker, AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line
} from '../internals.js'
/**
@ -47,6 +47,10 @@ export class YArray extends AbstractType {
* @private
*/
this._prelimContent = []
/**
* @type {Array<ArraySearchMarker>}
*/
this._searchMarker = []
}
/**
@ -80,6 +84,7 @@ export class YArray extends AbstractType {
* @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
*/
_callObserver (transaction, parentSubs) {
super._callObserver(transaction, parentSubs)
callTypeObservers(this, transaction, new YArrayEvent(this, transaction))
}

302
src/types/YText.js
View File

@ -20,11 +20,14 @@ import {
splitSnapshotAffectedStructs,
iterateDeletedStructs,
iterateStructs,
AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
findMarker,
updateMarkerChanges,
ArraySearchMarker, AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
} from '../internals.js'
import * as object from 'lib0/object.js'
import * as map from 'lib0/map.js'
import * as error from 'lib0/error.js'
/**
* @param {any} a
@ -33,75 +36,79 @@ import * as map from 'lib0/map.js'
*/
const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b))
export class ItemListPosition {
/**
* @param {Item|null} left
* @param {Item|null} right
*/
constructor (left, right) {
this.left = left
this.right = right
}
}
export class ItemTextListPosition extends ItemListPosition {
export class ItemTextListPosition {
/**
* @param {Item|null} left
* @param {Item|null} right
* @param {number} index
* @param {Map<string,any>} currentAttributes
*/
constructor (left, right, currentAttributes) {
super(left, right)
constructor (left, right, index, currentAttributes) {
this.left = left
this.right = right
this.index = index
this.currentAttributes = currentAttributes
}
}
export class ItemInsertionResult extends ItemListPosition {
/**
* @param {Item|null} left
* @param {Item|null} right
* @param {Map<string,any>} negatedAttributes
* Only call this if you know that this.right is defined
*/
constructor (left, right, negatedAttributes) {
super(left, right)
this.negatedAttributes = negatedAttributes
forward () {
if (this.right === null) {
error.unexpectedCase()
}
switch (this.right.content.constructor) {
case ContentEmbed:
case ContentString:
if (!this.right.deleted) {
this.index += this.right.length
}
break
case ContentFormat:
if (!this.right.deleted) {
updateCurrentAttributes(this.currentAttributes, /** @type {ContentFormat} */ (this.right.content))
}
break
}
this.left = this.right
this.right = this.right.right
}
}
/**
* @param {Transaction} transaction
* @param {Map<string,any>} currentAttributes
* @param {Item|null} left
* @param {Item|null} right
* @param {number} count
* @param {ItemTextListPosition} pos
* @param {number} count steps to move forward
* @return {ItemTextListPosition}
*
* @private
* @function
*/
const findNextPosition = (transaction, currentAttributes, left, right, count) => {
while (right !== null && count > 0) {
switch (right.content.constructor) {
const findNextPosition = (transaction, pos, count) => {
while (pos.right !== null && count > 0) {
switch (pos.right.content.constructor) {
case ContentEmbed:
case ContentString:
if (!right.deleted) {
if (count < right.length) {
if (!pos.right.deleted) {
if (count < pos.right.length) {
// split right
getItemCleanStart(transaction, createID(right.id.client, right.id.clock + count))
getItemCleanStart(transaction, createID(pos.right.id.client, pos.right.id.clock + count))
}
count -= right.length
pos.index += pos.right.length
count -= pos.right.length
}
break
case ContentFormat:
if (!right.deleted) {
updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
if (!pos.right.deleted) {
updateCurrentAttributes(pos.currentAttributes, /** @type {ContentFormat} */ (pos.right.content))
}
break
}
left = right
right = right.right
pos.left = pos.right
pos.right = pos.right.right
// pos.forward() - we don't forward because that would halve the performance because we already do the checks above
}
return new ItemTextListPosition(left, right, currentAttributes)
return pos
}
/**
@ -115,8 +122,14 @@ const findNextPosition = (transaction, currentAttributes, left, right, count) =>
*/
const findPosition = (transaction, parent, index) => {
const currentAttributes = new Map()
const right = parent._start
return findNextPosition(transaction, currentAttributes, null, right, index)
const marker = findMarker(parent, index)
if (marker) {
const pos = new ItemTextListPosition(marker.p.left, marker.p, marker.index, currentAttributes)
return findNextPosition(transaction, pos, index - marker.index)
} else {
const pos = new ItemTextListPosition(null, parent._start, 0, currentAttributes)
return findNextPosition(transaction, pos, index)
}
}
/**
@ -124,37 +137,35 @@ const findPosition = (transaction, parent, index) => {
*
* @param {Transaction} transaction
* @param {AbstractType<any>} parent
* @param {ItemListPosition} currPos
* @param {ItemTextListPosition} currPos
* @param {Map<string,any>} negatedAttributes
*
* @private
* @function
*/
const insertNegatedAttributes = (transaction, parent, currPos, negatedAttributes) => {
let { left, right } = currPos
// check if we really need to remove attributes
while (
right !== null && (
right.deleted === true || (
right.content.constructor === ContentFormat &&
equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key), /** @type {ContentFormat} */ (right.content).value)
currPos.right !== null && (
currPos.right.deleted === true || (
currPos.right.content.constructor === ContentFormat &&
equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (currPos.right.content).key), /** @type {ContentFormat} */ (currPos.right.content).value)
)
)
) {
if (!right.deleted) {
negatedAttributes.delete(/** @type {ContentFormat} */ (right.content).key)
if (!currPos.right.deleted) {
negatedAttributes.delete(/** @type {ContentFormat} */ (currPos.right.content).key)
}
left = right
right = right.right
currPos.forward()
}
const doc = transaction.doc
const ownClientId = doc.clientID
let left = currPos.left
const right = currPos.right
negatedAttributes.forEach((val, key) => {
left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val))
left.integrate(transaction, 0)
})
currPos.left = left
currPos.right = right
}
/**
@ -174,59 +185,51 @@ const updateCurrentAttributes = (currentAttributes, format) => {
}
/**
* @param {ItemListPosition} currPos
* @param {Map<string,any>} currentAttributes
* @param {ItemTextListPosition} currPos
* @param {Object<string,any>} attributes
*
* @private
* @function
*/
const minimizeAttributeChanges = (currPos, currentAttributes, attributes) => {
const minimizeAttributeChanges = (currPos, attributes) => {
// go right while attributes[right.key] === right.value (or right is deleted)
let { left, right } = currPos
while (true) {
if (right === null) {
if (currPos.right === null) {
break
} else if (right.deleted) {
// continue
} else if (right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (right.content)).key] || null, /** @type {ContentFormat} */ (right.content).value)) {
// found a format, update currentAttributes and continue
updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
} else if (currPos.right.deleted || (currPos.right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (currPos.right.content)).key] || null, /** @type {ContentFormat} */ (currPos.right.content).value))) {
//
} else {
break
}
left = right
right = right.right
currPos.forward()
}
currPos.left = left
currPos.right = right
}
/**
* @param {Transaction} transaction
* @param {AbstractType<any>} parent
* @param {ItemListPosition} currPos
* @param {Map<string,any>} currentAttributes
* @param {ItemTextListPosition} currPos
* @param {Object<string,any>} attributes
* @return {Map<string,any>}
*
* @private
* @function
**/
const insertAttributes = (transaction, parent, currPos, currentAttributes, attributes) => {
const insertAttributes = (transaction, parent, currPos, attributes) => {
const doc = transaction.doc
const ownClientId = doc.clientID
const negatedAttributes = new Map()
// insert format-start items
for (const key in attributes) {
const val = attributes[key]
const currentVal = currentAttributes.get(key) || null
const currentVal = currPos.currentAttributes.get(key) || null
if (!equalAttrs(currentVal, val)) {
// save negated attribute (set null if currentVal undefined)
negatedAttributes.set(key, currentVal)
const { left, right } = currPos
currPos.left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val))
currPos.left.integrate(transaction, 0)
currPos.right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val))
currPos.right.integrate(transaction, 0)
currPos.forward()
}
}
return negatedAttributes
@ -235,56 +238,59 @@ const insertAttributes = (transaction, parent, currPos, currentAttributes, attri
/**
* @param {Transaction} transaction
* @param {AbstractType<any>} parent
* @param {ItemListPosition} currPos
* @param {Map<string,any>} currentAttributes
* @param {ItemTextListPosition} currPos
* @param {string|object} text
* @param {Object<string,any>} attributes
*
* @private
* @function
**/
const insertText = (transaction, parent, currPos, currentAttributes, text, attributes) => {
currentAttributes.forEach((val, key) => {
const insertText = (transaction, parent, currPos, text, attributes) => {
currPos.currentAttributes.forEach((val, key) => {
if (attributes[key] === undefined) {
attributes[key] = null
}
})
const doc = transaction.doc
const ownClientId = doc.clientID
minimizeAttributeChanges(currPos, currentAttributes, attributes)
const negatedAttributes = insertAttributes(transaction, parent, currPos, currentAttributes, attributes)
minimizeAttributeChanges(currPos, attributes)
const negatedAttributes = insertAttributes(transaction, parent, currPos, attributes)
// insert content
const content = text.constructor === String ? new ContentString(/** @type {string} */ (text)) : new ContentEmbed(text)
const { left, right } = currPos
currPos.left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content)
currPos.left.integrate(transaction, 0)
return insertNegatedAttributes(transaction, parent, currPos, negatedAttributes)
let { left, right, index } = currPos
if (parent._searchMarker) {
updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength())
}
right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content)
right.integrate(transaction, 0)
currPos.right = right
currPos.index = index
currPos.forward()
insertNegatedAttributes(transaction, parent, currPos, negatedAttributes)
}
/**
* @param {Transaction} transaction
* @param {AbstractType<any>} parent
* @param {ItemListPosition} currPos
* @param {Map<string,any>} currentAttributes
* @param {ItemTextListPosition} currPos
* @param {number} length
* @param {Object<string,any>} attributes
*
* @private
* @function
*/
const formatText = (transaction, parent, currPos, currentAttributes, length, attributes) => {
const formatText = (transaction, parent, currPos, length, attributes) => {
const doc = transaction.doc
const ownClientId = doc.clientID
minimizeAttributeChanges(currPos, currentAttributes, attributes)
const negatedAttributes = insertAttributes(transaction, parent, currPos, currentAttributes, attributes)
let { left, right } = currPos
minimizeAttributeChanges(currPos, attributes)
const negatedAttributes = insertAttributes(transaction, parent, currPos, attributes)
// iterate until first non-format or null is found
// delete all formats with attributes[format.key] != null
while (length > 0 && right !== null) {
if (!right.deleted) {
switch (right.content.constructor) {
while (length > 0 && currPos.right !== null) {
if (!currPos.right.deleted) {
switch (currPos.right.content.constructor) {
case ContentFormat: {
const { key, value } = /** @type {ContentFormat} */ (right.content)
const { key, value } = /** @type {ContentFormat} */ (currPos.right.content)
const attr = attributes[key]
if (attr !== undefined) {
if (equalAttrs(attr, value)) {
@ -292,22 +298,20 @@ const formatText = (transaction, parent, currPos, currentAttributes, length, att
} else {
negatedAttributes.set(key, value)
}
right.delete(transaction)
currPos.right.delete(transaction)
}
updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
break
}
case ContentEmbed:
case ContentString:
if (length < right.length) {
getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length))
if (length < currPos.right.length) {
getItemCleanStart(transaction, createID(currPos.right.id.client, currPos.right.id.clock + length))
}
length -= right.length
length -= currPos.right.length
break
}
}
left = right
right = right.right
currPos.forward()
}
// Quill just assumes that the editor starts with a newline and that it always
// ends with a newline. We only insert that newline when a new newline is
@ -317,11 +321,10 @@ const formatText = (transaction, parent, currPos, currentAttributes, length, att
for (; length > 0; length--) {
newlines += '\n'
}
left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentString(newlines))
left.integrate(transaction, 0)
currPos.right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), currPos.left, currPos.left && currPos.left.lastId, currPos.right, currPos.right && currPos.right.id, parent, null, new ContentString(newlines))
currPos.right.integrate(transaction, 0)
currPos.forward()
}
currPos.left = left
currPos.right = right
insertNegatedAttributes(transaction, parent, currPos, negatedAttributes)
}
@ -431,42 +434,39 @@ export const cleanupYTextFormatting = type => {
/**
* @param {Transaction} transaction
* @param {ItemListPosition} currPos
* @param {Map<string,any>} currentAttributes
* @param {ItemTextListPosition} currPos
* @param {number} length
* @return {ItemListPosition}
* @return {ItemTextListPosition}
*
* @private
* @function
*/
const deleteText = (transaction, currPos, currentAttributes, length) => {
const startAttrs = map.copy(currentAttributes)
const deleteText = (transaction, currPos, length) => {
const startLength = length
const startAttrs = map.copy(currPos.currentAttributes)
const start = currPos.right
let { left, right } = currPos
while (length > 0 && right !== null) {
if (right.deleted === false) {
switch (right.content.constructor) {
case ContentFormat:
updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
break
while (length > 0 && currPos.right !== null) {
if (currPos.right.deleted === false) {
switch (currPos.right.content.constructor) {
case ContentEmbed:
case ContentString:
if (length < right.length) {
getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length))
if (length < currPos.right.length) {
getItemCleanStart(transaction, createID(currPos.right.id.client, currPos.right.id.clock + length))
}
length -= right.length
right.delete(transaction)
length -= currPos.right.length
currPos.right.delete(transaction)
break
}
}
left = right
right = right.right
currPos.forward()
}
if (start) {
cleanupFormattingGap(transaction, start, right, startAttrs, map.copy(currentAttributes))
cleanupFormattingGap(transaction, start, currPos.right, startAttrs, map.copy(currPos.currentAttributes))
}
const parent = /** @type {AbstractType<any>} */ (/** @type {Item} */ (currPos.left || currPos.right).parent)
if (parent._searchMarker) {
updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length)
}
currPos.left = left
currPos.right = right
return currPos
}
@ -729,6 +729,10 @@ export class YText extends AbstractType {
* @type {Array<function():void>?}
*/
this._pending = string !== undefined ? [() => this.insert(0, string)] : []
/**
* @type {Array<ArraySearchMarker>}
*/
this._searchMarker = []
}
/**
@ -765,6 +769,7 @@ export class YText extends AbstractType {
* @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
*/
_callObserver (transaction, parentSubs) {
super._callObserver(transaction, parentSubs)
const event = new YTextEvent(this, transaction)
const doc = transaction.doc
// If a remote change happened, we try to cleanup potential formatting duplicates.
@ -778,7 +783,7 @@ export class YText extends AbstractType {
}
iterateStructs(transaction, /** @type {Array<Item|GC>} */ (doc.store.clients.get(client)), clock, afterClock, item => {
// @ts-ignore
if (!item.deleted && item.content.constructor === ContentFormat) {
if (item.content.constructor === ContentFormat) {
foundFormattingItem = true
}
})
@ -786,7 +791,17 @@ export class YText extends AbstractType {
break
}
}
transact(doc, t => {
if (!foundFormattingItem) {
iterateDeletedStructs(transaction, transaction.deleteSet, item => {
if (item instanceof GC || foundFormattingItem) {
return
}
if (item.parent === this && item.content.constructor === ContentFormat) {
foundFormattingItem = true
}
})
}
transact(doc, (t) => {
if (foundFormattingItem) {
// If a formatting item was inserted, we simply clean the whole type.
// We need to compute currentAttributes for the current position anyway.
@ -795,7 +810,7 @@ export class YText extends AbstractType {
// If no formatting attribute was inserted, we can make due with contextless
// formatting cleanups.
// Contextless: it is not necessary to compute currentAttributes for the affected position.
iterateDeletedStructs(t, transaction.deleteSet, item => {
iterateDeletedStructs(t, t.deleteSet, item => {
if (item instanceof GC) {
return
}
@ -852,11 +867,7 @@ export class YText extends AbstractType {
applyDelta (delta, { sanitize = true } = {}) {
if (this.doc !== null) {
transact(this.doc, transaction => {
/**
* @type {ItemListPosition}
*/
const currPos = new ItemListPosition(null, this._start)
const currentAttributes = new Map()
const currPos = new ItemTextListPosition(null, this._start, 0, new Map())
for (let i = 0; i < delta.length; i++) {
const op = delta[i]
if (op.insert !== undefined) {
@ -867,12 +878,12 @@ export class YText extends AbstractType {
// paragraphs, but nothing bad will happen.
const ins = (!sanitize && typeof op.insert === 'string' && i === delta.length - 1 && currPos.right === null && op.insert.slice(-1) === '\n') ? op.insert.slice(0, -1) : op.insert
if (typeof ins !== 'string' || ins.length > 0) {
insertText(transaction, this, currPos, currentAttributes, ins, op.attributes || {})
insertText(transaction, this, currPos, ins, op.attributes || {})
}
} else if (op.retain !== undefined) {
formatText(transaction, this, currPos, currentAttributes, op.retain, op.attributes || {})
formatText(transaction, this, currPos, op.retain, op.attributes || {})
} else if (op.delete !== undefined) {
deleteText(transaction, currPos, currentAttributes, op.delete)
deleteText(transaction, currPos, op.delete)
}
}
})
@ -1004,13 +1015,13 @@ export class YText extends AbstractType {
const y = this.doc
if (y !== null) {
transact(y, transaction => {
const { left, right, currentAttributes } = findPosition(transaction, this, index)
const pos = findPosition(transaction, this, index)
if (!attributes) {
attributes = {}
// @ts-ignore
currentAttributes.forEach((v, k) => { attributes[k] = v })
pos.currentAttributes.forEach((v, k) => { attributes[k] = v })
}
insertText(transaction, this, new ItemListPosition(left, right), currentAttributes, text, attributes)
insertText(transaction, this, pos, text, attributes)
})
} else {
/** @type {Array<function>} */ (this._pending).push(() => this.insert(index, text, attributes))
@ -1034,8 +1045,8 @@ export class YText extends AbstractType {
const y = this.doc
if (y !== null) {
transact(y, transaction => {
const { left, right, currentAttributes } = findPosition(transaction, this, index)
insertText(transaction, this, new ItemListPosition(left, right), currentAttributes, embed, attributes)
const pos = findPosition(transaction, this, index)
insertText(transaction, this, pos, embed, attributes)
})
} else {
/** @type {Array<function>} */ (this._pending).push(() => this.insertEmbed(index, embed, attributes))
@ -1057,8 +1068,7 @@ export class YText extends AbstractType {
const y = this.doc
if (y !== null) {
transact(y, transaction => {
const { left, right, currentAttributes } = findPosition(transaction, this, index)
deleteText(transaction, new ItemListPosition(left, right), currentAttributes, length)
deleteText(transaction, findPosition(transaction, this, index), length)
})
} else {
/** @type {Array<function>} */ (this._pending).push(() => this.delete(index, length))
@ -1082,11 +1092,11 @@ export class YText extends AbstractType {
const y = this.doc
if (y !== null) {
transact(y, transaction => {
const { left, right, currentAttributes } = findPosition(transaction, this, index)
if (right === null) {
const pos = findPosition(transaction, this, index)
if (pos.right === null) {
return
}
formatText(transaction, this, new ItemListPosition(left, right), currentAttributes, length, attributes)
formatText(transaction, this, pos, length, attributes)
})
} else {
/** @type {Array<function>} */ (this._pending).push(() => this.format(index, length, attributes))

22
src/utils/logging.js Normal file
View File

@ -0,0 +1,22 @@
import {
AbstractType // eslint-disable-line
} from '../internals.js'
/**
* Convenient helper to log type information.
*
* Do not use in productive systems as the output can be immense!
*
* @param {AbstractType<any>} type
*/
export const logType = type => {
const res = []
let n = type._start
while (n) {
res.push(n)
n = n.right
}
console.log('Children: ', res)
console.log('Children content: ', res.filter(m => !m.deleted).map(m => m.content))
}

58
tests/testHelper.js
View File

@ -1,13 +1,3 @@
import * as Y from '../src/index.js'
import {
createDeleteSetFromStructStore,
getStateVector,
Item,
useV1Encoding,
useV2Encoding,
DeleteItem, DeleteSet, StructStore, Doc // eslint-disable-line
} from '../src/internals.js'
import * as t from 'lib0/testing.js'
import * as prng from 'lib0/prng.js'
@ -15,8 +5,14 @@ import * as encoding from 'lib0/encoding.js'
import * as decoding from 'lib0/decoding.js'
import * as syncProtocol from 'y-protocols/sync.js'
import * as object from 'lib0/object.js'
import * as Y from '../src/internals.js'
export * from '../src/internals.js'
if (typeof window !== 'undefined') {
// @ts-ignore
window.Y = Y // eslint-disable-line
}
/**
* @param {TestYInstance} y // publish message created by `y` to all other online clients
* @param {Uint8Array} m
@ -31,7 +27,7 @@ const broadcastMessage = (y, m) => {
}
}
export class TestYInstance extends Doc {
export class TestYInstance extends Y.Doc {
/**
* @param {TestConnector} testConnector
* @param {number} clientID
@ -232,7 +228,7 @@ export class TestConnector {
* @param {t.TestCase} tc
* @param {{users?:number}} conf
* @param {InitTestObjectCallback<T>} [initTestObject]
* @return {{testObjects:Array<any>,testConnector:TestConnector,users:Array<TestYInstance>,array0:Y.Array<any>,array1:Y.Array<any>,array2:Y.Array<any>,map0:Y.Map<any>,map1:Y.Map<any>,map2:Y.Map<any>,map3:Y.Map<any>,text0:Y.Text,text1:Y.Text,text2:Y.Text,xml0:Y.XmlElement,xml1:Y.XmlElement,xml2:Y.XmlElement}}
* @return {{testObjects:Array<any>,testConnector:TestConnector,users:Array<TestYInstance>,array0:Y.YArray<any>,array1:Y.YArray<any>,array2:Y.YArray<any>,map0:Y.YMap<any>,map1:Y.YMap<any>,map2:Y.YMap<any>,map3:Y.YMap<any>,text0:Y.YText,text1:Y.YText,text2:Y.YText,xml0:Y.YXmlElement,xml1:Y.YXmlElement,xml2:Y.YXmlElement}}
*/
export const init = (tc, { users = 5 } = {}, initTestObject) => {
/**
@ -244,9 +240,9 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => {
const gen = tc.prng
// choose an encoding approach at random
if (prng.bool(gen)) {
useV2Encoding()
Y.useV2Encoding()
} else {
useV1Encoding()
Y.useV1Encoding()
}
const testConnector = new TestConnector(gen)
@ -255,14 +251,14 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => {
const y = testConnector.createY(i)
y.clientID = i
result.users.push(y)
result['array' + i] = y.get('array', Y.Array)
result['map' + i] = y.get('map', Y.Map)
result['xml' + i] = y.get('xml', Y.XmlElement)
result['text' + i] = y.get('text', Y.Text)
result['array' + i] = y.getArray('array')
result['map' + i] = y.getMap('map')
result['xml' + i] = y.get('xml', Y.YXmlElement)
result['text' + i] = y.getText('text')
}
testConnector.syncAll()
result.testObjects = result.users.map(initTestObject || (() => null))
useV1Encoding()
Y.useV1Encoding()
return /** @type {any} */ (result)
}
@ -280,7 +276,7 @@ export const compare = users => {
while (users[0].tc.flushAllMessages()) {}
const userArrayValues = users.map(u => u.getArray('array').toJSON())
const userMapValues = users.map(u => u.getMap('map').toJSON())
const userXmlValues = users.map(u => u.get('xml', Y.XmlElement).toString())
const userXmlValues = users.map(u => u.get('xml', Y.YXmlElement).toString())
const userTextValues = users.map(u => u.getText('text').toDelta())
for (const u of users) {
t.assert(u.store.pendingDeleteReaders.length === 0)
@ -309,23 +305,23 @@ export const compare = users => {
t.compare(userXmlValues[i], userXmlValues[i + 1])
t.compare(userTextValues[i].map(/** @param {any} a */ a => typeof a.insert === 'string' ? a.insert : ' ').join('').length, users[i].getText('text').length)
t.compare(userTextValues[i], userTextValues[i + 1])
t.compare(getStateVector(users[i].store), getStateVector(users[i + 1].store))
compareDS(createDeleteSetFromStructStore(users[i].store), createDeleteSetFromStructStore(users[i + 1].store))
t.compare(Y.getStateVector(users[i].store), Y.getStateVector(users[i + 1].store))
compareDS(Y.createDeleteSetFromStructStore(users[i].store), Y.createDeleteSetFromStructStore(users[i + 1].store))
compareStructStores(users[i].store, users[i + 1].store)
}
users.map(u => u.destroy())
}
/**
* @param {Item?} a
* @param {Item?} b
* @param {Y.Item?} a
* @param {Y.Item?} b
* @return {boolean}
*/
export const compareItemIDs = (a, b) => a === b || (a !== null && b != null && Y.compareIDs(a.id, b.id))
/**
* @param {StructStore} ss1
* @param {StructStore} ss2
* @param {Y.StructStore} ss1
* @param {Y.StructStore} ss2
*/
export const compareStructStores = (ss1, ss2) => {
t.assert(ss1.clients.size === ss2.clients.size)
@ -345,9 +341,9 @@ export const compareStructStores = (ss1, ss2) => {
) {
t.fail('Structs dont match')
}
if (s1 instanceof Item) {
if (s1 instanceof Y.Item) {
if (
!(s2 instanceof Item) ||
!(s2 instanceof Y.Item) ||
!((s1.left === null && s2.left === null) || (s1.left !== null && s2.left !== null && Y.compareIDs(s1.left.lastId, s2.left.lastId))) ||
!compareItemIDs(s1.right, s2.right) ||
!Y.compareIDs(s1.origin, s2.origin) ||
@ -367,13 +363,13 @@ export const compareStructStores = (ss1, ss2) => {
}
/**
* @param {DeleteSet} ds1
* @param {DeleteSet} ds2
* @param {Y.DeleteSet} ds1
* @param {Y.DeleteSet} ds2
*/
export const compareDS = (ds1, ds2) => {
t.assert(ds1.clients.size === ds2.clients.size)
ds1.clients.forEach((deleteItems1, client) => {
const deleteItems2 = /** @type {Array<DeleteItem>} */ (ds2.clients.get(client))
const deleteItems2 = /** @type {Array<Y.DeleteItem>} */ (ds2.clients.get(client))
t.assert(deleteItems2 !== undefined && deleteItems1.length === deleteItems2.length)
for (let i = 0; i < deleteItems1.length; i++) {
const di1 = deleteItems1[i]

10
tests/y-array.tests.js
View File

@ -352,7 +352,10 @@ const arrayTransactions = [
content.push(uniqueNumber)
}
var pos = prng.int32(gen, 0, yarray.length)
const oldContent = yarray.toArray()
yarray.insert(pos, content)
oldContent.splice(pos, 0, ...content)
t.compareArrays(yarray.toArray(), oldContent) // we want to make sure that fastSearch markers insert at the correct position
},
function insertTypeArray (user, gen) {
const yarray = user.getArray('array')
@ -384,7 +387,10 @@ const arrayTransactions = [
type.delete(somePos, delLength)
}
} else {
const oldContent = yarray.toArray()
yarray.delete(somePos, delLength)
oldContent.splice(somePos, delLength)
t.compareArrays(yarray.toArray(), oldContent)
}
}
}
@ -393,8 +399,8 @@ const arrayTransactions = [
/**
* @param {t.TestCase} tc
*/
export const testRepeatGeneratingYarrayTests4 = tc => {
applyRandomTests(tc, arrayTransactions, 4)
export const testRepeatGeneratingYarrayTests6 = tc => {
applyRandomTests(tc, arrayTransactions, 6)
}
/**

146
tests/y-text.tests.js
View File

@ -205,6 +205,50 @@ export const testFormattingRemovedInMidText = tc => {
t.assert(Y.getTypeChildren(text0).length === 3)
}
/**
* @param {t.TestCase} tc
*/
export const testInsertAndDeleteAtRandomPositions = tc => {
const N = 10000
const { text0 } = init(tc, { users: 1 })
const gen = tc.prng
// create initial content
// let expectedResult = init
text0.insert(0, prng.word(gen, N / 2, N / 2))
// apply changes
for (let i = 0; i < N; i++) {
const pos = prng.uint32(gen, 0, text0.length)
if (prng.bool(gen)) {
const len = prng.uint32(gen, 1, 5)
const word = prng.word(gen, 0, len)
text0.insert(pos, word)
// expectedResult = expectedResult.slice(0, pos) + word + expectedResult.slice(pos)
} else {
const len = prng.uint32(gen, 0, math.min(3, text0.length - pos))
text0.delete(pos, len)
// expectedResult = expectedResult.slice(0, pos) + expectedResult.slice(pos + len)
}
}
// t.compareStrings(text0.toString(), expectedResult)
t.describe('final length', '' + text0.length)
}
/**
* @param {t.TestCase} tc
*/
export const testAppendChars = tc => {
const N = 10000
const { text0 } = init(tc, { users: 1 })
// apply changes
for (let i = 0; i < N; i++) {
text0.insert(text0.length, 'a')
}
t.assert(text0.length === N)
}
const id = Y.createID(0, 0)
const c = new Y.ContentString('a')
@ -281,6 +325,102 @@ export const testLargeFragmentedDocument = tc => {
let charCounter = 0
/**
* Random tests for pure text operations without formatting.
*
* @type Array<function(any,prng.PRNG):void>
*/
const textChanges = [
/**
* @param {Y.Doc} y
* @param {prng.PRNG} gen
*/
(y, gen) => { // insert text
const ytext = y.getText('text')
const insertPos = prng.int32(gen, 0, ytext.length)
const text = charCounter++ + prng.word(gen)
const prevText = ytext.toString()
ytext.insert(insertPos, text)
t.compareStrings(ytext.toString(), prevText.slice(0, insertPos) + text + prevText.slice(insertPos))
},
/**
* @param {Y.Doc} y
* @param {prng.PRNG} gen
*/
(y, gen) => { // delete text
const ytext = y.getText('text')
const contentLen = ytext.toString().length
const insertPos = prng.int32(gen, 0, contentLen)
const overwrite = math.min(prng.int32(gen, 0, contentLen - insertPos), 2)
const prevText = ytext.toString()
ytext.delete(insertPos, overwrite)
t.compareStrings(ytext.toString(), prevText.slice(0, insertPos) + prevText.slice(insertPos + overwrite))
}
]
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges5 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 5))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges30 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 30))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges40 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 40))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges50 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 50))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges70 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 70))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges90 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 90))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
/**
* @param {t.TestCase} tc
*/
export const testRepeatGenerateTextChanges300 = tc => {
const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 300))
const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
t.assert(cleanups === 0)
}
const marks = [
{ bold: true },
{ italic: true },
@ -293,6 +433,8 @@ const marksChoices = [
]
/**
* Random tests for all features of y-text (formatting, embeds, ..).
*
* @type Array<function(any,prng.PRNG):void>
*/
const qChanges = [
@ -302,7 +444,7 @@ const qChanges = [
*/
(y, gen) => { // insert text
const ytext = y.getText('text')
const insertPos = prng.int32(gen, 0, ytext.toString().length)
const insertPos = prng.int32(gen, 0, ytext.length)
const attrs = prng.oneOf(gen, marksChoices)
const text = charCounter++ + prng.word(gen)
ytext.insert(insertPos, text, attrs)
@ -313,7 +455,7 @@ const qChanges = [
*/
(y, gen) => { // insert embed
const ytext = y.getText('text')
const insertPos = prng.int32(gen, 0, ytext.toString().length)
const insertPos = prng.int32(gen, 0, ytext.length)
ytext.insertEmbed(insertPos, { image: 'https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png' })
},
/**