From e3c59b0aa7cbe761a86da26cfe6bb6bdc70a50e8 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 27 Jan 2020 03:42:32 +0100 Subject: [PATCH] more options to gc data (undomanager.clear and tryGc) --- package.json | 3 +- src/index.js | 3 +- src/structs/Item.js | 9 +-- src/utils/Doc.js | 4 +- src/utils/Transaction.js | 136 +++++++++++++++++++++++---------------- src/utils/UndoManager.js | 21 +++++- 6 files changed, 113 insertions(+), 63 deletions(-) diff --git a/package.json b/package.json index 55616ba4..2831711d 100644 --- a/package.json +++ b/package.json @@ -8,14 +8,13 @@ "sideEffects": false, "scripts": { "test": "npm run dist && node ./dist/tests.cjs --repitition-time 50", - "test-exhaustive": "npm run lint && npm run dist && node ./dist/tests.cjs --production --repitition-time 10000", + "test-extensive": "npm run lint && npm run dist && node ./dist/tests.cjs --production --repitition-time 10000", "dist": "rm -rf dist && rollup -c && tsc", "watch": "rollup -wc", "lint": "markdownlint README.md && standard && tsc", "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true", "serve-docs": "npm run docs && http-server ./docs/", "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000", - "postversion": "git push && git push --tags", "debug": "concurrently 'http-server -o test.html' 'npm run watch'", "trace-deopt": "clear && rollup -c && node --trace-deopt dist/test.cjs", "trace-opt": "clear && rollup -c && node --trace-opt dist/test.cjs" diff --git a/src/index.js b/src/index.js index cd40b1f8..d8c59511 100644 --- a/src/index.js +++ b/src/index.js @@ -55,5 +55,6 @@ export { isDeleted, isParentOf, equalSnapshots, - PermanentUserData // @TODO experimental + PermanentUserData, // @TODO experimental + tryGc } from './internals.js' diff --git a/src/structs/Item.js b/src/structs/Item.js index c500b197..253c6e0e 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -68,10 +68,11 @@ export const followRedone = (store, id) => { * sending it to other peers * * @param {Item|null} item + * @param {boolean} keep */ -export const keepItem = item => { - while (item !== null && !item.keep) { - item.keep = true +export const keepItem = (item, keep) => { + while (item !== null && item.keep !== keep) { + item.keep = keep item = item.parent._item } } @@ -220,7 +221,7 @@ export const redoItem = (transaction, item, redoitems) => { item.content.copy() ) item.redone = redoneItem.id - keepItem(redoneItem) + keepItem(redoneItem, true) redoneItem.integrate(transaction) return redoneItem } diff --git a/src/utils/Doc.js b/src/utils/Doc.js index dacc4bd7..fc5ca6dd 100644 --- a/src/utils/Doc.js +++ b/src/utils/Doc.js @@ -25,10 +25,12 @@ export class Doc extends Observable { /** * @param {Object} conf configuration * @param {boolean} [conf.gc] Disable garbage collection (default: gc=true) + * @param {function(Item):boolean} [conf.gcFilter] Will be called before an Item is garbage collected. Return false to keep the Item. */ - constructor ({ gc = true } = {}) { + constructor ({ gc = true, gcFilter = () => true } = {}) { super() this.gc = gc + this.gcFilter = gcFilter this.clientID = random.uint32() /** * @type {Map>} diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index 55ce90cc..fd312000 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -10,7 +10,7 @@ import { findIndexSS, callEventHandlerListeners, Item, - ID, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line + StructStore, ID, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' @@ -145,6 +145,85 @@ export const addChangedTypeToTransaction = (transaction, type, parentSub) => { } } +/** + * @param {Array} structs + * @param {number} pos + */ +const tryToMergeWithLeft = (structs, pos) => { + const left = structs[pos - 1] + const right = structs[pos] + if (left.deleted === right.deleted && left.constructor === right.constructor) { + if (left.mergeWith(right)) { + structs.splice(pos, 1) + if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) { + right.parent._map.set(right.parentSub, /** @type {Item} */ (left)) + } + } + } +} + +/** + * @param {DeleteSet} ds + * @param {StructStore} store + * @param {function(Item):boolean} gcFilter + */ +const tryGcDeleteSet = (ds, store, gcFilter) => { + for (const [client, deleteItems] of ds.clients) { + const structs = /** @type {Array} */ (store.clients.get(client)) + for (let di = deleteItems.length - 1; di >= 0; di--) { + const deleteItem = deleteItems[di] + const endDeleteItemClock = deleteItem.clock + deleteItem.len + for ( + let si = findIndexSS(structs, deleteItem.clock), struct = structs[si]; + si < structs.length && struct.id.clock < endDeleteItemClock; + struct = structs[++si] + ) { + const struct = structs[si] + if (deleteItem.clock + deleteItem.len <= struct.id.clock) { + break + } + if (struct instanceof Item && struct.deleted && !struct.keep && gcFilter(struct)) { + struct.gc(store, false) + } + } + } + } +} + +/** + * @param {DeleteSet} ds + * @param {StructStore} store + */ +const tryMergeDeleteSet = (ds, store) => { + // try to merge deleted / gc'd items + // merge from right to left for better efficiecy and so we don't miss any merge targets + for (const [client, deleteItems] of ds.clients) { + const structs = /** @type {Array} */ (store.clients.get(client)) + for (let di = deleteItems.length - 1; di >= 0; di--) { + const deleteItem = deleteItems[di] + // start with merging the item next to the last deleted item + const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1)) + for ( + let si = mostRightIndexToCheck, struct = structs[si]; + si > 0 && struct.id.clock >= deleteItem.clock; + struct = structs[--si] + ) { + tryToMergeWithLeft(structs, si) + } + } + } +} + +/** + * @param {DeleteSet} ds + * @param {StructStore} store + * @param {function(Item):boolean} gcFilter + */ +export const tryGc = (ds, store, gcFilter) => { + tryGcDeleteSet(ds, store, gcFilter) + tryMergeDeleteSet(ds, store) +} + /** * @param {Array} transactionCleanups * @param {number} i @@ -201,63 +280,12 @@ const cleanupTransactions = (transactionCleanups, i) => { }) callAll(fs, []) } finally { - /** - * @param {Array} structs - * @param {number} pos - */ - const tryToMergeWithLeft = (structs, pos) => { - const left = structs[pos - 1] - const right = structs[pos] - if (left.deleted === right.deleted && left.constructor === right.constructor) { - if (left.mergeWith(right)) { - structs.splice(pos, 1) - if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) { - right.parent._map.set(right.parentSub, /** @type {Item} */ (left)) - } - } - } - } // Replace deleted items with ItemDeleted / GC. // This is where content is actually remove from the Yjs Doc. if (doc.gc) { - for (const [client, deleteItems] of ds.clients) { - const structs = /** @type {Array} */ (store.clients.get(client)) - for (let di = deleteItems.length - 1; di >= 0; di--) { - const deleteItem = deleteItems[di] - const endDeleteItemClock = deleteItem.clock + deleteItem.len - for ( - let si = findIndexSS(structs, deleteItem.clock), struct = structs[si]; - si < structs.length && struct.id.clock < endDeleteItemClock; - struct = structs[++si] - ) { - const struct = structs[si] - if (deleteItem.clock + deleteItem.len <= struct.id.clock) { - break - } - if (struct instanceof Item && struct.deleted && !struct.keep) { - struct.gc(store, false) - } - } - } - } - } - // try to merge deleted / gc'd items - // merge from right to left for better efficiecy and so we don't miss any merge targets - for (const [client, deleteItems] of ds.clients) { - const structs = /** @type {Array} */ (store.clients.get(client)) - for (let di = deleteItems.length - 1; di >= 0; di--) { - const deleteItem = deleteItems[di] - // start with merging the item next to the last deleted item - const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1)) - for ( - let si = mostRightIndexToCheck, struct = structs[si]; - si > 0 && struct.id.clock >= deleteItem.clock; - struct = structs[--si] - ) { - tryToMergeWithLeft(structs, si) - } - } + tryGcDeleteSet(ds, store, doc.gcFilter) } + tryMergeDeleteSet(ds, store) // on all affected store.clients props, try to merge for (const [client, clock] of transaction.afterState) { diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index 667cc9ac..286aa84d 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -199,13 +199,32 @@ export class UndoManager extends Observable { // make sure that deleted structs are not gc'd iterateDeletedStructs(transaction, transaction.deleteSet, /** @param {Item|GC} item */ item => { if (item instanceof Item && this.scope.some(type => isParentOf(type, item))) { - keepItem(item) + keepItem(item, true) } }) this.emit('stack-item-added', [{ stackItem: stack[stack.length - 1], origin: transaction.origin, type: undoing ? 'redo' : 'undo' }, this]) }) } + clear () { + this.doc.transact(transaction => { + /** + * @param {StackItem} stackItem + */ + const clearItem = stackItem => { + iterateDeletedStructs(transaction, stackItem.ds, item => { + if (item instanceof Item && this.scope.some(type => isParentOf(type, item))) { + keepItem(item, false) + } + }) + } + this.undoStack.forEach(clearItem) + this.redoStack.forEach(clearItem) + }) + this.undoStack = [] + this.redoStack = [] + } + /** * UndoManager merges Undo-StackItem if they are created within time-gap * smaller than `options.captureTimeout`. Call `um.stopCapturing()` so that the next