From 36203af88e27034a44f60bd7afe82f87a22ae30e Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 29 Jun 2019 14:47:34 +0200 Subject: [PATCH 01/34] should not rely on all deconstructing features because not all parsers support it --- src/utils/UndoManager.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index d62f06dc..4a28205a 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -102,7 +102,10 @@ export class UndoManager extends Observable { * @param {Set} [trackedTransactionOrigins=new Set([null])] * @param {object} [options={captureTimeout=500}] */ - constructor (typeScope, trackedTransactionOrigins = new Set([null]), { captureTimeout = 500 } = {}) { + constructor (typeScope, trackedTransactionOrigins = new Set([null]), { captureTimeout } = {}) { + if (captureTimeout == null) { + captureTimeout = 500 + } super() this.scope = typeScope instanceof Array ? typeScope : [typeScope] trackedTransactionOrigins.add(this) From f0262ffaae1137798120a13370405b6c15510543 Mon Sep 17 00:00:00 2001 From: blackening Date: Tue, 9 Jul 2019 19:58:06 +0800 Subject: [PATCH 02/34] Updated documentation for Y.Array forEach Reference: https://github.com/y-js/yjs/blob/master/src/types/YArray.js#L186 https://github.com/y-js/yjs/blob/master/src/types/AbstractType.js#L239 --- README.v13.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.v13.md b/README.v13.md index 8adcab73..700e6c55 100644 --- a/README.v13.md +++ b/README.v13.md @@ -186,7 +186,7 @@ position 0.
length:number
- forEach(function(index:number,value:object|boolean|Array|string|number|Uint8Array|Y.Type)) + forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, index:number, array: Y.Array))
map(function(T, number, YArray):M):Array<M>
From 0e7da017fe7211965a2486a069afb3678caa3d48 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 9 Aug 2019 01:15:33 +0200 Subject: [PATCH 03/34] Use lib0/any-encoding instead of JSON --- package-lock.json | 55 ++++++++++++++----- package.json | 2 +- src/index.js | 1 + src/internals.js | 1 + src/structs/ContentAny.js | 108 ++++++++++++++++++++++++++++++++++++++ src/structs/Item.js | 4 +- src/types/AbstractType.js | 8 +-- tests/encoding.tests.js | 8 +-- 8 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 src/structs/ContentAny.js diff --git a/package-lock.json b/package-lock.json index 20a9d4a6..b81434e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1490,7 +1490,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -1511,12 +1512,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1531,17 +1534,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -1658,7 +1664,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -1670,6 +1677,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -1684,6 +1692,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1691,12 +1700,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -1715,6 +1726,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -1795,7 +1807,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -1807,6 +1820,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -1892,7 +1906,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -1928,6 +1943,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -1947,6 +1963,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -1990,12 +2007,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -2577,9 +2596,9 @@ } }, "lib0": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.0.5.tgz", - "integrity": "sha512-3ElV6/t5Lv0Eczlnh/05q+Uq3RxQ/Q0zdN6LVtaUERQIDDZsP/CUXEGLsV8KZTgZwVFNCPGXNWYE+3WTOo+SHw==" + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.0.6.tgz", + "integrity": "sha512-drb8LcwZu2rAmTsXN0d3hFtZVbPE5ZUrsWf307Boc/v7IrmLq3lM5+OOMY672EysHTWeXo/OH54wRHyD6eFXXw==" }, "linkify-it": { "version": "2.1.0", @@ -4366,6 +4385,14 @@ "dev": true, "requires": { "lib0": "0.0.5" + }, + "dependencies": { + "lib0": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.0.5.tgz", + "integrity": "sha512-3ElV6/t5Lv0Eczlnh/05q+Uq3RxQ/Q0zdN6LVtaUERQIDDZsP/CUXEGLsV8KZTgZwVFNCPGXNWYE+3WTOo+SHw==", + "dev": true + } } }, "yallist": { diff --git a/package.json b/package.json index cc0e099f..c6140240 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ }, "homepage": "http://y-js.org", "dependencies": { - "lib0": "0.0.5" + "lib0": "0.0.6" }, "devDependencies": { "concurrently": "^3.6.1", diff --git a/src/index.js b/src/index.js index 73d4e76c..b3288e06 100644 --- a/src/index.js +++ b/src/index.js @@ -21,6 +21,7 @@ export { ContentEmbed, ContentFormat, ContentJSON, + ContentAny, ContentString, ContentType, AbstractType, diff --git a/src/internals.js b/src/internals.js index 74ce0eba..d5ac7848 100644 --- a/src/internals.js +++ b/src/internals.js @@ -27,6 +27,7 @@ export * from './structs/ContentDeleted.js' export * from './structs/ContentEmbed.js' export * from './structs/ContentFormat.js' export * from './structs/ContentJSON.js' +export * from './structs/ContentAny.js' export * from './structs/ContentString.js' export * from './structs/ContentType.js' export * from './structs/Item.js' diff --git a/src/structs/ContentAny.js b/src/structs/ContentAny.js new file mode 100644 index 00000000..1367cbae --- /dev/null +++ b/src/structs/ContentAny.js @@ -0,0 +1,108 @@ +import { + Transaction, Item, StructStore // eslint-disable-line +} from '../internals.js' + +import * as encoding from 'lib0/encoding.js' +import * as decoding from 'lib0/decoding.js' + +/** + * @private + */ +export class ContentAny { + /** + * @param {Array} arr + */ + constructor (arr) { + /** + * @type {Array} + */ + this.arr = arr + } + /** + * @return {number} + */ + getLength () { + return this.arr.length + } + /** + * @return {Array} + */ + getContent () { + return this.arr + } + /** + * @return {boolean} + */ + isCountable () { + return true + } + /** + * @return {ContentAny} + */ + copy () { + return new ContentAny(this.arr) + } + /** + * @param {number} offset + * @return {ContentAny} + */ + splice (offset) { + const right = new ContentAny(this.arr.slice(offset)) + this.arr = this.arr.slice(0, offset) + return right + } + /** + * @param {ContentAny} right + * @return {boolean} + */ + mergeWith (right) { + this.arr = this.arr.concat(right.arr) + return true + } + /** + * @param {Transaction} transaction + * @param {Item} item + */ + integrate (transaction, item) {} + /** + * @param {Transaction} transaction + */ + delete (transaction) {} + /** + * @param {StructStore} store + */ + gc (store) {} + /** + * @param {encoding.Encoder} encoder + * @param {number} offset + */ + write (encoder, offset) { + const len = this.arr.length + encoding.writeVarUint(encoder, len - offset) + for (let i = offset; i < len; i++) { + const c = this.arr[i] + encoding.writeAny(encoder, c) + } + } + /** + * @return {number} + */ + getRef () { + return 8 + } +} + +/** + * @private + * + * @param {decoding.Decoder} decoder + * @return {ContentAny} + */ +export const readContentAny = decoder => { + const len = decoding.readVarUint(decoder) + const cs = [] + for (let i = 0; i < len; i++) { + cs.push(decoding.readAny(decoder)) + } + return new ContentAny(cs) +} diff --git a/src/structs/Item.js b/src/structs/Item.js index 9d7e46fe..95530b6e 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -18,6 +18,7 @@ import { readContentDeleted, readContentBinary, readContentJSON, + readContentAny, readContentString, readContentEmbed, readContentFormat, @@ -561,7 +562,8 @@ export const contentRefs = [ readContentString, readContentEmbed, readContentFormat, - readContentType + readContentType, + readContentAny ] /** diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index dd4ae2cd..9207c175 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -7,7 +7,7 @@ import { nextID, isVisible, ContentType, - ContentJSON, + ContentAny, ContentBinary, createID, getItemCleanStart, @@ -374,7 +374,7 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem, let jsonContent = [] const packJsonContent = () => { if (jsonContent.length > 0) { - left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentJSON(jsonContent)) + left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentAny(jsonContent)) left.integrate(transaction) jsonContent = [] } @@ -503,7 +503,7 @@ export const typeMapSet = (transaction, parent, key, value) => { const left = parent._map.get(key) || null let content if (value == null) { - content = new ContentJSON([value]) + content = new ContentAny([value]) } else { switch (value.constructor) { case Number: @@ -511,7 +511,7 @@ export const typeMapSet = (transaction, parent, key, value) => { case Boolean: case Array: case String: - content = new ContentJSON([value]) + content = new ContentAny([value]) break case Uint8Array: content = new ContentBinary(value) diff --git a/tests/encoding.tests.js b/tests/encoding.tests.js index 387c385d..a9160ef8 100644 --- a/tests/encoding.tests.js +++ b/tests/encoding.tests.js @@ -8,19 +8,21 @@ import { readContentJSON, readContentEmbed, readContentType, - readContentFormat + readContentFormat, + readContentAny } from '../src/internals.js' /** * @param {t.TestCase} tc */ export const testStructReferences = tc => { - t.assert(contentRefs.length === 8) + t.assert(contentRefs.length === 9) t.assert(contentRefs[1] === readContentDeleted) - t.assert(contentRefs[2] === readContentJSON) + t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json? t.assert(contentRefs[3] === readContentBinary) t.assert(contentRefs[4] === readContentString) t.assert(contentRefs[5] === readContentEmbed) t.assert(contentRefs[6] === readContentFormat) t.assert(contentRefs[7] === readContentType) + t.assert(contentRefs[8] === readContentAny) } From f5c66e41cbbd3922ad294df70b6db8d36cb69c0a Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 9 Aug 2019 01:16:40 +0200 Subject: [PATCH 04/34] audit --- package-lock.json | 107 ++++++++++++++++++---------------------------- package.json | 2 +- 2 files changed, 43 insertions(+), 66 deletions(-) diff --git a/package-lock.json b/package-lock.json index b81434e6..e0733180 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "@babel/parser": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz", - "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.5.5.tgz", + "integrity": "sha512-E5BN68cqR7dhKan1SfqgPGhQ178bkVKpXTPEXnFJBrEt8/DKRZlybmy+IgYLTeN7tp1R5Ccmbm2rBk17sHYU3g==", "dev": true }, "@types/estree": { @@ -431,12 +431,12 @@ "dev": true }, "catharsis": { - "version": "0.8.10", - "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.10.tgz", - "integrity": "sha512-l2OUaz/3PU3MZylspVFJvwHCVfWyvcduPq4lv3AzZ2pJzZCo7kNKFNyatwujD7XgvGkNAE/Jhhbh2uARNwNkfw==", + "version": "0.8.11", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.11.tgz", + "integrity": "sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g==", "dev": true, "requires": { - "lodash": "^4.17.11" + "lodash": "^4.17.14" } }, "chalk": { @@ -2514,22 +2514,22 @@ } }, "jsdoc": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.2.tgz", - "integrity": "sha512-S2vzg99C5+gb7FWlrK4TVdyzVPGGkdvpDkCEJH1JABi2PKzPeLu5/zZffcJUifgWUJqXWl41Hoc+MmuM2GukIg==", + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.3.tgz", + "integrity": "sha512-Yf1ZKA3r9nvtMWHO1kEuMZTlHOF8uoQ0vyo5eH7SQy5YeIiHM+B0DgKnn+X6y6KDYZcF7G2SPkKF+JORCXWE/A==", "dev": true, "requires": { "@babel/parser": "^7.4.4", "bluebird": "^3.5.4", - "catharsis": "^0.8.10", + "catharsis": "^0.8.11", "escape-string-regexp": "^2.0.0", "js2xmlparser": "^4.0.0", "klaw": "^3.0.0", "markdown-it": "^8.4.2", "markdown-it-anchor": "^5.0.2", - "marked": "^0.6.2", + "marked": "^0.7.0", "mkdirp": "^0.5.1", - "requizzle": "^0.2.2", + "requizzle": "^0.2.3", "strip-json-comments": "^3.0.1", "taffydb": "2.6.2", "underscore": "~1.9.1" @@ -2601,9 +2601,9 @@ "integrity": "sha512-drb8LcwZu2rAmTsXN0d3hFtZVbPE5ZUrsWf307Boc/v7IrmLq3lM5+OOMY672EysHTWeXo/OH54wRHyD6eFXXw==" }, "linkify-it": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.1.0.tgz", - "integrity": "sha512-4REs8/062kV2DSHxNfq5183zrqXMl7WP0WzABH9IeJI+NLm429FgE1PDecltYfnOoFDFlZGh2T8PfZn0r+GTRg==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz", + "integrity": "sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==", "dev": true, "requires": { "uc.micro": "^1.0.1" @@ -2653,9 +2653,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "lodash.assignin": { @@ -2701,9 +2701,9 @@ "dev": true }, "lodash.merge": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", - "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, "lodash.pick": { @@ -2784,15 +2784,15 @@ } }, "markdown-it-anchor": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.1.0.tgz", - "integrity": "sha512-wJOmyXzDUxI8iuowEsaQAKMQBButhSw8j64SpgcaL75QZYC/OSZV66Fnr50lfMLYNGtV0rJdw2fmLwXCT6T+bw==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.2.4.tgz", + "integrity": "sha512-n8zCGjxA3T+Mx1pG8HEgbJbkB8JFUuRkeTZQuIM8iPY6oQ8sWOPRZJDFC9a/pNg2QkHEjjGkhBEl/RSyzaDZ3A==", "dev": true }, "marked": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.6.2.tgz", - "integrity": "sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==", + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-0.7.0.tgz", + "integrity": "sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==", "dev": true }, "mdurl": { @@ -2865,9 +2865,9 @@ "dev": true }, "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", "dev": true, "requires": { "for-in": "^1.0.2", @@ -3437,12 +3437,12 @@ } }, "requizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.2.tgz", - "integrity": "sha512-oJ6y7JcUJkblRGhMByGNcszeLgU0qDxNKFCiUZR1XyzHyVsev+Mxb1tyygxLd1ORsKee1SA5BInFdUwY64GE/A==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.3.tgz", + "integrity": "sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ==", "dev": true, "requires": { - "lodash": "^4.17.11" + "lodash": "^4.17.14" } }, "resolve": { @@ -3648,9 +3648,9 @@ } }, "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", "dev": true, "requires": { "extend-shallow": "^2.0.1", @@ -4176,38 +4176,15 @@ "dev": true }, "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", "dev": true, "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", "is-extendable": "^0.1.1", - "set-value": "^0.4.3" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "dev": true, - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } - } + "set-value": "^2.0.1" } }, "uniq": { diff --git a/package.json b/package.json index c6140240..343d488e 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ }, "devDependencies": { "concurrently": "^3.6.1", - "jsdoc": "^3.6.2", + "jsdoc": "^3.6.3", "live-server": "^1.2.1", "rollup": "^1.11.3", "rollup-cli": "^1.0.9", From 1337d38ada427b3deabf493fa1c0657e6814b7f2 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 9 Aug 2019 01:18:15 +0200 Subject: [PATCH 05/34] 13.0.0-95 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index e0733180..5c229372 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-94", + "version": "13.0.0-95", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 343d488e..22ecb34e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-94", + "version": "13.0.0-95", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From 251c8aaefcd8b0f626c2ab23ed19aada49aed04b Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 20 Aug 2019 22:28:49 +0200 Subject: [PATCH 06/34] UndoManager configuration to filter deletes --- README.v13.md | 2 +- src/utils/UndoManager.js | 39 ++++++++++++++++++++++++++++++--------- tests/undo-redo.tests.js | 21 ++++++++++++++++++++- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/README.v13.md b/README.v13.md index 8adcab73..fc2afb51 100644 --- a/README.v13.md +++ b/README.v13.md @@ -662,7 +662,7 @@ ytext.toString() // => 'abc'
constructor(scope:Y.AbstractType|Array<Y.AbstractType>, - [trackedTransactionOrigins:Set<any>, [{captureTimeout: number}]]) + [[{captureTimeout:number,trackedOrigins:Set<any>,deleteFilter:function(item):boolean}]])
Accepts either single type as scope or an array of types.
undo()
diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index 4a28205a..789321a2 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -61,6 +61,10 @@ const popStackItem = (undoManager, stack, eventType) => { performedChange = redoItem(transaction, item, itemsToRedo) !== null || performedChange }) const structs = /** @type {Array} */ (store.clients.get(doc.clientID)) + /** + * @type {Array} + */ + const itemsToDelete = [] iterateStructs(transaction, structs, stackItem.start, stackItem.len, struct => { if (struct instanceof Item && !struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) { if (struct.redone !== null) { @@ -73,11 +77,18 @@ const popStackItem = (undoManager, stack, eventType) => { } struct = item } - keepItem(struct) - struct.delete(transaction) - performedChange = true + itemsToDelete.push(struct) } }) + // We want to delete in reverse order so that children are deleted before + // parents, so we have more information available when items are filtered. + for (let i = itemsToDelete.length - 1; i >= 0; i--) { + const item = itemsToDelete[i] + if (undoManager.deleteFilter(item)) { + item.delete(transaction) + performedChange = true + } + } result = stackItem if (result != null) { undoManager.emit('stack-item-popped', [{ stackItem: result, type: eventType }, undoManager]) @@ -87,6 +98,16 @@ const popStackItem = (undoManager, stack, eventType) => { return result } +/** + * @typedef {Object} UndoManagerOptions + * @property {number} [UndoManagerOptions.captureTimeout=500] + * @property {function(Item):boolean} [UndoManagerOptions.deleteFilter=()=>true] Sometimes + * it is necessary to filter whan an Undo/Redo operation can delete. If this + * filter returns false, the type/item won't be deleted even it is in the + * undo/redo scope. + * @property {Set} [UndoManagerOptions.trackedOrigins=new Set([null])] + */ + /** * Fires 'stack-item-added' event when a stack item was added to either the undo- or * the redo-stack. You may store additional stack information via the @@ -99,17 +120,17 @@ const popStackItem = (undoManager, stack, eventType) => { export class UndoManager extends Observable { /** * @param {AbstractType|Array>} typeScope Accepts either a single type, or an array of types - * @param {Set} [trackedTransactionOrigins=new Set([null])] - * @param {object} [options={captureTimeout=500}] + * @param {UndoManagerOptions} options */ - constructor (typeScope, trackedTransactionOrigins = new Set([null]), { captureTimeout } = {}) { + constructor (typeScope, { captureTimeout, deleteFilter = () => true, trackedOrigins = new Set([null]) } = {}) { if (captureTimeout == null) { captureTimeout = 500 } super() this.scope = typeScope instanceof Array ? typeScope : [typeScope] - trackedTransactionOrigins.add(this) - this.trackedTransactionOrigins = trackedTransactionOrigins + this.deleteFilter = deleteFilter + trackedOrigins.add(this) + this.trackedOrigins = trackedOrigins /** * @type {Array} */ @@ -129,7 +150,7 @@ export class UndoManager extends Observable { this.lastChange = 0 this.doc.on('afterTransaction', /** @param {Transaction} transaction */ transaction => { // Only track certain transactions - if (!this.scope.some(type => transaction.changedParentTypes.has(type)) || (!this.trackedTransactionOrigins.has(transaction.origin) && (!transaction.origin || !this.trackedTransactionOrigins.has(transaction.origin.constructor)))) { + if (!this.scope.some(type => transaction.changedParentTypes.has(type)) || (!this.trackedOrigins.has(transaction.origin) && (!transaction.origin || !this.trackedOrigins.has(transaction.origin.constructor)))) { return } const undoing = this.undoing diff --git a/tests/undo-redo.tests.js b/tests/undo-redo.tests.js index ca4051f8..cc4ee027 100644 --- a/tests/undo-redo.tests.js +++ b/tests/undo-redo.tests.js @@ -172,7 +172,7 @@ export const testUndoEvents = tc => { export const testTrackClass = tc => { const { users, text0 } = init(tc, { users: 3 }) // only track origins that are numbers - const undoManager = new UndoManager(text0, new Set([Number])) + const undoManager = new UndoManager(text0, { trackedOrigins: new Set([Number]) }) users[0].transact(() => { text0.insert(0, 'abc') }, 42) @@ -201,3 +201,22 @@ export const testTypeScope = tc => { undoManagerBoth.undo() t.assert(text1.toString() === '') } + +/** + * @param {t.TestCase} tc + */ +export const testUndoDeleteFilter = tc => { + /** + * @type {Array>} + */ + const array0 = /** @type {any} */ (init(tc, { users: 3 }).array0) + const undoManager = new UndoManager(array0, { deleteFilter: item => !(item instanceof Y.Item) || (item.content instanceof Y.ContentType && item.content.type._map.size === 0) }) + const map0 = new Y.Map() + map0.set('hi', 1) + const map1 = new Y.Map() + array0.insert(0, [map0, map1]) + undoManager.undo() + t.assert(array0.length === 1) + array0.get(0) + t.assert(Array.from(array0.get(0).keys()).length === 1) +} From fc4d6165b4ede8a3b0d83da82797cadb63777369 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 20 Aug 2019 22:29:56 +0200 Subject: [PATCH 07/34] 13.0.0-96 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5c229372..66a2f577 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-95", + "version": "13.0.0-96", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 22ecb34e..0f318536 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-95", + "version": "13.0.0-96", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From 13ad0c8464dd3886c8a66c5016d8398d2e3f40ba Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 27 Aug 2019 02:17:08 +0200 Subject: [PATCH 08/34] implement Y.XmlFragment.length --- src/types/YText.js | 2 +- src/types/YXmlFragment.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/types/YText.js b/src/types/YText.js index 167841b9..a2121a99 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -272,7 +272,7 @@ const formatText = (transaction, parent, left, right, currentAttributes, length, // iterate until first non-format or null is found // delete all formats with attributes[format.key] != null while (length > 0 && right !== null) { - if (right.deleted === false) { + if (!right.deleted) { switch (right.content.constructor) { case ContentFormat: const { key, value } = /** @type {ContentFormat} */ (right.content) diff --git a/src/types/YXmlFragment.js b/src/types/YXmlFragment.js index 62fa543e..247f4b1b 100644 --- a/src/types/YXmlFragment.js +++ b/src/types/YXmlFragment.js @@ -151,6 +151,10 @@ export class YXmlFragment extends AbstractType { return new YXmlFragment() } + get length () { + return this._prelimContent === null ? this._length : this._prelimContent.length + } + /** * Create a subtree of childNodes. * From 5fddcef3eac52afadb9e45efd8ee77d7499a228d Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 29 Aug 2019 12:51:16 +0200 Subject: [PATCH 09/34] Update logo --- README.v13.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.v13.md b/README.v13.md index de79010e..3b7d195c 100644 --- a/README.v13.md +++ b/README.v13.md @@ -1,5 +1,5 @@ -# ![Yjs](https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png) +# ![Yjs](https://yjs.dev/images/logo/yjs-120x120.png) > A CRDT framework with a powerful abstraction of shared data From c23bcb66ce4860f0f965004036d0f2b776bd0691 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 31 Aug 2019 16:44:07 +0200 Subject: [PATCH 10/34] delta format: use flat attr comparison --- src/types/YText.js | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/types/YText.js b/src/types/YText.js index a2121a99..b73e0e97 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -21,6 +21,14 @@ import { import * as decoding from 'lib0/decoding.js' // eslint-disable-line import * as encoding from 'lib0/encoding.js' +import * as object from 'lib0/object.js' + +/** + * @param {any} a + * @param {any} b + * @return {boolean} + */ +const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b)) export class ItemListPosition { /** @@ -130,7 +138,7 @@ const insertNegatedAttributes = (transaction, parent, left, right, negatedAttrib right !== null && ( right.deleted === true || ( right.content.constructor === ContentFormat && - (negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key) === /** @type {ContentFormat} */ (right.content).value) + equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key), /** @type {ContentFormat} */ (right.content).value) ) ) ) { @@ -180,7 +188,7 @@ const minimizeAttributeChanges = (left, right, currentAttributes, attributes) => break } else if (right.deleted) { // continue - } else if (right.content.constructor === ContentFormat && (attributes[(/** @type {ContentFormat} */ (right.content)).key] || null) === /** @type {ContentFormat} */ (right.content).value) { + } else if (right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (right.content)).key] || null, /** @type {ContentFormat} */ (right.content).value)) { // found a format, update currentAttributes and continue updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content)) } else { @@ -210,7 +218,7 @@ const insertAttributes = (transaction, parent, left, right, currentAttributes, a for (let key in attributes) { const val = attributes[key] const currentVal = currentAttributes.get(key) || null - if (currentVal !== val) { + if (!equalAttrs(currentVal, val)) { // save negated attribute (set null if currentVal undefined) negatedAttributes.set(key, currentVal) left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentFormat(key, val)) @@ -278,7 +286,7 @@ const formatText = (transaction, parent, left, right, currentAttributes, length, const { key, value } = /** @type {ContentFormat} */ (right.content) const attr = attributes[key] if (attr !== undefined) { - if (attr === value) { + if (equalAttrs(attr, value)) { negatedAttributes.delete(key) } else { negatedAttributes.set(key, value) @@ -516,11 +524,11 @@ export class YTextEvent extends YEvent { if (this.adds(item)) { if (!this.deletes(item)) { const curVal = currentAttributes.get(key) || null - if (curVal !== value) { + if (!equalAttrs(curVal, value)) { if (action === 'retain') { addOp() } - if (value === (oldAttributes.get(key) || null)) { + if (equalAttrs(value, (oldAttributes.get(key) || null))) { delete attributes[key] } else { attributes[key] = value @@ -532,7 +540,7 @@ export class YTextEvent extends YEvent { } else if (this.deletes(item)) { oldAttributes.set(key, value) const curVal = currentAttributes.get(key) || null - if (curVal !== value) { + if (!equalAttrs(curVal, value)) { if (action === 'retain') { addOp() } @@ -542,7 +550,7 @@ export class YTextEvent extends YEvent { oldAttributes.set(key, value) const attr = attributes[key] if (attr !== undefined) { - if (attr !== value) { + if (!equalAttrs(attr, value)) { if (action === 'retain') { addOp() } @@ -784,12 +792,12 @@ export class YText extends AbstractType { * * @param {number} index The index at which to start inserting. * @param {String} text The text to insert at the specified position. - * @param {TextAttributes} attributes Optionally define some formatting + * @param {TextAttributes} [attributes] Optionally define some formatting * information to apply on the inserted * Text. * @public */ - insert (index, text, attributes = {}) { + insert (index, text, attributes) { if (text.length <= 0) { return } @@ -797,6 +805,10 @@ export class YText extends AbstractType { if (y !== null) { transact(y, transaction => { const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + if (!attributes) { + attributes = {} + currentAttributes.forEach((v, k) => { attributes[k] = v }) + } insertText(transaction, this, left, right, currentAttributes, text, attributes) }) } else { From e78d84ee59f3cd9e79a5927c34b657071132a513 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 31 Aug 2019 16:47:12 +0200 Subject: [PATCH 11/34] md lint --- README.v13.md | 7 ++++++- src/types/YText.js | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.v13.md b/README.v13.md index 3b7d195c..ad5becef 100644 --- a/README.v13.md +++ b/README.v13.md @@ -186,7 +186,12 @@ position 0.
length:number
- forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, index:number, array: Y.Array)) + + +forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, + index:number, array: Y.Array)) + +
map(function(T, number, YArray):M):Array<M>
diff --git a/src/types/YText.js b/src/types/YText.js index b73e0e97..63e48175 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -28,7 +28,7 @@ import * as object from 'lib0/object.js' * @param {any} b * @return {boolean} */ -const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b)) +const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b)) export class ItemListPosition { /** From 8bcff6138c8b8256a37a15b3749d5799f5383b02 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 31 Aug 2019 22:42:18 +0200 Subject: [PATCH 12/34] Y.Text snapshot support (toDelta) --- src/index.js | 2 + src/types/YText.js | 84 +++++++++++++++++++++++----------------- src/utils/Snapshot.js | 34 +++++++++++++++- src/utils/Transaction.js | 5 +++ tests/y-text.tests.js | 66 ++++++++++++++++++++++++++++++- 5 files changed, 153 insertions(+), 38 deletions(-) diff --git a/src/index.js b/src/index.js index b3288e06..90598ba8 100644 --- a/src/index.js +++ b/src/index.js @@ -37,6 +37,8 @@ export { compareIDs, getState, Snapshot, + createSnapshot, + createSnapshotFromDoc, findRootTypeKey, typeListToArraySnapshot, typeMapGetSnapshot, diff --git a/src/types/YText.js b/src/types/YText.js index 63e48175..01f82ace 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -16,6 +16,7 @@ import { ContentEmbed, ContentFormat, ContentString, + splitSnapshotAffectedStructs, Doc, Item, Snapshot, StructStore, Transaction // eslint-disable-line } from '../internals.js' @@ -723,6 +724,7 @@ export class YText extends AbstractType { */ const ops = [] const currentAttributes = new Map() + const doc = /** @type {Doc} */ (this.doc) let str = '' let n = this._start function packStr () { @@ -748,42 +750,54 @@ export class YText extends AbstractType { str = '' } } - while (n !== null) { - if (isVisible(n, snapshot) || (prevSnapshot !== undefined && isVisible(n, prevSnapshot))) { - switch (n.content.constructor) { - case ContentString: - const cur = currentAttributes.get('ychange') - if (snapshot !== undefined && !isVisible(n, snapshot)) { - if (cur === undefined || cur.user !== n.id.client || cur.state !== 'removed') { - packStr() - currentAttributes.set('ychange', { user: n.id.client, state: 'removed' }) - } - } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { - if (cur === undefined || cur.user !== n.id.client || cur.state !== 'added') { - packStr() - currentAttributes.set('ychange', { user: n.id.client, state: 'added' }) - } - } else if (cur !== undefined) { - packStr() - currentAttributes.delete('ychange') - } - str += /** @type {ContentString} */ (n.content).str - break - case ContentEmbed: - packStr() - ops.push({ - insert: /** @type {ContentEmbed} */ (n.content).embed - }) - break - case ContentFormat: - packStr() - updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (n.content)) - break - } + // snapshots are merged again after the transaction, so we need to keep the + // transalive until we are done + transact(doc, transaction => { + if (snapshot) { + splitSnapshotAffectedStructs(transaction, snapshot) } - n = n.right - } - packStr() + if (prevSnapshot) { + splitSnapshotAffectedStructs(transaction, prevSnapshot) + } + while (n !== null) { + if (isVisible(n, snapshot) || (prevSnapshot !== undefined && isVisible(n, prevSnapshot))) { + switch (n.content.constructor) { + case ContentString: + const cur = currentAttributes.get('ychange') + if (snapshot !== undefined && !isVisible(n, snapshot)) { + if (cur === undefined || cur.user !== n.id.client || cur.state !== 'removed') { + packStr() + currentAttributes.set('ychange', { user: n.id.client, state: 'removed' }) + } + } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { + if (cur === undefined || cur.user !== n.id.client || cur.state !== 'added') { + packStr() + currentAttributes.set('ychange', { user: n.id.client, state: 'added' }) + } + } else if (cur !== undefined) { + packStr() + currentAttributes.delete('ychange') + } + str += /** @type {ContentString} */ (n.content).str + break + case ContentEmbed: + packStr() + ops.push({ + insert: /** @type {ContentEmbed} */ (n.content).embed + }) + break + case ContentFormat: + if (isVisible(n, snapshot)) { + packStr() + updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (n.content)) + } + break + } + } + n = n.right + } + packStr() + }, splitSnapshotAffectedStructs) return ops } diff --git a/src/utils/Snapshot.js b/src/utils/Snapshot.js index 2d33941f..3fb807d2 100644 --- a/src/utils/Snapshot.js +++ b/src/utils/Snapshot.js @@ -1,9 +1,17 @@ import { isDeleted, - DeleteSet, Item // eslint-disable-line + createDeleteSetFromStructStore, + getStateVector, + getItemCleanStart, + createID, + iterateDeletedStructs, + Transaction, Doc, DeleteSet, Item // eslint-disable-line } from '../internals.js' +import * as map from 'lib0/map.js' +import * as set from 'lib0/set.js' + export class Snapshot { /** * @param {DeleteSet} ds @@ -27,9 +35,16 @@ export class Snapshot { /** * @param {DeleteSet} ds * @param {Map} sm + * @return {Snapshot} */ export const createSnapshot = (ds, sm) => new Snapshot(ds, sm) +/** + * @param {Doc} doc + * @return {Snapshot} + */ +export const createSnapshotFromDoc = doc => createSnapshot(createDeleteSetFromStructStore(doc.store), getStateVector(doc.store)) + /** * @param {Item} item * @param {Snapshot|undefined} snapshot @@ -40,3 +55,20 @@ export const createSnapshot = (ds, sm) => new Snapshot(ds, sm) export const isVisible = (item, snapshot) => snapshot === undefined ? !item.deleted : ( snapshot.sm.has(item.id.client) && (snapshot.sm.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item.id) ) + +/** + * @param {Transaction} transaction + * @param {Snapshot} snapshot + */ +export const splitSnapshotAffectedStructs = (transaction, snapshot) => { + const meta = map.setIfUndefined(transaction.meta, splitSnapshotAffectedStructs, set.create) + const store = transaction.doc.store + // check if we already split for this snapshot + if (!meta.has(snapshot)) { + snapshot.sm.forEach((clock, client) => { + getItemCleanStart(transaction, store, createID(client, clock)) + }) + iterateDeletedStructs(transaction, snapshot.ds, store, item => {}) + meta.add(snapshot) + } +} diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index 2db80eeb..503bccbe 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -90,6 +90,11 @@ export class Transaction { * @type {any} */ this.origin = origin + /** + * Stores meta information on the transaction + * @type {Map} + */ + this.meta = new Map() } } diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index 2e860a79..1c9b0a85 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -1,6 +1,6 @@ -import { init, compare } from './testHelper.js' - +import * as Y from './testHelper.js' import * as t from 'lib0/testing.js' +const { init, compare } = Y /** * @param {t.TestCase} tc @@ -87,3 +87,65 @@ export const testGetDeltaWithEmbeds = tc => { insert: {linebreak: 's'} }]) } + +/** + * @param {t.TestCase} tc + */ +export const testSnapshot = tc => { + const { text0 } = init(tc, { users: 1 }) + const doc0 = /** @type {Y.Doc} */ (text0.doc) + doc0.gc = false + text0.applyDelta([{ + insert: 'abcd' + }]) + const snapshot1 = Y.createSnapshotFromDoc(doc0) + text0.applyDelta([{ + retain: 1 + }, { + insert: 'x' + }, { + delete: 1 + }]) + const snapshot2 = Y.createSnapshotFromDoc(doc0) + text0.applyDelta([{ + retain: 2 + }, { + delete: 3 + }, { + insert: 'x' + }, { + delete: 1 + }]) + const state1 = text0.toDelta(snapshot1) + t.compare(state1, [{ insert: 'abcd' }]) + const state2 = text0.toDelta(snapshot2) + t.compare(state2, [{ insert: 'axcd' }]) + const state2Diff = text0.toDelta(snapshot2, snapshot1) + // @ts-ignore Remove userid info + state2Diff.forEach(v => { + if (v.attributes && v.attributes.ychange) { + delete v.attributes.ychange.user + } + }) + t.compare(state2Diff, [{insert: 'a'}, {insert: 'x', attributes: {ychange: { state: 'added' }}}, {insert: 'b', attributes: {ychange: { state: 'removed' }}}, { insert: 'cd' }]) +} + +/** + * @param {t.TestCase} tc + */ +export const testSnapshotDeleteAfter = tc => { + const { text0 } = init(tc, { users: 1 }) + const doc0 = /** @type {Y.Doc} */ (text0.doc) + doc0.gc = false + text0.applyDelta([{ + insert: 'abcd' + }]) + const snapshot1 = Y.createSnapshotFromDoc(doc0) + text0.applyDelta([{ + retain: 4 + }, { + insert: 'e' + }]) + const state1 = text0.toDelta(snapshot1) + t.compare(state1, [{ insert: 'abcd' }]) +} From bb1c0b809f77f020031b1c16c686105c43407829 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 3 Sep 2019 16:33:29 +0200 Subject: [PATCH 13/34] implement snapshot & event.changes --- src/index.js | 9 ++- src/types/AbstractType.js | 2 +- src/utils/DeleteSet.js | 27 ++++++++- src/utils/Snapshot.js | 80 +++++++++++++++++++++++--- src/utils/YEvent.js | 116 +++++++++++++++++++++++++++++++++++++- src/utils/encoding.js | 33 ++++++----- tests/y-array.tests.js | 27 +++++++++ tests/y-map.tests.js | 48 ++++++++++++++++ tests/y-text.tests.js | 6 +- 9 files changed, 319 insertions(+), 29 deletions(-) diff --git a/src/index.js b/src/index.js index 90598ba8..74876ddc 100644 --- a/src/index.js +++ b/src/index.js @@ -38,7 +38,8 @@ export { getState, Snapshot, createSnapshot, - createSnapshotFromDoc, + snapshot, + emptySnapshot, findRootTypeKey, typeListToArraySnapshot, typeMapGetSnapshot, @@ -46,5 +47,9 @@ export { applyUpdate, encodeStateAsUpdate, encodeStateVector, - UndoManager + UndoManager, + decodeSnapshot, + encodeSnapshot, + isDeleted, + equalSnapshots } from './internals.js' diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 9207c175..82aa8fc8 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -584,7 +584,7 @@ export const typeMapHas = (parent, key) => { */ export const typeMapGetSnapshot = (parent, key, snapshot) => { let v = parent._map.get(key) || null - while (v !== null && (!snapshot.sm.has(v.id.client) || v.id.clock >= (snapshot.sm.get(v.id.client) || 0))) { + while (v !== null && (!snapshot.sv.has(v.id.client) || v.id.clock >= (snapshot.sv.get(v.id.client) || 0))) { v = v.left } return v !== null && isVisible(v, snapshot) ? v.content.getContent()[v.length - 1] : undefined diff --git a/src/utils/DeleteSet.js b/src/utils/DeleteSet.js index 747d8162..ea3b7b59 100644 --- a/src/utils/DeleteSet.js +++ b/src/utils/DeleteSet.js @@ -169,6 +169,8 @@ export const addToDeleteSet = (ds, id, length) => { map.setIfUndefined(ds.clients, id.client, () => []).push(new DeleteItem(id.clock, length)) } +export const createDeleteSet = () => new DeleteSet() + /** * @param {StructStore} ss * @return {DeleteSet} Merged and sorted DeleteSet @@ -177,7 +179,7 @@ export const addToDeleteSet = (ds, id, length) => { * @function */ export const createDeleteSetFromStructStore = ss => { - const ds = new DeleteSet() + const ds = createDeleteSet() ss.clients.forEach((structs, client) => { /** * @type {Array} @@ -224,6 +226,26 @@ export const writeDeleteSet = (encoder, ds) => { }) } +/** + * @param {decoding.Decoder} decoder + * @return {DeleteSet} + * + * @private + * @function + */ +export const readDeleteSet = decoder => { + const ds = new DeleteSet() + const numClients = decoding.readVarUint(decoder) + for (let i = 0; i < numClients; i++) { + const client = decoding.readVarUint(decoder) + const numberOfDeletes = decoding.readVarUint(decoder) + for (let i = 0; i < numberOfDeletes; i++) { + addToDeleteSet(ds, createID(client, decoding.readVarUint(decoder)), decoding.readVarUint(decoder)) + } + } + return ds +} + /** * @param {decoding.Decoder} decoder * @param {Transaction} transaction @@ -232,7 +254,7 @@ export const writeDeleteSet = (encoder, ds) => { * @private * @function */ -export const readDeleteSet = (decoder, transaction, store) => { +export const readAndApplyDeleteSet = (decoder, transaction, store) => { const unappliedDS = new DeleteSet() const numClients = decoding.readVarUint(decoder) for (let i = 0; i < numClients; i++) { @@ -279,6 +301,7 @@ export const readDeleteSet = (decoder, transaction, store) => { } } if (unappliedDS.clients.size > 0) { + // TODO: no need for encoding+decoding ds anymore const unappliedDSEncoder = encoding.createEncoder() writeDeleteSet(unappliedDSEncoder, unappliedDS) store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toUint8Array(unappliedDSEncoder))) diff --git a/src/utils/Snapshot.js b/src/utils/Snapshot.js index 3fb807d2..f9d2714e 100644 --- a/src/utils/Snapshot.js +++ b/src/utils/Snapshot.js @@ -6,18 +6,26 @@ import { getItemCleanStart, createID, iterateDeletedStructs, + writeDeleteSet, + writeStateVector, + readDeleteSet, + readStateVector, + createDeleteSet, + getState, Transaction, Doc, DeleteSet, Item // eslint-disable-line } from '../internals.js' import * as map from 'lib0/map.js' import * as set from 'lib0/set.js' +import * as encoding from 'lib0/encoding.js' +import * as decoding from 'lib0/decoding.js' export class Snapshot { /** * @param {DeleteSet} ds - * @param {Map} sm state map + * @param {Map} sv state map */ - constructor (ds, sm) { + constructor (ds, sv) { /** * @type {DeleteSet} * @private @@ -28,10 +36,64 @@ export class Snapshot { * @type {Map} * @private */ - this.sm = sm + this.sv = sv } } +/** + * @param {Snapshot} snap1 + * @param {Snapshot} snap2 + * @return {boolean} + */ +export const equalSnapshots = (snap1, snap2) => { + const ds1 = snap1.ds.clients + const ds2 = snap2.ds.clients + const sv1 = snap1.sv + const sv2 = snap2.sv + if (sv1.size !== sv2.size || ds1.size !== ds2.size) { + return false + } + for (const [key, value] of sv1) { + if (sv2.get(key) !== value) { + return false + } + } + for (const [client, dsitems1] of ds1) { + const dsitems2 = ds2.get(client) || [] + if (dsitems1.length !== dsitems2.length) { + return false + } + for (let i = 0; i < dsitems1.length; i++) { + const dsitem1 = dsitems1[i] + const dsitem2 = dsitems2[i] + if (dsitem1.clock !== dsitem2.clock || dsitem1.len !== dsitem2.len) { + return false + } + } + } + return true +} + +/** + * @param {Snapshot} snapshot + * @return {Uint8Array} + */ +export const encodeSnapshot = snapshot => { + const encoder = encoding.createEncoder() + writeDeleteSet(encoder, snapshot.ds) + writeStateVector(encoder, snapshot.sv) + return encoding.toUint8Array(encoder) +} + +/** + * @param {Uint8Array} buf + * @return {Snapshot} + */ +export const decodeSnapshot = buf => { + const decoder = decoding.createDecoder(buf) + return new Snapshot(readDeleteSet(decoder), readStateVector(decoder)) +} + /** * @param {DeleteSet} ds * @param {Map} sm @@ -39,11 +101,13 @@ export class Snapshot { */ export const createSnapshot = (ds, sm) => new Snapshot(ds, sm) +export const emptySnapshot = createSnapshot(createDeleteSet(), new Map()) + /** * @param {Doc} doc * @return {Snapshot} */ -export const createSnapshotFromDoc = doc => createSnapshot(createDeleteSetFromStructStore(doc.store), getStateVector(doc.store)) +export const snapshot = doc => createSnapshot(createDeleteSetFromStructStore(doc.store), getStateVector(doc.store)) /** * @param {Item} item @@ -53,7 +117,7 @@ export const createSnapshotFromDoc = doc => createSnapshot(createDeleteSetFromSt * @function */ export const isVisible = (item, snapshot) => snapshot === undefined ? !item.deleted : ( - snapshot.sm.has(item.id.client) && (snapshot.sm.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item.id) + snapshot.sv.has(item.id.client) && (snapshot.sv.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item.id) ) /** @@ -65,8 +129,10 @@ export const splitSnapshotAffectedStructs = (transaction, snapshot) => { const store = transaction.doc.store // check if we already split for this snapshot if (!meta.has(snapshot)) { - snapshot.sm.forEach((clock, client) => { - getItemCleanStart(transaction, store, createID(client, clock)) + snapshot.sv.forEach((clock, client) => { + if (clock < getState(store, client)) { + getItemCleanStart(transaction, store, createID(client, clock)) + } }) iterateDeletedStructs(transaction, snapshot.ds, store, item => {}) meta.add(snapshot) diff --git a/src/utils/YEvent.js b/src/utils/YEvent.js index 63435563..05429ee7 100644 --- a/src/utils/YEvent.js +++ b/src/utils/YEvent.js @@ -1,9 +1,12 @@ import { isDeleted, - AbstractType, Transaction, AbstractStruct // eslint-disable-line + Item, AbstractType, Transaction, AbstractStruct // eslint-disable-line } from '../internals.js' +import * as set from 'lib0/set.js' +import * as array from 'lib0/array.js' + /** * YEvent describes the changes on a YType. */ @@ -28,6 +31,10 @@ export class YEvent { * @type {Transaction} */ this.transaction = transaction + /** + * @type {Object|null} + */ + this._changes = null } /** @@ -65,6 +72,113 @@ export class YEvent { adds (struct) { return struct.id.clock >= (this.transaction.beforeState.get(struct.id.client) || 0) } + + /** + * @return {{added:Set,deleted:Set,delta:Array<{insert:Array}|{delete:number}|{retain:number}>}} + */ + get changes () { + let changes = this._changes + if (changes === null) { + const target = this.target + const added = set.create() + const deleted = set.create() + /** + * @type {Array<{insert:Array}|{delete:number}|{retain:number}>} + */ + const delta = [] + /** + * @type {Map} + */ + const keys = new Map() + changes = { + added, deleted, delta, keys + } + const changed = /** @type Set */ (this.transaction.changed.get(target)) + if (changed.has(null)) { + /** + * @type {any} + */ + let lastOp = null + const packOp = () => { + if (lastOp) { + delta.push(lastOp) + } + } + for (let item = target._start; item !== null; item = item.right) { + if (item.deleted) { + if (this.deletes(item)) { + if (lastOp === null || lastOp.delete === undefined) { + packOp() + lastOp = { delete: 0 } + } + lastOp.delete += item.length + deleted.add(item) + } // else nop + } else { + if (this.adds(item)) { + if (lastOp === null || lastOp.insert === undefined) { + packOp() + lastOp = { insert: [] } + } + lastOp.insert = lastOp.insert.concat(item.content.getContent()) + added.add(item) + } else { + if (lastOp === null || lastOp.retain === undefined) { + packOp() + lastOp = { retain: 0 } + } + lastOp.retain += item.length + } + } + } + if (lastOp !== null && lastOp.retain === undefined) { + packOp() + } + } + changed.forEach(key => { + if (key !== null) { + const item = /** @type {Item} */ (target._map.get(key)) + /** + * @type {'delete' | 'add' | 'update'} + */ + let action + let oldValue + if (this.adds(item)) { + let prev = item.left + while (prev !== null && this.adds(prev)) { + prev = prev.left + } + if (this.deletes(item)) { + if (prev !== null && this.deletes(prev)) { + action = 'delete' + oldValue = array.last(prev.content.getContent()) + } else { + return + } + } else { + if (prev !== null && this.deletes(prev)) { + action = 'update' + oldValue = array.last(prev.content.getContent()) + } else { + action = 'add' + oldValue = undefined + } + } + } else { + if (this.deletes(item)) { + action = 'delete' + oldValue = array.last(/** @type {Item} */ item.content.getContent()) + } else { + return // nop + } + } + keys.set(key, { action, oldValue }) + } + }) + this._changes = changes + } + return changes + } } /** diff --git a/src/utils/encoding.js b/src/utils/encoding.js index c4ee0d2a..a0ebf23f 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -23,7 +23,7 @@ import { readID, getState, getStateVector, - readDeleteSet, + readAndApplyDeleteSet, writeDeleteSet, createDeleteSetFromStructStore, Doc, Transaction, AbstractStruct, StructStore, ID // eslint-disable-line @@ -230,7 +230,7 @@ export const tryResumePendingDeleteReaders = (transaction, store) => { const pendingReaders = store.pendingDeleteReaders store.pendingDeleteReaders = [] for (let i = 0; i < pendingReaders.length; i++) { - readDeleteSet(pendingReaders[i], transaction, store) + readAndApplyDeleteSet(pendingReaders[i], transaction, store) } } @@ -301,7 +301,7 @@ export const readStructs = (decoder, transaction, store) => { export const readUpdate = (decoder, ydoc, transactionOrigin) => ydoc.transact(transaction => { readStructs(decoder, transaction, ydoc.store) - readDeleteSet(decoder, transaction, ydoc.store) + readAndApplyDeleteSet(decoder, transaction, ydoc.store) }, transactionOrigin) /** @@ -381,6 +381,22 @@ export const readStateVector = decoder => { */ export const decodeStateVector = decodedState => readStateVector(decoding.createDecoder(decodedState)) +/** + * Write State Vector to `lib0/encoding.js#Encoder`. + * + * @param {encoding.Encoder} encoder + * @param {Map} sv + * @function + */ +export const writeStateVector = (encoder, sv) => { + encoding.writeVarUint(encoder, sv.size) + sv.forEach((clock, client) => { + encoding.writeVarUint(encoder, client) + encoding.writeVarUint(encoder, clock) + }) + return encoder +} + /** * Write State Vector to `lib0/encoding.js#Encoder`. * @@ -389,16 +405,7 @@ export const decodeStateVector = decodedState => readStateVector(decoding.create * * @function */ -export const writeDocumentStateVector = (encoder, doc) => { - encoding.writeVarUint(encoder, doc.store.clients.size) - doc.store.clients.forEach((structs, client) => { - const struct = structs[structs.length - 1] - const id = struct.id - encoding.writeVarUint(encoder, id.client) - encoding.writeVarUint(encoder, id.clock + struct.length) - }) - return encoder -} +export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encoder, getStateVector(doc.store)) /** * Encode State as Uint8Array. diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index b3bb8454..953e862f 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -191,6 +191,33 @@ export const testInsertAndDeleteEventsForTypes = tc => { compare(users) } +/** + * @param {t.TestCase} tc + */ +export const testChangeEvent = tc => { + const { array0, users } = init(tc, { users: 2 }) + /** + * @type {any} + */ + let changes = null + array0.observe(e => { + changes = e.changes + }) + const newArr = new Y.Array() + array0.insert(0, [newArr, 4, 'dtrn']) + t.assert(changes !== null && changes.added.size === 2 && changes.deleted.size === 0) + t.compare(changes.delta, [{insert: [newArr, 4, 'dtrn']}]) + changes = null + array0.delete(0, 2) + t.assert(changes !== null && changes.added.size === 0 && changes.deleted.size === 2) + t.compare(changes.delta, [{ delete: 2 }]) + changes = null + array0.insert(1, [0.1]) + t.assert(changes !== null && changes.added.size === 1 && changes.deleted.size === 0) + t.compare(changes.delta, [{ retain: 1 }, { insert: [0.1] }]) + compare(users) +} + /** * @param {t.TestCase} tc */ diff --git a/tests/y-map.tests.js b/tests/y-map.tests.js index c47c9e46..becfde34 100644 --- a/tests/y-map.tests.js +++ b/tests/y-map.tests.js @@ -292,6 +292,54 @@ export const testThrowsAddAndUpdateAndDeleteEvents = tc => { compare(users) } +/** + * @param {t.TestCase} tc + */ +export const testChangeEvent = tc => { + const { map0, users } = init(tc, { users: 2 }) + /** + * @type {any} + */ + let changes = null + /** + * @type {any} + */ + let keyChange = null + map0.observe(e => { + changes = e.changes + }) + map0.set('a', 1) + keyChange = changes.keys.get('a') + t.assert(changes !== null && keyChange.action === 'add' && keyChange.oldValue === undefined) + map0.set('a', 2) + keyChange = changes.keys.get('a') + t.assert(changes !== null && keyChange.action === 'update' && keyChange.oldValue === 1) + users[0].transact(() => { + map0.set('a', 3) + map0.set('a', 4) + }) + keyChange = changes.keys.get('a') + t.assert(changes !== null && keyChange.action === 'update' && keyChange.oldValue === 2) + users[0].transact(() => { + map0.set('b', 1) + map0.set('b', 2) + }) + keyChange = changes.keys.get('b') + t.assert(changes !== null && keyChange.action === 'add' && keyChange.oldValue === undefined) + users[0].transact(() => { + map0.set('c', 1) + map0.delete('c') + }) + t.assert(changes !== null && changes.keys.size === 0) + users[0].transact(() => { + map0.set('d', 1) + map0.set('d', 2) + }) + keyChange = changes.keys.get('d') + t.assert(changes !== null && keyChange.action === 'add' && keyChange.oldValue === undefined) + compare(users) +} + /** * @param {t.TestCase} tc */ diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index 1c9b0a85..f422ed45 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -98,7 +98,7 @@ export const testSnapshot = tc => { text0.applyDelta([{ insert: 'abcd' }]) - const snapshot1 = Y.createSnapshotFromDoc(doc0) + const snapshot1 = Y.snapshot(doc0) text0.applyDelta([{ retain: 1 }, { @@ -106,7 +106,7 @@ export const testSnapshot = tc => { }, { delete: 1 }]) - const snapshot2 = Y.createSnapshotFromDoc(doc0) + const snapshot2 = Y.snapshot(doc0) text0.applyDelta([{ retain: 2 }, { @@ -140,7 +140,7 @@ export const testSnapshotDeleteAfter = tc => { text0.applyDelta([{ insert: 'abcd' }]) - const snapshot1 = Y.createSnapshotFromDoc(doc0) + const snapshot1 = Y.snapshot(doc0) text0.applyDelta([{ retain: 4 }, { From 6cb64b37078ab198354c6166724b7b2f795dcc44 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 4 Sep 2019 13:05:03 +0200 Subject: [PATCH 14/34] move repository to yjs org --- .jsdoc.json | 2 +- README.md | 1106 ++++++++++++++++++++++++++++++---------- README.v12.md | 305 +++++++++++ README.v13.md | 874 ------------------------------- package.json | 14 +- tests/y-array.tests.js | 2 +- 6 files changed, 1142 insertions(+), 1161 deletions(-) create mode 100644 README.v12.md delete mode 100644 README.v13.md diff --git a/.jsdoc.json b/.jsdoc.json index 1cca0d93..86397dac 100644 --- a/.jsdoc.json +++ b/.jsdoc.json @@ -17,7 +17,7 @@ "useCollapsibles": true, "collapse": true, "resources": { - "y-js.org": "yjs.website" + "yjs.dev": "Yjs website" }, "logo": { "url": "https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png", diff --git a/README.md b/README.md index 42e54230..be5c2aee 100644 --- a/README.md +++ b/README.md @@ -1,305 +1,855 @@ -# ![Yjs](https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png) +# ![Yjs](https://yjs.dev/images/logo/yjs-120x120.png) -Yjs is a framework for offline-first p2p shared editing on structured data like -text, richtext, json, or XML. It is fairly easy to get started, as Yjs hides -most of the complexity of concurrent editing. For additional information, demos, -and tutorials visit [y-js.org](http://y-js.org/). +> A CRDT framework with a powerful abstraction of shared data -:warning: Checkout the [v13 docs](./README.v13.md) for the upcoming release :warning: +Yjs is a [CRDT implementation](#Yjs-CRDT-Algorithm) that exposes its internal +data structure as *shared types*. Shared types are common data types like `Map` +or `Array` with superpowers: changes are automatically distributed to other +peers and merged without merge conflicts. -### Extensions -Yjs only knows how to resolve conflicts on shared data. You have to choose a .. +Yjs is **network agnostic** (p2p!), supports many existing **rich text +editors**, **offline editing**, **version snapshots**, **undo/redo** and +**shared cursors**. It scales well with an unlimited number of users and is well +suited for even large documents. -* *Connector* - a communication protocol that propagates changes to the clients -* *Database* - a database to store your changes -* one or more *Types* - that represent the shared data +* Demos: [https://github.com/yjs/yjs-demos](https://github.com/yjs/yjs-demos) +* Discuss: [https://discuss.yjs.dev](https://discuss.yjs.dev) +* Benchmarks: + [https://github.com/dmonad/crdt-benchmarks](https://github.com/dmonad/crdt-benchmarks) -Connectors, Databases, and Types are available as modules that extend Yjs. Here -is a list of the modules we know of: +:warning: This is the documentation for v13 (still in alpha). For the stable v12 +release checkout the [v12 docs](./README.v12.md) :warning: -##### Connectors +## Table of Contents -|Name | Description | -|----------------|-----------------------------------| -|[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC| -|[websockets](https://github.com/y-js/y-websockets-client) | Set up [a central server](https://github.com/y-js/y-websockets-client), and connect to it via websockets | -|[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))| -|[ipfs](https://github.com/ipfs-labs/y-ipfs-connector) | Connector for the [Interplanetary File System](https://ipfs.io/)!| -|[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios| +* [Overview](#Overview) + * [Bindings](#Bindings) + * [Providers](#Providers) +* [Getting Started](#Getting-Started) +* [API](#API) + * [Shared Types](#Shared-Types) + * [Y.Doc](#YDoc) + * [Document Updates](#Document-Updates) + * [Relative Positions](#Relative-Positions) + * [Y.UndoManager](#YUndoManager) +* [Miscellaneous](#Miscellaneous) + * [Typescript Declarations](#Typescript-Declarations) +* [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm) +* [Evaluation](#Evaluation) + * [Existing shared editing libraries](#Exisisting-Javascript-Libraries) + * [CRDT Algorithms](#CRDT-Algorithms) + * [Comparison of CRDT with OT](#Comparing-CRDT-with-OT) + * [Comparison of CRDT Algorithms](#Comparing-CRDT-Algorithms) + * [Comparison of Yjs with other Implementations](#Comparing-Yjs-with-other-Implementations) +* [License and Author](#License-and-Author) -##### Database adapters +## Overview -|Name | Description | -|----------------|-----------------------------------| -|[memory](https://github.com/y-js/y-memory) | In-memory storage. | -|[indexeddb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | -|[leveldb](https://github.com/y-js/y-leveldb) | Persistent storage for node apps | +This repository contains a collection of shared types that can be observed for +changes and manipulated concurrently. Network functionality and two-way-bindings +are implemented in separate modules. -##### Types +### Bindings -| Name | Description | -|----------|-------------------| -|[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | -|[array](https://github.com/y-js/y-array) | A shared Array implementation | -|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | -|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to the [Ace Editor](https://ace.c9.io), [CodeMirror](https://codemirror.net/), [Monaco](https://github.com/Microsoft/monaco-editor), textareas, input elements, and HTML elements (e.g. <*h1*>, or <*p*>) | -|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| +| Name | Cursors | Binding | Demo | +|---|:-:|---|---| +| [ProseMirror](https://prosemirror.net/)                                                   | ✔ | [y-prosemirror](http://github.com/yjs/y-prosemirror) | [demo](https://yjs-demos.now.sh/prosemirror/) | +| [Quill](https://quilljs.com/) | | [y-quill](http://github.com/yjs/y-quill) | [demo](https://yjs-demos.now.sh/quill/) | +| [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/yjs/y-codemirror) | [demo](https://yjs-demos.now.sh/codemirror/) | +| [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/yjs/y-monaco) | [demo](https://yjs-demos.now.sh/monaco/) | +| [Ace](https://ace.c9.io/) | | [y-ace](http://github.com/yjs/y-ace) | [demo](https://yjs-demos.now.sh/ace/) | +| [Textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) | | [y-textarea](http://github.com/yjs/y-textarea) | [demo](https://yjs-demos.now.sh/textarea/) | +| [DOM](https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model) | | [y-dom](http://github.com/yjs/y-dom) | [demo](https://yjs-demos.now.sh/dom/) | -##### Other +### Providers -| Name | Description | -|-----------|-------------------| -|[y-element](http://y-js.org/y-element/) | Yjs Polymer Element | +Setting up the communication between clients, managing awareness information, +and storing shared data for offline usage is quite a hassle. **Providers** +manage all that for you and are the perfect starting point for your +collaborative app. -## Use it! -Install Yjs, and its modules with [bower](http://bower.io/), or -[npm](https://www.npmjs.org/package/yjs). +
+
y-websocket
+
+A module that contains a simple websocket backend and a websocket client that +connects to that backend. The backend can be extended to persist updates in a +leveldb database. +
+
y-mesh
+
+[WIP] Creates a connected graph of webrtc connections with a high +strength. It +requires a signalling server that connects a client to the first peer. But after +that the network manages itself. It is well suited for large and small networks. +
+
y-dat
+
+[WIP] Write document updates effinciently to the dat network using +multifeed. Each client has +an append-only log of CRDT local updates (hypercore). Multifeed manages and sync +hypercores and y-dat listens to changes and applies them to the Yjs document. +
+
-### Bower +## Getting Started -``` -bower install --save yjs y-array % add all y-* modules you want to use -``` -You only need to include the `y.js` file. Yjs is able to automatically require -missing modules. -``` - -``` +Install Yjs and a provider with your favorite package manager: -### CDN - -``` - - - - - - -// .. -// do the same for all modules you want to use -``` - -### Npm - -``` -npm install --save yjs % add all y-* modules you want to use -``` - -If you don't include via script tag, you have to explicitly include all modules! -(Same goes for other module systems) -``` -var Y = require('yjs') -require('y-array')(Y) // add the y-array type to Yjs -require('y-websockets-client')(Y) -require('y-memory')(Y) -require('y-map')(Y) -require('y-text')(Y) -// .. -// do the same for all modules you want to use -``` - -### ES6 Syntax - -``` -import Y from 'yjs' -import yArray from 'y-array' -import yWebsocketsClient from 'y-webrtc' -import yMemory from 'y-memory' -import yMap from 'y-map' -import yText from 'y-text' -// .. -Y.extend(yArray, yWebsocketsClient, yMemory, yArray, yMap, yText /*, .. */) -``` - -# Text editing example - -Install dependencies -``` -bower i yjs y-memory y-webrtc y-array y-text -``` - -Here is a simple example of a shared textarea -```HTML - - - - - - - - - -``` - -## Get Help & Give Help -There are some friendly people on [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/y-js/yjs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) who are eager to help, and answer questions. Please join! - -Report _any_ issues to the -[Github issue page](https://github.com/y-js/yjs/issues)! I try to fix them very -soon, if possible. - -# API - -### Y(options) -* Y.extend(module1, module2, ..) - * Add extensions to Y - * `Y.extend(require('y-webrtc'))` has the same semantics as - `require('y-webrtc')(Y)` -* options.db - * Will be forwarded to the database adapter. Specify the database adaper on - `options.db.name`. - * Have a look at the used database adapter repository to see all available - options. -* options.connector - * Will be forwarded to the connector adapter. Specify the connector adaper on - `options.connector.name`. - * All our connectors implement a `room` property. Clients that specify the - same room share the same data. - * All of our connectors specify an `url` property that defines the connection - endpoint of the used connector. - * All of our connectors also have a default connection endpoint that you can - use for development. - * Set `options.connector.generateUserId = true` in order to genenerate a - userid, instead of receiving one from the server. This way the `Y(..)` is - immediately going to be resolved, without waiting for any confirmation from - the server. Use with caution. - * Have a look at the used connector repository to see all available options. - * *Only if you know what you are doing:* Set - `options.connector.preferUntransformed = true` in order receive the shared - data untransformed. This is very efficient as the database content is simply - copied to this client. This does only work if this client receives content - from only one client. -* options.sourceDir (browser only) - * Path where all y-* modules are stored - * Defaults to `/bower_components` - * Not required when running on `nodejs` / `iojs` - * When using nodejs you need to manually extend Yjs: -``` -var Y = require('yjs') -// you have to require a db, connector, and *all* types you use! -require('y-memory')(Y) -require('y-webrtc')(Y) -require('y-map')(Y) -// .. -``` -* options.share - * Specify on `options.share[arbitraryName]` types that are shared among all - users. - * E.g. Specify `options.share[arbitraryName] = 'Array'` to require y-array and - create an y-array type on `y.share[arbitraryName]`. - * If userA doesn't specify `options.share[arbitraryName]`, it won't be - available for userA. - * If userB specifies `options.share[arbitraryName]`, it still won't be - available for userA. But all the updates are send from userB to userA. - * In contrast to y-map, types on `y.share.*` cannot be overwritten or deleted. - Instead, they are merged among all users. This feature is only available on - `y.share.*` - * Weird behavior: It is supported that two users specify different types with - the same property name. - E.g. userA specifies `options.share.x = 'Array'`, and userB specifies - `options.share.x = 'Text'`. But they only share data if they specified the - same type with the same property name -* options.type (browser only) - * Array of modules that Yjs needs to require, before instantiating a shared - type. - * By default Yjs requires the specified database adapter, the specified - connector, and all modules that are used in `options.share.*` - * Put all types here that you intend to use, but are not used in y.share.* - -### Instantiated Y object (y) -`Y(options)` returns a promise that is fulfilled when.. - -* All modules are loaded - * The specified database adapter is loaded - * The specified connector is loaded - * All types are included -* The connector is initialized, and a unique user id is set (received from the - server) - * Note: When using y-indexeddb, a retrieved user id is stored on `localStorage` - -The promise returns an instance of Y. We denote it with a lower case `y`. - -* y.share.* - * Instances of the types you specified on options.share.* - * y.share.* can only be defined once when you instantiate Y! -* y.connector is an instance of Y.AbstractConnector -* y.connector.onUserEvent(function (event) {..}) - * Observe user events (event.action is either 'userLeft' or 'userJoined') -* y.connector.whenSynced(listener) - * `listener` is executed when y synced with at least one user. - * `listener` is not called when no other user is in the same room. - * y-websockets-client aways waits to sync with the server -* y.connector.disconnect() - * Force to disconnect this instance from the other instances -* y.connector.connect() - * Try to reconnect to the other instances (needs to be supported by the - connector) - * Not supported by y-xmpp -* y.close() - * Destroy this object. - * Destroys all types (they will throw weird errors if you still use them) - * Disconnects from the other instances (via connector) - * Returns a promise -* y.destroy() - * calls y.close() - * Removes all data from the database - * Returns a promise -* y.db.stopGarbageCollector() - * Stop the garbage collector. Call y.db.garbageCollect() to continue garbage - collection -* y.db.gc :: Boolean - * Whether gc is turned on -* y.db.gcTimeout :: Number (defaults to 50000 ms) - * Time interval between two garbage collect cycles - * It is required that all instances exchanged all messages after two garbage - collect cycles (after 100000 ms per default) -* y.db.userId :: String - * The used user id for this client. **Never overwrite this** - -### Logging -Yjs uses [debug](https://github.com/visionmedia/debug) for logging. The flag -`y*` enables logging for all y-* components. You can selectively remove -components you are not interested in: E.g. The flag `y*,-y:connector-message` -will not log the long `y:connector-message` messages. - -##### Enable logging in Node.js ```sh -DEBUG=y* node app.js +npm i yjs@13.0.0-97 y-websocket@1.0.0-6 ``` -Remove the colors in order to log to a file: +Start the y-websocket server: + ```sh -DEBUG_COLORS=0 DEBUG=y* node app.js > log +PORT=1234 node ./node_modules/y-websocket/bin/server.js ``` -##### Enable logging in the browser +### Example: Observe types + ```js -localStorage.debug = 'y*' +const yarray = doc.getArray('my-array') +yarray.observe(event => { + console.log('yarray was modified') +}) +// every time a local or remote client modifies yarray, the observer is called +yarray.insert(0, ['val']) // => "yarray was modified" ``` -## License -Yjs is licensed under the [MIT License](./LICENSE). +### Example: Nest types + +Remember, shared types are just plain old data types. The only limitation is +that a shared type must exist only once in the shared document. + +```js +const ymap = doc.getMap('map') +const foodArray = new Y.Array() +foodArray.insert(0, ['apple', 'banana']) +ymap.set('food', foodArray) +ymap.get('food') === foodArray // => true +ymap.set('fruit', foodArray) // => Error! foodArray is already defined +``` + +Now you understand how types are defined on a shared document. Next you can jump +to the [demo repository](https://github.com/yjs/yjs-demos) or continue reading +the API docs. + +## API + +```js +import * as Y from 'yjs' +``` + +### Shared Types + +
+ Y.Array +
+

+A shareable Array-like type that supports efficient insert/delete of elements +at any position. Internally it uses a linked list of Arrays that is split when +necessary. +

+ 
const yarray = new Y.Array()
+
+ insert(index:number, content:Array<object|boolean|Array|string|number|Uint8Array|Y.Type>) +
+Insert content at index. Note that content is an array of elements. +I.e. array.insert(0, [1] splices the list and inserts 1 at +position 0. +
+ push(Array<Object|boolean|Array|string|number|Uint8Array|Y.Type>) +
+ delete(index:number, length:number) +
+ get(index:number) +
+ length:number +
+ + +forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, + index:number, array: Y.Array)) + + +
+ map(function(T, number, YArray):M):Array<M> +
+ toArray():Array<object|boolean|Array|string|number|Uint8Array|Y.Type> +
Copies the content of this YArray to a new Array.
+ toJSON():Array<Object|boolean|Array|string|number> +
+Copies the content of this YArray to a new Array. It transforms all child types +to JSON using their toJSON method. +
+ [Symbol.Iterator] +
+ Returns an YArray Iterator that contains the values for each index in the array. + 
for (let value of yarray) { .. }
+
+ observe(function(YArrayEvent, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. +
+ unobserve(function(YArrayEvent, Transaction):void) +
+ Removes an observe event listener from this type. +
+ observeDeep(function(Array<YEvent>, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. +
+ unobserveDeep(function(Array<YEvent>, Transaction):void) +
+ Removes an observeDeep event listener from this type. +
+
+
+
+ Y.Map +
+

+ A shareable Map type. +

+ 
const ymap = new Y.Map()
+
+ get(key:string):object|boolean|string|number|Uint8Array|Y.Type +
+ set(key:string, value:object|boolean|string|number|Uint8Array|Y.Type) +
+ delete(key:string) +
+ has(key:string):boolean +
+ get(index:number) +
+ toJSON():Object<string, Object|boolean|Array|string|number|Uint8Array> +
+Copies the [key,value] pairs of this YMap to a new Object.It +transforms all child types to JSON using their toJSON method. +
+ forEach(function(key:string,value:object|boolean|Array|string|number|Uint8Array|Y.Type)) +
+ Execute the provided function once for every key-value pair. +
+ [Symbol.Iterator] +
+ Returns an Iterator of [key, value] pairs. + 
for (let [key, value] of ymap) { .. }
+
+ entries() +
+ Returns an Iterator of [key, value] pairs. +
+ values() +
+ Returns an Iterator of all values. +
+ keys() +
+ Returns an Iterator of all keys. +
+ observe(function(YMapEvent, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. +
+ unobserve(function(YMapEvent, Transaction):void) +
+ Removes an observe event listener from this type. +
+ observeDeep(function(Array<YEvent>, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. +
+ unobserveDeep(function(Array<YEvent>, Transaction):void) +
+ Removes an observeDeep event listener from this type. +
+
+
+ +
+ Y.Text +
+

+A shareable type that is optimized for shared editing on text. It allows to +assign properties to ranges in the text. This makes it possible to implement +rich-text bindings to this type. +

+

+This type can also be transformed to the +delta format. Similarly the +YTextEvents compute changes as deltas. +

+ 
const ytext = new Y.Text()
+
+ insert(index:number, content:string, [formattingAttributes:Object<string,string>]) +
+ Insert a string at index and assign formatting attributes to it. + 
ytext.insert(0, 'bold text', { bold: true })
+
+ delete(index:number, length:number) +
+ format(index:number, length:number, formattingAttributes:Object<string,string>) +
Assign formatting attributes to a range in the text
+ applyDelta(delta) +
See Quill Delta
+ length:number +
+ toString():string +
Transforms this type, without formatting options, into a string.
+ toJSON():string +
See toString
+ toDelta():Delta +
+Transforms this type to a Quill Delta +
+ observe(function(YTextEvent, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. +
+ unobserve(function(YTextEvent, Transaction):void) +
+ Removes an observe event listener from this type. +
+ observeDeep(function(Array<YEvent>, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. +
+ unobserveDeep(function(Array<YEvent>, Transaction):void) +
+ Removes an observeDeep event listener from this type. +
+
+
+ +
+ YXmlFragment +
+

+ A container that holds an Array of Y.XmlElements. +

+ 
const yxml = new Y.XmlFragment()
+
+ insert(index:number, content:Array<Y.XmlElement|Y.XmlText>) +
+ delete(index:number, length:number) +
+ get(index:number) +
+ length:number +
+ toArray():Array<Y.XmlElement|Y.XmlText> +
Copies the children to a new Array.
+ toDOM():DocumentFragment +
Transforms this type and all children to new DOM elements.
+ toString():string +
Get the XML serialization of all descendants.
+ toJSON():string +
See toString.
+ observe(function(YXmlEvent, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. +
+ unobserve(function(YXmlEvent, Transaction):void) +
+ Removes an observe event listener from this type. +
+ observeDeep(function(Array<YEvent>, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. +
+ unobserveDeep(function(Array<YEvent>, Transaction):void) +
+ Removes an observeDeep event listener from this type. +
+
+
+ +
+ Y.XmlElement +
+

+A shareable type that represents an XML Element. It has a nodeName, +attributes, and a list of children. But it makes no effort to validate its +content and be actually XML compliant. +

+ 
const yxml = new Y.XmlElement()
+
+ insert(index:number, content:Array<Y.XmlElement|Y.XmlText>) +
+ delete(index:number, length:number) +
+ get(index:number) +
+ length:number +
+ setAttribute(attributeName:string, attributeValue:string) +
+ removeAttribute(attributeName:string) +
+ getAttribute(attributeName:string):string +
+ getAttributes(attributeName:string):Object<string,string> +
+ toArray():Array<Y.XmlElement|Y.XmlText> +
Copies the children to a new Array.
+ toDOM():Element +
Transforms this type and all children to a new DOM element.
+ toString():string +
Get the XML serialization of all descendants.
+ toJSON():string +
See toString.
+ observe(function(YXmlEvent, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every +time this type is modified. In the case this type is modified in the event +listener, the event listener will be called again after the current event +listener returns. +
+ unobserve(function(YXmlEvent, Transaction):void) +
+ Removes an observe event listener from this type. +
+ observeDeep(function(Array<YEvent>, Transaction):void) +
+Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. +
+ unobserveDeep(function(Array<YEvent>, Transaction):void) +
+ Removes an observeDeep event listener from this type. +
+
+
+ +### Y.Doc + +```js +const doc = new Y.Doc() +``` + +
+ clientID +
A unique id that identifies this client. (readonly)
+ transact(function(Transaction):void [, origin:any]) +
+Every change on the shared document happens in a transaction. Observer calls and +the update event are called after each transaction. You should +bundle changes into a single transaction to reduce the amount of event +calls. I.e. doc.transact(() => { yarray.insert(..); ymap.set(..) }) +triggers a single change event.
You can specify an optional origin +parameter that is stored on transaction.origin and +on('update', (update, origin) => ..). +
+ get(string, Y.[TypeClass]):[Type] +
Define a shared type.
+ getArray(string):Y.Array +
Define a shared Y.Array type. Is equivalent to y.get(string, Y.Array).
+ getMap(string):Y.Map +
Define a shared Y.Map type. Is equivalent to y.get(string, Y.Map).
+ getXmlFragment(string):Y.XmlFragment +
Define a shared Y.XmlFragment type. Is equivalent to y.get(string, Y.XmlFragment).
+ on(string, function) +
Register an event listener on the shared type
+ off(string, function) +
Unregister an event listener from the shared type
+
+ +#### Y.Doc Events + +
+ on('update', function(updateMessage:Uint8Array, origin:any, Y.Doc):void) +
+Listen to document updates. Document updates must be transmitted to all other +peers. You can apply document updates in any order and multiple times. +
+ on('beforeTransaction', function(Y.Transaction, Y.Doc):void) +
Emitted before each transaction.
+ on('afterTransaction', function(Y.Transaction, Y.Doc):void) +
Emitted after each transaction.
+
+ +### Document Updates + +Changes on the shared document are encoded into *document updates*. Document +updates are *commutative* and *idempotent*. This means that they can be applied +in any order and multiple times. + +#### Example: Listen to update events and apply them on remote client + +```js +const doc1 = new Y.Doc() +const doc2 = new Y.Doc() + +doc1.on('update', update => { + Y.applyUpdate(doc2, update) +}) + +doc2.on('update', update => { + Y.applyUpdate(doc1, update) +}) + +// All changes are also applied to the other document +doc1.getArray('myarray').insert(0, ['Hello doc2, you got this?']) +doc2.getArray('myarray').get(0) // => 'Hello doc2, you got this?' +``` + +Yjs internally maintains a [state vector](#State-Vector) that denotes the next +expected clock from each client. In a different interpretation it holds the +number of structs created by each client. When two clients sync, you can either +exchange the complete document structure or only the differences by sending the +state vector to compute the differences. + +#### Example: Sync two clients by exchanging the complete document structure + +```js +const state1 = Y.encodeStateAsUpdate(ydoc1) +const state2 = Y.encodeStateAsUpdate(ydoc2) +Y.applyUpdate(ydoc1, state2) +Y.applyUpdate(ydoc2, state1) +``` + +#### Example: Sync two clients by computing the differences + +This example shows how to sync two clients with the minimal amount of exchanged +data by computing only the differences using the state vector of the remote +client. Syncing clients using the state vector requires another roundtrip, but +can safe a lot of bandwidth. + +```js +const stateVector1 = Y.encodeStateVector(ydoc1) +const stateVector2 = Y.encodeStateVector(ydoc2) +const diff1 = Y.encodeStateAsUpdate(ydoc1, stateVector2) +const diff2 = Y.encodeStateAsUpdate(ydoc2, stateVector1) +Y.applyUpdate(ydoc1, diff2) +Y.applyUpdate(ydoc2, diff1) +``` + +
+ Y.applyUpdate(Y.Doc, update:Uint8Array, [transactionOrigin:any]) +
+Apply a document update on the shared document. Optionally you can specify +transactionOrigin that will be stored on +transaction.origin +and ydoc.on('update', (update, origin) => ..). +
+ Y.encodeStateAsUpdate(Y.Doc, [encodedTargetStateVector:Uint8Array]):Uint8Array +
+Encode the document state as a single update message that can be applied on the +remote document. Optionally specify the target state vector to only write the +differences to the update message. +
+ Y.encodeStateVector(Y.Doc):Uint8Array +
Computes the state vector and encodes it into an Uint8Array.
+
+ +### Relative Positions + +> This API is not stable yet + +This feature is intended for managing selections / cursors. When working with +other users that manipulate the shared document, you can't trust that an index +position (an integer) will stay at the intended location. A *relative position* +is fixated to an element in the shared document and is not affected by remote +changes. I.e. given the document `"a|c"`, the relative position is attached to +`c`. When a remote user modifies the document by inserting a character before +the cursor, the cursor will stay attached to the character `c`. `insert(1, +'x')("a|c") = "ax|c"`. When the *relative position* is set to the end of the +document, it will stay attached to the end of the document. + +#### Example: Transform to RelativePosition and back + +```js +const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) +const pos = Y.createAbsolutePositionFromRelativePosition(relPos, doc) +pos.type === ytext // => true +pos.index === 2 // => true +``` + +#### Example: Send relative position to remote client (json) + +```js +const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) +const encodedRelPos = JSON.stringify(relPos) +// send encodedRelPos to remote client.. +const parsedRelPos = JSON.parse(encodedRelPos) +const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) +pos.type === remoteytext // => true +pos.index === 2 // => true +``` + +#### Example: Send relative position to remote client (Uint8Array) + +```js +const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) +const encodedRelPos = Y.encodeRelativePosition(relPos) +// send encodedRelPos to remote client.. +const parsedRelPos = Y.decodeRelativePosition(encodedRelPos) +const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) +pos.type === remoteytext // => true +pos.index === 2 // => true +``` + +
+ Y.createRelativePositionFromTypeIndex(Uint8Array|Y.Type, number) +
+ Y.createAbsolutePositionFromRelativePosition(RelativePosition, Y.Doc) +
+ Y.encodeRelativePosition(RelativePosition):Uint8Array +
+ Y.decodeRelativePosition(Uint8Array):RelativePosition +
+
+ +### Y.UndoManager + +Yjs ships with an Undo/Redo manager for selective undo/redo of of changes on a +Yjs type. The changes can be optionally scoped to transaction origins. + +```js +const ytext = doc.getArray('array') +const undoManager = new Y.UndoManager(ytext) + +ytext.insert(0, 'abc') +undoManager.undo() +ytext.toString() // => '' +undoManager.redo() +ytext.toString() // => 'abc' +``` + +
+ constructor(scope:Y.AbstractType|Array<Y.AbstractType>, + [[{captureTimeout:number,trackedOrigins:Set<any>,deleteFilter:function(item):boolean}]]) +
Accepts either single type as scope or an array of types.
+ undo() +
+ redo() +
+ stopCapturing() +
+ + +on('stack-item-added', { stackItem: { meta: Map<any,any> }, type: 'undo' +| 'redo' }) + + +
+Register an event that is called when a StackItem is added to the +undo- or the redo-stack. +
+ + +on('stack-item-popped', { stackItem: { meta: Map<any,any> }, type: 'undo' +| 'redo' }) + + +
+Register an event that is called when a StackItem is popped from +the undo- or the redo-stack. +
+
+ +#### Example: Stop Capturing + +UndoManager merges Undo-StackItems if they are created within time-gap +smaller than `options.captureTimeout`. Call `um.stopCapturing()` so that the next +StackItem won't be merged. + +```js +// without stopCapturing +ytext.insert(0, 'a') +ytext.insert(1, 'b') +um.undo() +ytext.toString() // => '' (note that 'ab' was removed) +// with stopCapturing +ytext.insert(0, 'a') +um.stopCapturing() +ytext.insert(0, 'b') +um.undo() +ytext.toString() // => 'a' (note that only 'b' was removed) +``` + +#### Example: Specify tracked origins + +Every change on the shared document has an origin. If no origin was specified, +it defaults to `null`. By specifying `trackedTransactionOrigins` you can +selectively specify which changes should be tracked by `UndoManager`. The +UndoManager instance is always added to `trackedTransactionOrigins`. + +```js +class CustomBinding {} + +const ytext = doc.getArray('array') +const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) + +ytext.insert(0, 'abc') +undoManager.undo() +ytext.toString() // => 'abc' (does not track because origin `null` and not part + // of `trackedTransactionOrigins`) +ytext.delete(0, 3) // revert change + +doc.transact(() => { + ytext.insert(0, 'abc') +}, 42) +undoManager.undo() +ytext.toString() // => '' (tracked because origin is an instance of `trackedTransactionorigins`) + +doc.transact(() => { + ytext.insert(0, 'abc') +}, 41) +undoManager.undo() +ytext.toString() // => '' (not tracked because 41 is not an instance of + // `trackedTransactionorigins`) +ytext.delete(0, 3) // revert change + +doc.transact(() => { + ytext.insert(0, 'abc') +}, new CustomBinding()) +undoManager.undo() +ytext.toString() // => '' (tracked because origin is a `CustomBinding` and + // `CustomBinding` is in `trackedTransactionorigins`) +``` + +#### Example: Add additional information to the StackItems + +When undoing or redoing a previous action, it is often expected to restore +additional meta information like the cursor location or the view on the +document. You can assign meta-information to Undo-/Redo-StackItems. + +```js +const ytext = doc.getArray('array') +const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) + +undoManager.on('stack-item-added', event => { + // save the current cursor location on the stack-item + event.stackItem.meta.set('cursor-location', getRelativeCursorLocation()) +}) + +undoManager.on('stack-item-popped', event => { + // restore the current cursor location on the stack-item + restoreCursorLocation(event.stackItem.meta.get('cursor-location')) +}) +``` + +## Miscellaneous + +### Typescript Declarations + +Yjs has type descriptions. But until [this +ticket](https://github.com/Microsoft/TypeScript/issues/7546) is fixed, this is +how you can make use of Yjs type declarations. + +```json +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + }, + "maxNodeModuleJsDepth": 5 +} +``` + +## Yjs CRDT Algorithm + +*Conflict-free replicated data types* (CRDT) for collaborative editing are an +alternative approach to *operational transformation* (OT). A very simple +differenciation between the two approaches is that OT attempts to transform +index positions to ensure convergence (all clients end up with the same +content), while CRDTs use mathematical models that usually do not involve index +transformations, like linked lists. OT is currently the de-facto standard for +shared editing on text. OT approaches that support shared editing without a +central source of truth (a central server) require too much bookkeeping to be +viable in practice. CRDTs are better suited for distributed systems, provide +additional guarantees that the document can be synced with remote clients, and +do not require a central source of truth. + +Yjs implements a modified version of the algorithm described in [this +paper](https://www.researchgate.net/publication/310212186_Near_Real-Time_Peer-to-Peer_Shared_Editing_on_Extensible_Data_Types). +I will eventually publish a paper that describes why this approach works so well +in practice. Note: Since operations make up the document structure, we prefer +the term *struct* now. + +CRDTs suitable for shared text editing suffer from the fact that they only grow +in size. There are CRDTs that do not grow in size, but they do not have the +characteristics that are benificial for shared text editing (like intention +preservation). Yjs implements many improvements to the original algorithm that +diminish the trade-off that the document only grows in size. We can't garbage +collect deleted structs (tombstones) while ensuring a unique order of the +structs. But we can 1. merge preceeding structs into a single struct to reduce +the amount of meta information, 2. we can delete content from the struct if it +is deleted, and 3. we can garbage collect tombstones if we don't care about the +order of the structs anymore (e.g. if the parent was deleted). + +**Examples:** + +1. If a user inserts elements in sequence, the struct will be merged into a + single struct. E.g. `array.insert(0, ['a']), array.insert(0, ['b']);` is + first represented as two structs (`[{id: {client, clock: 0}, content: 'a'}, + {id: {client, clock: 1}, content: 'b'}`) and then merged into a single + struct: `[{id: {client, clock: 0}, content: 'ab'}]`. +2. When a struct that contains content (e.g. `ItemString`) is deleted, the + struct will be replaced with an `ItemDeleted` that does not contain content + anymore. +3. When a type is deleted, all child elements are transformed to `GC` structs. A + `GC` struct only denotes the existence of a struct and that it is deleted. + `GC` structs can always be merged with other `GC` structs if the id's are + adjacent. + +Especially when working on structured content (e.g. shared editing on +ProseMirror), these improvements yield very good results when +[benchmarking](https://github.com/dmonad/crdt-benchmarks) random document edits. +In practice they show even better results, because users usually edit text in +sequence, resulting in structs that can easily be merged. The benchmarks show +that even in the worst case scenario that a user edits text from right to left, +Yjs achieves good performance even for huge documents. + +### State Vector + +Yjs has the ability to exchange only the differences when syncing two clients. +We use lamport timestamps to identify structs and to track in which order a +client created them. Each struct has an `struct.id = { client: number, clock: +number}` that uniquely identifies a struct. We define the next expected `clock` +by each client as the *state vector*. This data structure is similar to the +[version vectors](https://en.wikipedia.org/wiki/Version_vector) data structure. +But we use state vectors only to describe the state of the local document, so we +can compute the missing struct of the remote client. We do not use it to track +causality. + +## License and Author + +Yjs and all related projects are [**MIT licensed**](./LICENSE). + +Yjs is based on my research as a student at the [RWTH +i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time. + +Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or +hiring [me](https://github.com/dmonad) for professional support. diff --git a/README.v12.md b/README.v12.md new file mode 100644 index 00000000..f750f721 --- /dev/null +++ b/README.v12.md @@ -0,0 +1,305 @@ + +# ![Yjs](https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png) + +Yjs is a framework for offline-first p2p shared editing on structured data like +text, richtext, json, or XML. It is fairly easy to get started, as Yjs hides +most of the complexity of concurrent editing. For additional information, demos, +and tutorials visit [y-js.org](http://y-js.org/). + +:warning: Checkout the [v13 docs](./README.md) for the upcoming release :warning: + +### Extensions +Yjs only knows how to resolve conflicts on shared data. You have to choose a .. + +* *Connector* - a communication protocol that propagates changes to the clients +* *Database* - a database to store your changes +* one or more *Types* - that represent the shared data + +Connectors, Databases, and Types are available as modules that extend Yjs. Here +is a list of the modules we know of: + +##### Connectors + +|Name | Description | +|----------------|-----------------------------------| +|[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC| +|[websockets](https://github.com/y-js/y-websockets-client) | Set up [a central server](https://github.com/y-js/y-websockets-client), and connect to it via websockets | +|[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))| +|[ipfs](https://github.com/ipfs-labs/y-ipfs-connector) | Connector for the [Interplanetary File System](https://ipfs.io/)!| +|[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios| + +##### Database adapters + +|Name | Description | +|----------------|-----------------------------------| +|[memory](https://github.com/y-js/y-memory) | In-memory storage. | +|[indexeddb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | +|[leveldb](https://github.com/y-js/y-leveldb) | Persistent storage for node apps | + +##### Types + +| Name | Description | +|----------|-------------------| +|[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | +|[array](https://github.com/y-js/y-array) | A shared Array implementation | +|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | +|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to the [Ace Editor](https://ace.c9.io), [CodeMirror](https://codemirror.net/), [Monaco](https://github.com/Microsoft/monaco-editor), textareas, input elements, and HTML elements (e.g. <*h1*>, or <*p*>) | +|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| + +##### Other + +| Name | Description | +|-----------|-------------------| +|[y-element](http://y-js.org/y-element/) | Yjs Polymer Element | + +## Use it! +Install Yjs, and its modules with [bower](http://bower.io/), or +[npm](https://www.npmjs.org/package/yjs). + +### Bower + +``` +bower install --save yjs y-array % add all y-* modules you want to use +``` +You only need to include the `y.js` file. Yjs is able to automatically require +missing modules. +``` + +``` + +### CDN + +``` + + + + + + +// .. +// do the same for all modules you want to use +``` + +### Npm + +``` +npm install --save yjs % add all y-* modules you want to use +``` + +If you don't include via script tag, you have to explicitly include all modules! +(Same goes for other module systems) +``` +var Y = require('yjs') +require('y-array')(Y) // add the y-array type to Yjs +require('y-websockets-client')(Y) +require('y-memory')(Y) +require('y-map')(Y) +require('y-text')(Y) +// .. +// do the same for all modules you want to use +``` + +### ES6 Syntax + +``` +import Y from 'yjs' +import yArray from 'y-array' +import yWebsocketsClient from 'y-webrtc' +import yMemory from 'y-memory' +import yMap from 'y-map' +import yText from 'y-text' +// .. +Y.extend(yArray, yWebsocketsClient, yMemory, yArray, yMap, yText /*, .. */) +``` + +# Text editing example + +Install dependencies +``` +bower i yjs y-memory y-webrtc y-array y-text +``` + +Here is a simple example of a shared textarea +```HTML + + + + + + + + + +``` + +## Get Help & Give Help +There are some friendly people on [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/y-js/yjs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) who are eager to help, and answer questions. Please join! + +Report _any_ issues to the +[Github issue page](https://github.com/y-js/yjs/issues)! I try to fix them very +soon, if possible. + +# API + +### Y(options) +* Y.extend(module1, module2, ..) + * Add extensions to Y + * `Y.extend(require('y-webrtc'))` has the same semantics as + `require('y-webrtc')(Y)` +* options.db + * Will be forwarded to the database adapter. Specify the database adaper on + `options.db.name`. + * Have a look at the used database adapter repository to see all available + options. +* options.connector + * Will be forwarded to the connector adapter. Specify the connector adaper on + `options.connector.name`. + * All our connectors implement a `room` property. Clients that specify the + same room share the same data. + * All of our connectors specify an `url` property that defines the connection + endpoint of the used connector. + * All of our connectors also have a default connection endpoint that you can + use for development. + * Set `options.connector.generateUserId = true` in order to genenerate a + userid, instead of receiving one from the server. This way the `Y(..)` is + immediately going to be resolved, without waiting for any confirmation from + the server. Use with caution. + * Have a look at the used connector repository to see all available options. + * *Only if you know what you are doing:* Set + `options.connector.preferUntransformed = true` in order receive the shared + data untransformed. This is very efficient as the database content is simply + copied to this client. This does only work if this client receives content + from only one client. +* options.sourceDir (browser only) + * Path where all y-* modules are stored + * Defaults to `/bower_components` + * Not required when running on `nodejs` / `iojs` + * When using nodejs you need to manually extend Yjs: +``` +var Y = require('yjs') +// you have to require a db, connector, and *all* types you use! +require('y-memory')(Y) +require('y-webrtc')(Y) +require('y-map')(Y) +// .. +``` +* options.share + * Specify on `options.share[arbitraryName]` types that are shared among all + users. + * E.g. Specify `options.share[arbitraryName] = 'Array'` to require y-array and + create an y-array type on `y.share[arbitraryName]`. + * If userA doesn't specify `options.share[arbitraryName]`, it won't be + available for userA. + * If userB specifies `options.share[arbitraryName]`, it still won't be + available for userA. But all the updates are send from userB to userA. + * In contrast to y-map, types on `y.share.*` cannot be overwritten or deleted. + Instead, they are merged among all users. This feature is only available on + `y.share.*` + * Weird behavior: It is supported that two users specify different types with + the same property name. + E.g. userA specifies `options.share.x = 'Array'`, and userB specifies + `options.share.x = 'Text'`. But they only share data if they specified the + same type with the same property name +* options.type (browser only) + * Array of modules that Yjs needs to require, before instantiating a shared + type. + * By default Yjs requires the specified database adapter, the specified + connector, and all modules that are used in `options.share.*` + * Put all types here that you intend to use, but are not used in y.share.* + +### Instantiated Y object (y) +`Y(options)` returns a promise that is fulfilled when.. + +* All modules are loaded + * The specified database adapter is loaded + * The specified connector is loaded + * All types are included +* The connector is initialized, and a unique user id is set (received from the + server) + * Note: When using y-indexeddb, a retrieved user id is stored on `localStorage` + +The promise returns an instance of Y. We denote it with a lower case `y`. + +* y.share.* + * Instances of the types you specified on options.share.* + * y.share.* can only be defined once when you instantiate Y! +* y.connector is an instance of Y.AbstractConnector +* y.connector.onUserEvent(function (event) {..}) + * Observe user events (event.action is either 'userLeft' or 'userJoined') +* y.connector.whenSynced(listener) + * `listener` is executed when y synced with at least one user. + * `listener` is not called when no other user is in the same room. + * y-websockets-client aways waits to sync with the server +* y.connector.disconnect() + * Force to disconnect this instance from the other instances +* y.connector.connect() + * Try to reconnect to the other instances (needs to be supported by the + connector) + * Not supported by y-xmpp +* y.close() + * Destroy this object. + * Destroys all types (they will throw weird errors if you still use them) + * Disconnects from the other instances (via connector) + * Returns a promise +* y.destroy() + * calls y.close() + * Removes all data from the database + * Returns a promise +* y.db.stopGarbageCollector() + * Stop the garbage collector. Call y.db.garbageCollect() to continue garbage + collection +* y.db.gc :: Boolean + * Whether gc is turned on +* y.db.gcTimeout :: Number (defaults to 50000 ms) + * Time interval between two garbage collect cycles + * It is required that all instances exchanged all messages after two garbage + collect cycles (after 100000 ms per default) +* y.db.userId :: String + * The used user id for this client. **Never overwrite this** + +### Logging +Yjs uses [debug](https://github.com/visionmedia/debug) for logging. The flag +`y*` enables logging for all y-* components. You can selectively remove +components you are not interested in: E.g. The flag `y*,-y:connector-message` +will not log the long `y:connector-message` messages. + +##### Enable logging in Node.js +```sh +DEBUG=y* node app.js +``` + +Remove the colors in order to log to a file: +```sh +DEBUG_COLORS=0 DEBUG=y* node app.js > log +``` + +##### Enable logging in the browser +```js +localStorage.debug = 'y*' +``` + +## License +Yjs is licensed under the [MIT License](./LICENSE). diff --git a/README.v13.md b/README.v13.md deleted file mode 100644 index ad5becef..00000000 --- a/README.v13.md +++ /dev/null @@ -1,874 +0,0 @@ - -# ![Yjs](https://yjs.dev/images/logo/yjs-120x120.png) - -> A CRDT framework with a powerful abstraction of shared data - -Yjs is a [CRDT implementation](#Yjs-CRDT-Algorithm) that exposes its internal -data structure as *shared types*. Shared types are common data types like `Map` -or `Array` with superpowers: changes are automatically distributed to other -peers and merged without merge conflicts. - -Yjs is **network agnostic** (p2p!), supports many existing **rich text -editors**, **offline editing**, **version snapshots**, **undo/redo** and -**shared cursors**. It scales well with an unlimited number of users and is well -suited for even large documents. - -* Chat: [https://gitter.im/y-js/yjs](https://gitter.im/y-js/yjs) -* Demos: [https://github.com/y-js/yjs-demos](https://github.com/y-js/yjs-demos) -* Benchmarks: [https://github.com/dmonad/crdt-benchmarks](https://github.com/dmonad/crdt-benchmarks) - -## Table of Contents - -* [Overview](#Overview) - * [Bindings](#Bindings) - * [Providers](#Providers) -* [Getting Started](#Getting-Started) -* [API](#API) - * [Shared Types](#Shared-Types) - * [Y.Doc](#YDoc) - * [Document Updates](#Document-Updates) - * [Relative Positions](#Relative-Positions) - * [Y.UndoManager](#YUndoManager) -* [Miscellaneous](#Miscellaneous) - * [Typescript Declarations](#Typescript-Declarations) -* [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm) -* [Evaluation](#Evaluation) - * [Existing shared editing libraries](#Exisisting-Javascript-Libraries) - * [CRDT Algorithms](#CRDT-Algorithms) - * [Comparison of CRDT with OT](#Comparing-CRDT-with-OT) - * [Comparison of CRDT Algorithms](#Comparing-CRDT-Algorithms) - * [Comparison of Yjs with other Implementations](#Comparing-Yjs-with-other-Implementations) -* [License and Author](#License-and-Author) - -## Overview - -This repository contains a collection of shared types that can be observed for -changes and manipulated concurrently. Network functionality and two-way-bindings -are implemented in separate modules. - -### Bindings - -| Name | Cursors | Binding | Demo | -|---|:-:|---|---| -| [ProseMirror](https://prosemirror.net/)                                                   | ✔ | [y-prosemirror](http://github.com/y-js/y-prosemirror) | [demo](https://yjs-demos.now.sh/prosemirror/) | -| [Quill](https://quilljs.com/) | | [y-quill](http://github.com/y-js/y-quill) | [demo](https://yjs-demos.now.sh/quill/) | -| [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/y-js/y-codemirror) | [demo](https://yjs-demos.now.sh/codemirror/) | -| [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/y-js/y-monaco) | [demo](https://yjs-demos.now.sh/monaco/) | -| [Ace](https://ace.c9.io/) | | [y-ace](http://github.com/y-js/y-ace) | [demo](https://yjs-demos.now.sh/ace/) | -| [Textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) | | [y-textarea](http://github.com/y-js/y-textarea) | [demo](https://yjs-demos.now.sh/textarea/) | -| [DOM](https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model) | | [y-dom](http://github.com/y-js/y-dom) | [demo](https://yjs-demos.now.sh/dom/) | - -### Providers - -Setting up the communication between clients, managing awareness information, -and storing shared data for offline usage is quite a hassle. **Providers** -manage all that for you and are the perfect starting point for your -collaborative app. - -
-
y-websocket
-
-A module that contains a simple websocket backend and a websocket client that -connects to that backend. The backend can be extended to persist updates in a -leveldb database. -
-
y-mesh
-
-[WIP] Creates a connected graph of webrtc connections with a high -strength. It -requires a signalling server that connects a client to the first peer. But after -that the network manages itself. It is well suited for large and small networks. -
-
y-dat
-
-[WIP] Write document updates effinciently to the dat network using -multifeed. Each client has -an append-only log of CRDT local updates (hypercore). Multifeed manages and sync -hypercores and y-dat listens to changes and applies them to the Yjs document. -
-
- -## Getting Started - -Install Yjs and a provider with your favorite package manager: - -```sh -npm i yjs@13.0.0-82 y-websocket@1.0.0-3 y-textarea -``` - -Start the y-websocket server: - -```sh -PORT=1234 node ./node_modules/y-websocket/bin/server.js -``` - -### Example: Textarea Binding - -This is a complete example on how to create a connection to a -[y-websocket](https://github.com/y-js/y-websocket) server instance, sync the -shared document to all clients in a *room*, and bind a Y.Text type to a dom -textarea. All changes to the textarea are automatically shared with everyone in -the same room. - -```js -import * as Y from 'yjs' -import { WebsocketProvider } from 'y-websocket' -import { TextareaBinding } from 'y-textarea' - -const doc = Y.Doc() -const provider = new WebsocketProvider('ws://localhost:1234', 'roomname', doc) - -// Define a shared type on the document. -const ytext = doc.getText('my resume') - -// use data bindings to bind types to editors -const binding = new TextareaBinding(ytext, document.querySelector('textarea')) -``` - -#### Example: Observe types - -```js -const yarray = doc.getArray('my-array') -yarray.observe(event => { - console.log('yarray was modified') -}) -// every time a local or remote client modifies yarray, the observer is called -yarray.insert(0, ['val']) // => "yarray was modified" -``` - -#### Example: Nest types - -Remember, shared types are just plain old data types. The only limitation is -that a shared type must exist only once in the shared document. - -```js -const ymap = doc.getMap('map') -const foodArray = new Y.Array() -foodArray.insert(0, ['apple', 'banana']) -ymap.set('food', foodArray) -ymap.get('food') === foodArray // => true -ymap.set('fruit', foodArray) // => Error! foodArray is already defined -``` - -Now you understand how types are defined on a shared document. Next you can jump -to the [demo repository](https://github.com/y-js/yjs-demos) or continue reading -the API docs. - -## API - -```js -import * as Y from 'yjs' -``` - -### Shared Types - -
- Y.Array -
-

-A shareable Array-like type that supports efficient insert/delete of elements -at any position. Internally it uses a linked list of Arrays that is split when -necessary. -

- 
const yarray = new Y.Array()
-
- insert(index:number, content:Array<object|boolean|Array|string|number|Uint8Array|Y.Type>) -
-Insert content at index. Note that content is an array of elements. -I.e. array.insert(0, [1] splices the list and inserts 1 at -position 0. -
- push(Array<Object|boolean|Array|string|number|Uint8Array|Y.Type>) -
- delete(index:number, length:number) -
- get(index:number) -
- length:number -
- - -forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, - index:number, array: Y.Array)) - - -
- map(function(T, number, YArray):M):Array<M> -
- toArray():Array<object|boolean|Array|string|number|Uint8Array|Y.Type> -
Copies the content of this YArray to a new Array.
- toJSON():Array<Object|boolean|Array|string|number> -
-Copies the content of this YArray to a new Array. It transforms all child types -to JSON using their toJSON method. -
- [Symbol.Iterator] -
- Returns an YArray Iterator that contains the values for each index in the array. - 
for (let value of yarray) { .. }
-
- observe(function(YArrayEvent, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. -
- unobserve(function(YArrayEvent, Transaction):void) -
- Removes an observe event listener from this type. -
- observeDeep(function(Array<YEvent>, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. -
- unobserveDeep(function(Array<YEvent>, Transaction):void) -
- Removes an observeDeep event listener from this type. -
-
-
-
- Y.Map -
-

- A shareable Map type. -

- 
const ymap = new Y.Map()
-
- get(key:string):object|boolean|string|number|Uint8Array|Y.Type -
- set(key:string, value:object|boolean|string|number|Uint8Array|Y.Type) -
- delete(key:string) -
- has(key:string):boolean -
- get(index:number) -
- toJSON():Object<string, Object|boolean|Array|string|number|Uint8Array> -
-Copies the [key,value] pairs of this YMap to a new Object.It -transforms all child types to JSON using their toJSON method. -
- forEach(function(key:string,value:object|boolean|Array|string|number|Uint8Array|Y.Type)) -
- Execute the provided function once for every key-value pair. -
- [Symbol.Iterator] -
- Returns an Iterator of [key, value] pairs. - 
for (let [key, value] of ymap) { .. }
-
- entries() -
- Returns an Iterator of [key, value] pairs. -
- values() -
- Returns an Iterator of all values. -
- keys() -
- Returns an Iterator of all keys. -
- observe(function(YMapEvent, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. -
- unobserve(function(YMapEvent, Transaction):void) -
- Removes an observe event listener from this type. -
- observeDeep(function(Array<YEvent>, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. -
- unobserveDeep(function(Array<YEvent>, Transaction):void) -
- Removes an observeDeep event listener from this type. -
-
-
- -
- Y.Text -
-

-A shareable type that is optimized for shared editing on text. It allows to -assign properties to ranges in the text. This makes it possible to implement -rich-text bindings to this type. -

-

-This type can also be transformed to the -delta format. Similarly the -YTextEvents compute changes as deltas. -

- 
const ytext = new Y.Text()
-
- insert(index:number, content:string, [formattingAttributes:Object<string,string>]) -
- Insert a string at index and assign formatting attributes to it. - 
ytext.insert(0, 'bold text', { bold: true })
-
- delete(index:number, length:number) -
- format(index:number, length:number, formattingAttributes:Object<string,string>) -
Assign formatting attributes to a range in the text
- applyDelta(delta) -
See Quill Delta
- length:number -
- toString():string -
Transforms this type, without formatting options, into a string.
- toJSON():string -
See toString
- toDelta():Delta -
-Transforms this type to a Quill Delta -
- observe(function(YTextEvent, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. -
- unobserve(function(YTextEvent, Transaction):void) -
- Removes an observe event listener from this type. -
- observeDeep(function(Array<YEvent>, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. -
- unobserveDeep(function(Array<YEvent>, Transaction):void) -
- Removes an observeDeep event listener from this type. -
-
-
- -
- YXmlFragment -
-

- A container that holds an Array of Y.XmlElements. -

- 
const yxml = new Y.XmlFragment()
-
- insert(index:number, content:Array<Y.XmlElement|Y.XmlText>) -
- delete(index:number, length:number) -
- get(index:number) -
- length:number -
- toArray():Array<Y.XmlElement|Y.XmlText> -
Copies the children to a new Array.
- toDOM():DocumentFragment -
Transforms this type and all children to new DOM elements.
- toString():string -
Get the XML serialization of all descendants.
- toJSON():string -
See toString.
- observe(function(YXmlEvent, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. -
- unobserve(function(YXmlEvent, Transaction):void) -
- Removes an observe event listener from this type. -
- observeDeep(function(Array<YEvent>, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. -
- unobserveDeep(function(Array<YEvent>, Transaction):void) -
- Removes an observeDeep event listener from this type. -
-
-
- -
- Y.XmlElement -
-

-A shareable type that represents an XML Element. It has a nodeName, -attributes, and a list of children. But it makes no effort to validate its -content and be actually XML compliant. -

- 
const yxml = new Y.XmlElement()
-
- insert(index:number, content:Array<Y.XmlElement|Y.XmlText>) -
- delete(index:number, length:number) -
- get(index:number) -
- length:number -
- setAttribute(attributeName:string, attributeValue:string) -
- removeAttribute(attributeName:string) -
- getAttribute(attributeName:string):string -
- getAttributes(attributeName:string):Object<string,string> -
- toArray():Array<Y.XmlElement|Y.XmlText> -
Copies the children to a new Array.
- toDOM():Element -
Transforms this type and all children to a new DOM element.
- toString():string -
Get the XML serialization of all descendants.
- toJSON():string -
See toString.
- observe(function(YXmlEvent, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every -time this type is modified. In the case this type is modified in the event -listener, the event listener will be called again after the current event -listener returns. -
- unobserve(function(YXmlEvent, Transaction):void) -
- Removes an observe event listener from this type. -
- observeDeep(function(Array<YEvent>, Transaction):void) -
-Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. -
- unobserveDeep(function(Array<YEvent>, Transaction):void) -
- Removes an observeDeep event listener from this type. -
-
-
- -### Y.Doc - -```js -const doc = new Y.Doc() -``` - -
- clientID -
A unique id that identifies this client. (readonly)
- transact(function(Transaction):void [, origin:any]) -
-Every change on the shared document happens in a transaction. Observer calls and -the update event are called after each transaction. You should -bundle changes into a single transaction to reduce the amount of event -calls. I.e. doc.transact(() => { yarray.insert(..); ymap.set(..) }) -triggers a single change event.
You can specify an optional origin -parameter that is stored on transaction.origin and -on('update', (update, origin) => ..). -
- get(string, Y.[TypeClass]):[Type] -
Define a shared type.
- getArray(string):Y.Array -
Define a shared Y.Array type. Is equivalent to y.get(string, Y.Array).
- getMap(string):Y.Map -
Define a shared Y.Map type. Is equivalent to y.get(string, Y.Map).
- getXmlFragment(string):Y.XmlFragment -
Define a shared Y.XmlFragment type. Is equivalent to y.get(string, Y.XmlFragment).
- on(string, function) -
Register an event listener on the shared type
- off(string, function) -
Unregister an event listener from the shared type
-
- -#### Y.Doc Events - -
- on('update', function(updateMessage:Uint8Array, origin:any, Y.Doc):void) -
-Listen to document updates. Document updates must be transmitted to all other -peers. You can apply document updates in any order and multiple times. -
- on('beforeTransaction', function(Y.Transaction, Y.Doc):void) -
Emitted before each transaction.
- on('afterTransaction', function(Y.Transaction, Y.Doc):void) -
Emitted after each transaction.
-
- -### Document Updates - -Changes on the shared document are encoded into *document updates*. Document -updates are *commutative* and *idempotent*. This means that they can be applied -in any order and multiple times. - -#### Example: Listen to update events and apply them on remote client - -```js -const doc1 = new Y.Doc() -const doc2 = new Y.Doc() - -doc1.on('update', update => { - Y.applyUpdate(doc2, update) -}) - -doc2.on('update', update => { - Y.applyUpdate(doc1, update) -}) - -// All changes are also applied to the other document -doc1.getArray('myarray').insert(0, ['Hello doc2, you got this?']) -doc2.getArray('myarray').get(0) // => 'Hello doc2, you got this?' -``` - -Yjs internally maintains a [state vector](#State-Vector) that denotes the next -expected clock from each client. In a different interpretation it holds the -number of structs created by each client. When two clients sync, you can either -exchange the complete document structure or only the differences by sending the -state vector to compute the differences. - -#### Example: Sync two clients by exchanging the complete document structure - -```js -const state1 = Y.encodeStateAsUpdate(ydoc1) -const state2 = Y.encodeStateAsUpdate(ydoc2) -Y.applyUpdate(ydoc1, state2) -Y.applyUpdate(ydoc2, state1) -``` - -#### Example: Sync two clients by computing the differences - -This example shows how to sync two clients with the minimal amount of exchanged -data by computing only the differences using the state vector of the remote -client. Syncing clients using the state vector requires another roundtrip, but -can safe a lot of bandwidth. - -```js -const stateVector1 = Y.encodeStateVector(ydoc1) -const stateVector2 = Y.encodeStateVector(ydoc2) -const diff1 = Y.encodeStateAsUpdate(ydoc1, stateVector2) -const diff2 = Y.encodeStateAsUpdate(ydoc2, stateVector1) -Y.applyUpdate(ydoc1, diff2) -Y.applyUpdate(ydoc2, diff1) -``` - -
- Y.applyUpdate(Y.Doc, update:Uint8Array, [transactionOrigin:any]) -
-Apply a document update on the shared document. Optionally you can specify -transactionOrigin that will be stored on -transaction.origin -and ydoc.on('update', (update, origin) => ..). -
- Y.encodeStateAsUpdate(Y.Doc, [encodedTargetStateVector:Uint8Array]):Uint8Array -
-Encode the document state as a single update message that can be applied on the -remote document. Optionally specify the target state vector to only write the -differences to the update message. -
- Y.encodeStateVector(Y.Doc):Uint8Array -
Computes the state vector and encodes it into an Uint8Array.
-
- -### Relative Positions - -> This API is not stable yet - -This feature is intended for managing selections / cursors. When working with -other users that manipulate the shared document, you can't trust that an index -position (an integer) will stay at the intended location. A *relative position* -is fixated to an element in the shared document and is not affected by remote -changes. I.e. given the document `"a|c"`, the relative position is attached to -`c`. When a remote user modifies the document by inserting a character before -the cursor, the cursor will stay attached to the character `c`. `insert(1, -'x')("a|c") = "ax|c"`. When the *relative position* is set to the end of the -document, it will stay attached to the end of the document. - -#### Example: Transform to RelativePosition and back - -```js -const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) -const pos = Y.createAbsolutePositionFromRelativePosition(relPos, doc) -pos.type === ytext // => true -pos.index === 2 // => true -``` - -#### Example: Send relative position to remote client (json) - -```js -const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) -const encodedRelPos = JSON.stringify(relPos) -// send encodedRelPos to remote client.. -const parsedRelPos = JSON.parse(encodedRelPos) -const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) -pos.type === remoteytext // => true -pos.index === 2 // => true -``` - -#### Example: Send relative position to remote client (Uint8Array) - -```js -const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) -const encodedRelPos = Y.encodeRelativePosition(relPos) -// send encodedRelPos to remote client.. -const parsedRelPos = Y.decodeRelativePosition(encodedRelPos) -const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) -pos.type === remoteytext // => true -pos.index === 2 // => true -``` - -
- Y.createRelativePositionFromTypeIndex(Uint8Array|Y.Type, number) -
- Y.createAbsolutePositionFromRelativePosition(RelativePosition, Y.Doc) -
- Y.encodeRelativePosition(RelativePosition):Uint8Array -
- Y.decodeRelativePosition(Uint8Array):RelativePosition -
-
- -### Y.UndoManager - -Yjs ships with an Undo/Redo manager for selective undo/redo of of changes on a -Yjs type. The changes can be optionally scoped to transaction origins. - -```js -const ytext = doc.getArray('array') -const undoManager = new Y.UndoManager(ytext) - -ytext.insert(0, 'abc') -undoManager.undo() -ytext.toString() // => '' -undoManager.redo() -ytext.toString() // => 'abc' -``` - -
- constructor(scope:Y.AbstractType|Array<Y.AbstractType>, - [[{captureTimeout:number,trackedOrigins:Set<any>,deleteFilter:function(item):boolean}]]) -
Accepts either single type as scope or an array of types.
- undo() -
- redo() -
- stopCapturing() -
- - -on('stack-item-added', { stackItem: { meta: Map<any,any> }, type: 'undo' -| 'redo' }) - - -
-Register an event that is called when a StackItem is added to the -undo- or the redo-stack. -
- - -on('stack-item-popped', { stackItem: { meta: Map<any,any> }, type: 'undo' -| 'redo' }) - - -
-Register an event that is called when a StackItem is popped from -the undo- or the redo-stack. -
-
- -#### Example: Stop Capturing - -UndoManager merges Undo-StackItems if they are created within time-gap -smaller than `options.captureTimeout`. Call `um.stopCapturing()` so that the next -StackItem won't be merged. - -```js -// without stopCapturing -ytext.insert(0, 'a') -ytext.insert(1, 'b') -um.undo() -ytext.toString() // => '' (note that 'ab' was removed) -// with stopCapturing -ytext.insert(0, 'a') -um.stopCapturing() -ytext.insert(0, 'b') -um.undo() -ytext.toString() // => 'a' (note that only 'b' was removed) -``` - -#### Example: Specify tracked origins - -Every change on the shared document has an origin. If no origin was specified, -it defaults to `null`. By specifying `trackedTransactionOrigins` you can -selectively specify which changes should be tracked by `UndoManager`. The -UndoManager instance is always added to `trackedTransactionOrigins`. - -```js -class CustomBinding {} - -const ytext = doc.getArray('array') -const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) - -ytext.insert(0, 'abc') -undoManager.undo() -ytext.toString() // => 'abc' (does not track because origin `null` and not part - // of `trackedTransactionOrigins`) -ytext.delete(0, 3) // revert change - -doc.transact(() => { - ytext.insert(0, 'abc') -}, 42) -undoManager.undo() -ytext.toString() // => '' (tracked because origin is an instance of `trackedTransactionorigins`) - -doc.transact(() => { - ytext.insert(0, 'abc') -}, 41) -undoManager.undo() -ytext.toString() // => '' (not tracked because 41 is not an instance of - // `trackedTransactionorigins`) -ytext.delete(0, 3) // revert change - -doc.transact(() => { - ytext.insert(0, 'abc') -}, new CustomBinding()) -undoManager.undo() -ytext.toString() // => '' (tracked because origin is a `CustomBinding` and - // `CustomBinding` is in `trackedTransactionorigins`) -``` - -#### Example: Add additional information to the StackItems - -When undoing or redoing a previous action, it is often expected to restore -additional meta information like the cursor location or the view on the -document. You can assign meta-information to Undo-/Redo-StackItems. - -```js -const ytext = doc.getArray('array') -const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) - -undoManager.on('stack-item-added', event => { - // save the current cursor location on the stack-item - event.stackItem.meta.set('cursor-location', getRelativeCursorLocation()) -}) - -undoManager.on('stack-item-popped', event => { - // restore the current cursor location on the stack-item - restoreCursorLocation(event.stackItem.meta.get('cursor-location')) -}) -``` - -## Miscellaneous - -### Typescript Declarations - -Yjs has type descriptions. But until [this -ticket](https://github.com/Microsoft/TypeScript/issues/7546) is fixed, this is -how you can make use of Yjs type declarations. - -```json -{ - "compilerOptions": { - "allowJs": true, - "checkJs": true, - }, - "maxNodeModuleJsDepth": 5 -} -``` - -## Yjs CRDT Algorithm - -*Conflict-free replicated data types* (CRDT) for collaborative editing are an -alternative approach to *operational transformation* (OT). A very simple -differenciation between the two approaches is that OT attempts to transform -index positions to ensure convergence (all clients end up with the same -content), while CRDTs use mathematical models that usually do not involve index -transformations, like linked lists. OT is currently the de-facto standard for -shared editing on text. OT approaches that support shared editing without a -central source of truth (a central server) require too much bookkeeping to be -viable in practice. CRDTs are better suited for distributed systems, provide -additional guarantees that the document can be synced with remote clients, and -do not require a central source of truth. - -Yjs implements a modified version of the algorithm described in [this -paper](https://www.researchgate.net/publication/310212186_Near_Real-Time_Peer-to-Peer_Shared_Editing_on_Extensible_Data_Types). -I will eventually publish a paper that describes why this approach works so well -in practice. Note: Since operations make up the document structure, we prefer -the term *struct* now. - -CRDTs suitable for shared text editing suffer from the fact that they only grow -in size. There are CRDTs that do not grow in size, but they do not have the -characteristics that are benificial for shared text editing (like intention -preservation). Yjs implements many improvements to the original algorithm that -diminish the trade-off that the document only grows in size. We can't garbage -collect deleted structs (tombstones) while ensuring a unique order of the -structs. But we can 1. merge preceeding structs into a single struct to reduce -the amount of meta information, 2. we can delete content from the struct if it -is deleted, and 3. we can garbage collect tombstones if we don't care about the -order of the structs anymore (e.g. if the parent was deleted). - -**Examples:** - -1. If a user inserts elements in sequence, the struct will be merged into a - single struct. E.g. `array.insert(0, ['a']), array.insert(0, ['b']);` is - first represented as two structs (`[{id: {client, clock: 0}, content: 'a'}, - {id: {client, clock: 1}, content: 'b'}`) and then merged into a single - struct: `[{id: {client, clock: 0}, content: 'ab'}]`. -2. When a struct that contains content (e.g. `ItemString`) is deleted, the - struct will be replaced with an `ItemDeleted` that does not contain content - anymore. -3. When a type is deleted, all child elements are transformed to `GC` structs. A - `GC` struct only denotes the existence of a struct and that it is deleted. - `GC` structs can always be merged with other `GC` structs if the id's are - adjacent. - -Especially when working on structured content (e.g. shared editing on -ProseMirror), these improvements yield very good results when -[benchmarking](https://github.com/dmonad/crdt-benchmarks) random document edits. -In practice they show even better results, because users usually edit text in -sequence, resulting in structs that can easily be merged. The benchmarks show -that even in the worst case scenario that a user edits text from right to left, -Yjs achieves good performance even for huge documents. - -### State Vector - -Yjs has the ability to exchange only the differences when syncing two clients. -We use lamport timestamps to identify structs and to track in which order a -client created them. Each struct has an `struct.id = { client: number, clock: -number}` that uniquely identifies a struct. We define the next expected `clock` -by each client as the *state vector*. This data structure is similar to the -[version vectors](https://en.wikipedia.org/wiki/Version_vector) data structure. -But we use state vectors only to describe the state of the local document, so we -can compute the missing struct of the remote client. We do not use it to track -causality. - -## License and Author - -Yjs and all related projects are [**MIT licensed**](./LICENSE). - -Yjs is based on my research as a student at the [RWTH -i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time. - -Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or -hiring [me](https://github.com/dmonad) for professional support. diff --git a/package.json b/package.json index 0f318536..941d2f9a 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,8 @@ "test-exhaustive": "npm run lint && npm run dist && node ./dist/tests.js --repitition-time 10000", "dist": "rm -rf dist && rollup -c", "watch": "rollup -wc", - "lint": "markdownlint README.v13.md && standard && tsc", - "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.v13.md --package ./package.json || true", + "lint": "markdownlint README.md && standard && tsc", + "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true", "serve-docs": "npm run docs && serve ./docs/", "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.js --repitition-time 1000", "postversion": "git push && git push --tags", @@ -38,20 +38,20 @@ }, "repository": { "type": "git", - "url": "https://github.com/y-js/yjs.git" + "url": "https://github.com/yjs/yjs.git" }, "keywords": [ "crdt" ], "author": "Kevin Jahns", - "email": "kevin.jahns@rwth-aachen.de", + "email": "kevin.jahns@protonmail.com", "license": "MIT", "bugs": { - "url": "https://github.com/y-js/yjs/issues" + "url": "https://github.com/yjs/yjs/issues" }, - "homepage": "http://y-js.org", + "homepage": "https://yjs.dev", "dependencies": { - "lib0": "0.0.6" + "lib0": "^0.1.0" }, "devDependencies": { "concurrently": "^3.6.1", diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index 953e862f..36804780 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -238,7 +238,7 @@ export const testInsertAndDeleteEventsForTypes2 = tc => { } /** - * This issue has been reported here https://github.com/y-js/yjs/issues/155 + * This issue has been reported here https://github.com/yjs/yjs/issues/155 * @param {t.TestCase} tc */ export const testNewChildDoesNotEmitEventInTransaction = tc => { From 6e7529723d336e4d91a2260fbb4d9d8e9b848618 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 4 Sep 2019 13:15:16 +0200 Subject: [PATCH 15/34] update lib0 --- package-lock.json | 36 ++++++++++++++++++++++-------------- package.json | 4 ++-- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 66a2f577..1f7e6f67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,9 +42,9 @@ } }, "acorn": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", - "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.0.0.tgz", + "integrity": "sha512-PaF/MduxijYYt7unVGRuds1vBC9bFxbNf+VWqhOClfdgy7RlVkQqt610ig1/yxTgsDIfW1cWDel5EBbOy3jdtQ==", "dev": true }, "acorn-jsx": { @@ -2596,9 +2596,9 @@ } }, "lib0": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.0.6.tgz", - "integrity": "sha512-drb8LcwZu2rAmTsXN0d3hFtZVbPE5ZUrsWf307Boc/v7IrmLq3lM5+OOMY672EysHTWeXo/OH54wRHyD6eFXXw==" + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.1.0.tgz", + "integrity": "sha512-pkpnv2IJEOb6iwpcJ6BVQu9GkZ9VINKeQ/0BcArHpozqaGQYWe+ychf2p9wHKToHUnivPoGZZ7rFqrxNXjqFBg==" }, "linkify-it": { "version": "2.2.0", @@ -3492,14 +3492,22 @@ } }, "rollup": { - "version": "1.12.4", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.12.4.tgz", - "integrity": "sha512-sHg0F05oTMJzM592MWU8irsPx8LIFMKSCnEkcp6vp/gnj+oJ9GJEBW9hl8jUqy2L6Q2uUxFzPgvoExLbfuSODA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.20.3.tgz", + "integrity": "sha512-/OMCkY0c6E8tleeVm4vQVDz24CkVgvueK3r8zTYu2AQNpjrcaPwO9hE+pWj5LTFrvvkaxt4MYIp2zha4y0lRvg==", "dev": true, "requires": { "@types/estree": "0.0.39", - "@types/node": "^12.0.2", - "acorn": "^6.1.1" + "@types/node": "^12.7.2", + "acorn": "^7.0.0" + }, + "dependencies": { + "@types/node": { + "version": "12.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.4.tgz", + "integrity": "sha512-W0+n1Y+gK/8G2P/piTkBBN38Qc5Q1ZSO6B5H3QmPCUewaiXOo2GCAWZ4ElZCcNhjJuBSUSLGFUJnmlCn5+nxOQ==", + "dev": true + } } }, "rollup-cli": { @@ -4158,9 +4166,9 @@ "dev": true }, "typescript": { - "version": "3.4.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.5.tgz", - "integrity": "sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.2.tgz", + "integrity": "sha512-lmQ4L+J6mnu3xweP8+rOrUwzmN+MRAj7TgtJtDaXE5PMyX2kCrklhg3rvOsOIfNeAWMQWO2F1GPc1kMD2vLAfw==", "dev": true }, "uc.micro": { diff --git a/package.json b/package.json index 941d2f9a..ad543dd6 100644 --- a/package.json +++ b/package.json @@ -57,12 +57,12 @@ "concurrently": "^3.6.1", "jsdoc": "^3.6.3", "live-server": "^1.2.1", - "rollup": "^1.11.3", + "rollup": "^1.20.3", "rollup-cli": "^1.0.9", "rollup-plugin-node-resolve": "^4.2.4", "standard": "^11.0.1", "tui-jsdoc-template": "^1.2.2", - "typescript": "^3.4.5", + "typescript": "^3.6.2", "y-protocols": "0.0.6" } } From 7d5db917da4ca66dca3755c8fb7013b6e9a278e6 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 4 Sep 2019 13:19:25 +0200 Subject: [PATCH 16/34] fix type error >= tsc@3.6 --- tests/testHelper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testHelper.js b/tests/testHelper.js index 913e9878..6296b7a6 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -378,7 +378,7 @@ export const compareDS = (ds1, ds2) => { */ export const applyRandomTests = (tc, mods, iterations, initTestObject) => { const gen = tc.prng - const result = init(tc, { users: 5 }, initTestObject || (() => null)) + const result = init(tc, { users: 5 }, initTestObject) const { testConnector, users } = result for (let i = 0; i < iterations; i++) { if (prng.int31(gen, 0, 100) <= 2) { From d9fface0be90f5c9b74d28bd1d4863bfbce807b4 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 4 Sep 2019 13:21:10 +0200 Subject: [PATCH 17/34] 13.0.0-97 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1f7e6f67..896b5cb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-96", + "version": "13.0.0-97", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index ad543dd6..18c1f157 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-96", + "version": "13.0.0-97", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From 1d297601e8bc8344ca4b9efea29e49487d101d80 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 4 Sep 2019 22:08:05 +0200 Subject: [PATCH 18/34] export .createDeleteSet functionality --- src/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/index.js b/src/index.js index 74876ddc..9f63300c 100644 --- a/src/index.js +++ b/src/index.js @@ -38,6 +38,8 @@ export { getState, Snapshot, createSnapshot, + createDeleteSet, + createDeleteSetFromStructStore, snapshot, emptySnapshot, findRootTypeKey, From d1f5ff0f59e5845e2110bd9872f0cc3ef358b6de Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 17 Sep 2019 18:53:59 +0200 Subject: [PATCH 19/34] implement PermanentUserData storage prototype --- package-lock.json | 6 +- package.json | 2 +- src/index.js | 3 +- src/internals.js | 7 +- src/structs/Item.js | 10 +-- src/types/AbstractType.js | 6 +- src/types/YText.js | 31 ++++---- src/utils/DeleteSet.js | 37 +++++---- src/utils/PermanentUserData.js | 134 +++++++++++++++++++++++++++++++++ src/utils/Snapshot.js | 4 +- src/utils/StructStore.js | 7 +- src/utils/Transaction.js | 14 +++- src/utils/UndoManager.js | 10 +-- src/utils/encoding.js | 5 +- tests/y-array.tests.js | 5 +- tests/y-text.tests.js | 2 +- 16 files changed, 215 insertions(+), 68 deletions(-) create mode 100644 src/utils/PermanentUserData.js diff --git a/package-lock.json b/package-lock.json index 896b5cb7..3180ad9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2596,9 +2596,9 @@ } }, "lib0": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.1.0.tgz", - "integrity": "sha512-pkpnv2IJEOb6iwpcJ6BVQu9GkZ9VINKeQ/0BcArHpozqaGQYWe+ychf2p9wHKToHUnivPoGZZ7rFqrxNXjqFBg==" + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.1.1.tgz", + "integrity": "sha512-ghjoI4xL/xzVR1fRLYEOnJjYMguoI2dnDUf5HYOpTfD6R5GPKLml6xNKl4ZfBVmczkIOQPNthhukp6nlgbmDLw==" }, "linkify-it": { "version": "2.2.0", diff --git a/package.json b/package.json index 18c1f157..502e605e 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ }, "homepage": "https://yjs.dev", "dependencies": { - "lib0": "^0.1.0" + "lib0": "^0.1.1" }, "devDependencies": { "concurrently": "^3.6.1", diff --git a/src/index.js b/src/index.js index 9f63300c..7ddcf46e 100644 --- a/src/index.js +++ b/src/index.js @@ -53,5 +53,6 @@ export { decodeSnapshot, encodeSnapshot, isDeleted, - equalSnapshots + equalSnapshots, + PermanentUserData // @TODO experimental } from './internals.js' diff --git a/src/internals.js b/src/internals.js index d5ac7848..288cb404 100644 --- a/src/internals.js +++ b/src/internals.js @@ -1,13 +1,16 @@ + export * from './utils/DeleteSet.js' +export * from './utils/Doc.js' +export * from './utils/encoding.js' export * from './utils/EventHandler.js' export * from './utils/ID.js' export * from './utils/isParentOf.js' +export * from './utils/PermanentUserData.js' export * from './utils/RelativePosition.js' export * from './utils/Snapshot.js' export * from './utils/StructStore.js' export * from './utils/Transaction.js' export * from './utils/UndoManager.js' -export * from './utils/Doc.js' export * from './utils/YEvent.js' export * from './types/AbstractType.js' @@ -31,5 +34,3 @@ export * from './structs/ContentAny.js' export * from './structs/ContentString.js' export * from './structs/ContentType.js' export * from './structs/Item.js' - -export * from './utils/encoding.js' diff --git a/src/structs/Item.js b/src/structs/Item.js index 95530b6e..3abfa325 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -135,7 +135,7 @@ export const splitItem = (transaction, leftItem, diff) => { */ export const redoItem = (transaction, item, redoitems) => { if (item.redone !== null) { - return getItemCleanStart(transaction, transaction.doc.store, item.redone) + return getItemCleanStart(transaction, item.redone) } let parentItem = item.parent._item /** @@ -175,7 +175,7 @@ export const redoItem = (transaction, item, redoitems) => { } if (parentItem !== null && parentItem.redone !== null) { while (parentItem.redone !== null) { - parentItem = getItemCleanStart(transaction, transaction.doc.store, parentItem.redone) + parentItem = getItemCleanStart(transaction, parentItem.redone) } // find next cloned_redo items while (left !== null) { @@ -185,7 +185,7 @@ export const redoItem = (transaction, item, redoitems) => { let leftTrace = left // trace redone until parent matches while (leftTrace !== null && leftTrace.parent._item !== parentItem) { - leftTrace = leftTrace.redone === null ? null : getItemCleanStart(transaction, transaction.doc.store, leftTrace.redone) + leftTrace = leftTrace.redone === null ? null : getItemCleanStart(transaction, leftTrace.redone) } if (leftTrace !== null && leftTrace.parent._item === parentItem) { left = leftTrace @@ -200,7 +200,7 @@ export const redoItem = (transaction, item, redoitems) => { let rightTrace = right // trace redone until parent matches while (rightTrace !== null && rightTrace.parent._item !== parentItem) { - rightTrace = rightTrace.redone === null ? null : getItemCleanStart(transaction, transaction.doc.store, rightTrace.redone) + rightTrace = rightTrace.redone === null ? null : getItemCleanStart(transaction, rightTrace.redone) } if (rightTrace !== null && rightTrace.parent._item === parentItem) { right = rightTrace @@ -726,7 +726,7 @@ export class ItemRef extends AbstractStructRef { } const left = this.left === null ? null : getItemCleanEnd(transaction, store, this.left) - const right = this.right === null ? null : getItemCleanStart(transaction, store, this.right) + const right = this.right === null ? null : getItemCleanStart(transaction, this.right) let parent = null let parentSub = this.parentSub if (this.parent !== null) { diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 82aa8fc8..731e12d4 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -428,7 +428,7 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => { if (index <= n.length) { if (index < n.length) { // insert in-between - getItemCleanStart(transaction, transaction.doc.store, createID(n.id.client, n.id.clock + index)) + getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index)) } break } @@ -454,7 +454,7 @@ export const typeListDelete = (transaction, parent, index, length) => { for (; n !== null && index > 0; n = n.right) { if (!n.deleted && n.countable) { if (index < n.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(n.id.client, n.id.clock + index)) + getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index)) } index -= n.length } @@ -463,7 +463,7 @@ export const typeListDelete = (transaction, parent, index, length) => { while (length > 0 && n !== null) { if (!n.deleted) { if (length < n.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(n.id.client, n.id.clock + length)) + getItemCleanStart(transaction, createID(n.id.client, n.id.clock + length)) } n.delete(transaction) length -= n.length diff --git a/src/types/YText.js b/src/types/YText.js index 01f82ace..2520f3ee 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -17,7 +17,7 @@ import { ContentFormat, ContentString, splitSnapshotAffectedStructs, - Doc, Item, Snapshot, StructStore, Transaction // eslint-disable-line + ID, Doc, Item, Snapshot, Transaction // eslint-disable-line } from '../internals.js' import * as decoding from 'lib0/decoding.js' // eslint-disable-line @@ -68,7 +68,6 @@ export class ItemInsertionResult extends ItemListPosition { /** * @param {Transaction} transaction - * @param {StructStore} store * @param {Map} currentAttributes * @param {Item|null} left * @param {Item|null} right @@ -78,7 +77,7 @@ export class ItemInsertionResult extends ItemListPosition { * @private * @function */ -const findNextPosition = (transaction, store, currentAttributes, left, right, count) => { +const findNextPosition = (transaction, currentAttributes, left, right, count) => { while (right !== null && count > 0) { switch (right.content.constructor) { case ContentEmbed: @@ -86,7 +85,7 @@ const findNextPosition = (transaction, store, currentAttributes, left, right, co if (!right.deleted) { if (count < right.length) { // split right - getItemCleanStart(transaction, store, createID(right.id.client, right.id.clock + count)) + getItemCleanStart(transaction, createID(right.id.client, right.id.clock + count)) } count -= right.length } @@ -105,7 +104,6 @@ const findNextPosition = (transaction, store, currentAttributes, left, right, co /** * @param {Transaction} transaction - * @param {StructStore} store * @param {AbstractType} parent * @param {number} index * @return {ItemTextListPosition} @@ -113,11 +111,11 @@ const findNextPosition = (transaction, store, currentAttributes, left, right, co * @private * @function */ -const findPosition = (transaction, store, parent, index) => { +const findPosition = (transaction, parent, index) => { let currentAttributes = new Map() let left = null let right = parent._start - return findNextPosition(transaction, store, currentAttributes, left, right, index) + return findNextPosition(transaction, currentAttributes, left, right, index) } /** @@ -299,7 +297,7 @@ const formatText = (transaction, parent, left, right, currentAttributes, length, case ContentEmbed: case ContentString: if (length < right.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(right.id.client, right.id.clock + length)) + getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length)) } length -= right.length break @@ -343,7 +341,7 @@ const deleteText = (transaction, left, right, currentAttributes, length) => { case ContentEmbed: case ContentString: if (length < right.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(right.id.client, right.id.clock + length)) + getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length)) } length -= right.length right.delete(transaction) @@ -714,11 +712,12 @@ export class YText extends AbstractType { * * @param {Snapshot} [snapshot] * @param {Snapshot} [prevSnapshot] + * @param {function('removed' | 'added', ID):any} [computeYChange] * @return {any} The Delta representation of this type. * * @public */ - toDelta (snapshot, prevSnapshot) { + toDelta (snapshot, prevSnapshot, computeYChange) { /** * @type{Array} */ @@ -767,12 +766,12 @@ export class YText extends AbstractType { if (snapshot !== undefined && !isVisible(n, snapshot)) { if (cur === undefined || cur.user !== n.id.client || cur.state !== 'removed') { packStr() - currentAttributes.set('ychange', { user: n.id.client, state: 'removed' }) + currentAttributes.set('ychange', computeYChange ? computeYChange('removed', n.id) : { type: 'removed' }) } } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { if (cur === undefined || cur.user !== n.id.client || cur.state !== 'added') { packStr() - currentAttributes.set('ychange', { user: n.id.client, state: 'added' }) + currentAttributes.set('ychange', computeYChange ? computeYChange('added', n.id) : { type: 'added' }) } } else if (cur !== undefined) { packStr() @@ -818,7 +817,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + const { left, right, currentAttributes } = findPosition(transaction, this, index) if (!attributes) { attributes = {} currentAttributes.forEach((v, k) => { attributes[k] = v }) @@ -847,7 +846,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + const { left, right, currentAttributes } = findPosition(transaction, this, index) insertText(transaction, this, left, right, currentAttributes, embed, attributes) }) } else { @@ -870,7 +869,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + const { left, right, currentAttributes } = findPosition(transaction, this, index) deleteText(transaction, left, right, currentAttributes, length) }) } else { @@ -892,7 +891,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - let { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + let { left, right, currentAttributes } = findPosition(transaction, this, index) if (right === null) { return } diff --git a/src/utils/DeleteSet.js b/src/utils/DeleteSet.js index ea3b7b59..16c4a2ba 100644 --- a/src/utils/DeleteSet.js +++ b/src/utils/DeleteSet.js @@ -8,6 +8,7 @@ import { Item, GC, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' +import * as array from 'lib0/array.js' import * as math from 'lib0/math.js' import * as map from 'lib0/map.js' import * as encoding from 'lib0/encoding.js' @@ -52,14 +53,13 @@ export class DeleteSet { * * @param {Transaction} transaction * @param {DeleteSet} ds - * @param {StructStore} store * @param {function(GC|Item):void} f * * @function */ -export const iterateDeletedStructs = (transaction, ds, store, f) => +export const iterateDeletedStructs = (transaction, ds, f) => ds.clients.forEach((deletes, clientid) => { - const structs = /** @type {Array} */ (store.clients.get(clientid)) + const structs = /** @type {Array} */ (transaction.doc.store.clients.get(clientid)) for (let i = 0; i < deletes.length; i++) { const del = deletes[i] iterateStructs(transaction, structs, del.clock, del.len, f) @@ -137,22 +137,27 @@ export const sortAndMergeDeleteSet = ds => { } /** - * @param {DeleteSet} ds1 - * @param {DeleteSet} ds2 + * @param {Array} dss * @return {DeleteSet} A fresh DeleteSet */ -export const mergeDeleteSets = (ds1, ds2) => { +export const mergeDeleteSets = dss => { const merged = new DeleteSet() - // Write all keys from ds1 to merged. If ds2 has the same key, combine the sets. - ds1.clients.forEach((dels1, client) => - merged.clients.set(client, dels1.concat(ds2.clients.get(client) || [])) - ) - // Write all missing keys from ds2 to merged. - ds2.clients.forEach((dels2, client) => { - if (!merged.clients.has(client)) { - merged.clients.set(client, dels2) - } - }) + for (let dssI = 0; dssI < dss.length; dssI++) { + dss[dssI].clients.forEach((delsLeft, client) => { + if (!merged.clients.has(client)) { + // Write all missing keys from current ds and all following. + // If merged already contains `client` current ds has already been added. + /** + * @type {Array} + */ + const dels = delsLeft.slice() + for (let i = dssI + 1; i < dss.length; i++) { + array.appendTo(dels, dss[i].clients.get(client) || []) + } + merged.clients.set(client, dels) + } + }) + } sortAndMergeDeleteSet(merged) return merged } diff --git a/src/utils/PermanentUserData.js b/src/utils/PermanentUserData.js new file mode 100644 index 00000000..289a3755 --- /dev/null +++ b/src/utils/PermanentUserData.js @@ -0,0 +1,134 @@ + +import { + YArray, + YMap, + readDeleteSet, + writeDeleteSet, + createDeleteSet, + ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line +} from '../internals.js' + +import * as decoding from 'lib0/decoding.js' +import * as encoding from 'lib0/encoding.js' +import { mergeDeleteSets, isDeleted } from './DeleteSet.js' + +export class PermanentUserData { + /** + * @param {Doc} doc + * @param {string} key + */ + constructor (doc, key = 'users') { + const users = doc.getMap(key) + /** + * @type {Map} + */ + const dss = new Map() + this.yusers = users + this.doc = doc + /** + * Maps from clientid to userDescription + * + * @type {Map} + */ + this.clients = new Map() + this.dss = dss + /** + * @param {YMap} user + * @param {string} userDescription + */ + const initUser = (user, userDescription) => { + /** + * @type {YArray} + */ + const ds = user.get('ds') + const ids = user.get('ids') + const addClientId = /** @param {number} clientid */ clientid => this.clients.set(clientid, userDescription) + ds.observe(/** @param {YArrayEvent} event */ event => { + event.changes.added.forEach(item => { + item.content.getContent().forEach(encodedDs => { + if (encodedDs instanceof Uint8Array) { + this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(decoding.createDecoder(encodedDs))])) + } + }) + }) + }) + this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(decoding.createDecoder(encodedDs))))) + ids.observe(/** @param {YArrayEvent} event */ event => + event.changes.added.forEach(item => item.content.getContent().forEach(addClientId)) + ) + ids.forEach(addClientId) + } + // observe users + users.observe(event => { + event.keysChanged.forEach(userDescription => + initUser(users.get(userDescription), userDescription) + ) + }) + // add intial data + users.forEach(initUser) + } + /** + * @param {Doc} doc + * @param {number} clientid + * @param {string} userDescription + */ + setUserMapping (doc, clientid, userDescription) { + const users = this.yusers + let user = users.get(userDescription) + if (!user) { + user = new YMap() + user.set('ids', new YArray()) + user.set('ds', new YArray()) + users.set(userDescription, user) + } + user.get('ids').push([clientid]) + users.observe(event => { + const userOverwrite = users.get(userDescription) + if (userOverwrite !== user) { + // user was overwritten, port all data over to the next user object + // @todo Experiment with Y.Sets here + user = userOverwrite + // @todo iterate over old type + this.clients.forEach((_userDescription, clientid) => { + if (userDescription === _userDescription) { + user.get('ids').push([clientid]) + } + }) + const encoder = encoding.createEncoder() + const ds = this.dss.get(userDescription) + if (ds) { + writeDeleteSet(encoder, ds) + user.get('ds').push([encoding.toUint8Array(encoder)]) + } + } + }) + doc.on('afterTransaction', /** @param {Transaction} transaction */ transaction => { + const yds = user.get('ds') + const ds = transaction.deleteSet + if (transaction.local && ds.clients.size > 0) { + const encoder = encoding.createEncoder() + writeDeleteSet(encoder, ds) + yds.push([encoding.toUint8Array(encoder)]) + } + }) + } + /** + * @param {number} clientid + * @return {any} + */ + getUserByClientId (clientid) { + return this.clients.get(clientid) || null + } + /** + * @param {ID} id + * @return {string | null} + */ + getUserByDeletedId (id) { + for (const [userDescription, ds] of this.dss) { + if (isDeleted(ds, id)) { + return userDescription + } + } + return null + } +} diff --git a/src/utils/Snapshot.js b/src/utils/Snapshot.js index f9d2714e..716ddeed 100644 --- a/src/utils/Snapshot.js +++ b/src/utils/Snapshot.js @@ -131,10 +131,10 @@ export const splitSnapshotAffectedStructs = (transaction, snapshot) => { if (!meta.has(snapshot)) { snapshot.sv.forEach((clock, client) => { if (clock < getState(store, client)) { - getItemCleanStart(transaction, store, createID(client, clock)) + getItemCleanStart(transaction, createID(client, clock)) } }) - iterateDeletedStructs(transaction, snapshot.ds, store, item => {}) + iterateDeletedStructs(transaction, snapshot.ds, item => {}) meta.add(snapshot) } } diff --git a/src/utils/StructStore.js b/src/utils/StructStore.js index 84dd4b93..969acfa4 100644 --- a/src/utils/StructStore.js +++ b/src/utils/StructStore.js @@ -197,16 +197,15 @@ export const findIndexCleanStart = (transaction, structs, clock) => { * Expects that id is actually in store. This function throws or is an infinite loop otherwise. * * @param {Transaction} transaction - * @param {StructStore} store * @param {ID} id * @return {Item} * * @private * @function */ -export const getItemCleanStart = (transaction, store, id) => { - const structs = /** @type {Array} */ (store.clients.get(id.client)) - return /** @type {Item} */ (structs[findIndexCleanStart(transaction, structs, id.clock)]) +export const getItemCleanStart = (transaction, id) => { + const structs = /** @type {Array} */ (transaction.doc.store.clients.get(id.client)) + return structs[findIndexCleanStart(transaction, structs, id.clock)] } /** diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index 503bccbe..abaa547d 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -46,8 +46,9 @@ export class Transaction { /** * @param {Doc} doc * @param {any} origin + * @param {boolean} local */ - constructor (doc, origin) { + constructor (doc, origin, local) { /** * The Yjs instance. * @type {Doc} @@ -95,6 +96,11 @@ export class Transaction { * @type {Map} */ this.meta = new Map() + /** + * Whether this change originates from this doc. + * @type {boolean} + */ + this.local = local } } @@ -143,17 +149,17 @@ export const addChangedTypeToTransaction = (transaction, type, parentSub) => { * * @param {Doc} doc * @param {function(Transaction):void} f - * @param {any} [origin] + * @param {any} [origin=true] * * @private * @function */ -export const transact = (doc, f, origin = null) => { +export const transact = (doc, f, origin = null, local = true) => { const transactionCleanups = doc._transactionCleanups let initialCall = false if (doc._transaction === null) { initialCall = true - doc._transaction = new Transaction(doc, origin) + doc._transaction = new Transaction(doc, origin, local) transactionCleanups.push(doc._transaction) doc.emit('beforeTransaction', [doc._transaction, doc]) } diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index 789321a2..2066fa9e 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -52,7 +52,7 @@ const popStackItem = (undoManager, stack, eventType) => { const stackItem = /** @type {StackItem} */ (stack.pop()) const itemsToRedo = new Set() let performedChange = false - iterateDeletedStructs(transaction, stackItem.ds, store, struct => { + iterateDeletedStructs(transaction, stackItem.ds, struct => { if (struct instanceof Item && scope.some(type => isParentOf(type, struct))) { itemsToRedo.add(struct) } @@ -70,10 +70,10 @@ const popStackItem = (undoManager, stack, eventType) => { if (struct.redone !== null) { let { item, diff } = followRedone(store, struct.id) if (diff > 0) { - item = getItemCleanStart(transaction, store, createID(item.id.client, item.id.clock + diff)) + item = getItemCleanStart(transaction, createID(item.id.client, item.id.clock + diff)) } if (item.length > stackItem.len) { - getItemCleanStart(transaction, store, createID(item.id.client, item.id.clock + stackItem.len)) + getItemCleanStart(transaction, createID(item.id.client, item.id.clock + stackItem.len)) } struct = item } @@ -168,7 +168,7 @@ export class UndoManager extends Observable { if (now - this.lastChange < captureTimeout && stack.length > 0 && !undoing && !redoing) { // append change to last stack op const lastOp = stack[stack.length - 1] - lastOp.ds = mergeDeleteSets(lastOp.ds, transaction.deleteSet) + lastOp.ds = mergeDeleteSets([lastOp.ds, transaction.deleteSet]) lastOp.len = afterState - lastOp.start } else { // create a new stack op @@ -178,7 +178,7 @@ export class UndoManager extends Observable { this.lastChange = now } // make sure that deleted structs are not gc'd - iterateDeletedStructs(transaction, transaction.deleteSet, transaction.doc.store, /** @param {Item|GC} item */ item => { + iterateDeletedStructs(transaction, transaction.deleteSet, /** @param {Item|GC} item */ item => { if (item instanceof Item && this.scope.some(type => isParentOf(type, item))) { keepItem(item) } diff --git a/src/utils/encoding.js b/src/utils/encoding.js index a0ebf23f..00aee8ac 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -26,6 +26,7 @@ import { readAndApplyDeleteSet, writeDeleteSet, createDeleteSetFromStructStore, + transact, Doc, Transaction, AbstractStruct, StructStore, ID // eslint-disable-line } from '../internals.js' @@ -299,10 +300,10 @@ export const readStructs = (decoder, transaction, store) => { * @function */ export const readUpdate = (decoder, ydoc, transactionOrigin) => - ydoc.transact(transaction => { + transact(ydoc, transaction => { readStructs(decoder, transaction, ydoc.store) readAndApplyDeleteSet(decoder, transaction, ydoc.store) - }, transactionOrigin) + }, transactionOrigin, false) /** * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`. diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index 36804780..27a7f147 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -3,6 +3,7 @@ import { init, compare, applyRandomTests, Doc } from './testHelper.js' // eslint import * as Y from '../src/index.js' import * as t from 'lib0/testing.js' import * as prng from 'lib0/prng.js' +import * as math from 'lib0/math.js' /** * @param {t.TestCase} tc @@ -362,12 +363,12 @@ const arrayTransactions = [ var length = yarray.length if (length > 0) { var somePos = prng.int31(gen, 0, length - 1) - var delLength = prng.int31(gen, 1, Math.min(2, length - somePos)) + var delLength = prng.int31(gen, 1, math.min(2, length - somePos)) if (prng.bool(gen)) { var type = yarray.get(somePos) if (type.length > 0) { somePos = prng.int31(gen, 0, type.length - 1) - delLength = prng.int31(gen, 0, Math.min(2, type.length - somePos)) + delLength = prng.int31(gen, 0, math.min(2, type.length - somePos)) type.delete(somePos, delLength) } } else { diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index f422ed45..f76ea659 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -127,7 +127,7 @@ export const testSnapshot = tc => { delete v.attributes.ychange.user } }) - t.compare(state2Diff, [{insert: 'a'}, {insert: 'x', attributes: {ychange: { state: 'added' }}}, {insert: 'b', attributes: {ychange: { state: 'removed' }}}, { insert: 'cd' }]) + t.compare(state2Diff, [{insert: 'a'}, {insert: 'x', attributes: {ychange: { type: 'added' }}}, {insert: 'b', attributes: {ychange: { type: 'removed' }}}, { insert: 'cd' }]) } /** From 6c4971ae2518d03f6292a722e14236670c0847ec Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 17 Sep 2019 18:55:04 +0200 Subject: [PATCH 20/34] 13.0.0-98 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3180ad9b..aaa4c06b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-97", + "version": "13.0.0-98", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 502e605e..5d721911 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-97", + "version": "13.0.0-98", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From b38a8d99e5544b541b93fe12c34dfaf4297909cd Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Mon, 23 Sep 2019 11:05:50 +0200 Subject: [PATCH 21/34] fix absolute position calculation --- src/utils/RelativePosition.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/RelativePosition.js b/src/utils/RelativePosition.js index d2f7eb10..333111ed 100644 --- a/src/utils/RelativePosition.js +++ b/src/utils/RelativePosition.js @@ -228,7 +228,7 @@ export const createAbsolutePositionFromRelativePosition = (rpos, doc) => { return null } type = right.parent - if (type._item !== null && !type._item.deleted) { + if (type._item === null || !type._item.deleted) { index = right.deleted || !right.countable ? 0 : res.diff let n = right.left while (n !== null) { From 1d5947c6029321fbf8f1f12534c0f90217795df1 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 23 Sep 2019 11:11:45 +0200 Subject: [PATCH 22/34] 13.0.0-99 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index aaa4c06b..1db1f772 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-98", + "version": "13.0.0-99", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 5d721911..f180f443 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-98", + "version": "13.0.0-99", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From 7b8eee6b2556525837e9098b649dcfdb564204df Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Mon, 23 Sep 2019 11:22:24 +0200 Subject: [PATCH 23/34] update quill cursors support --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be5c2aee..5ca22fff 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ are implemented in separate modules. | Name | Cursors | Binding | Demo | |---|:-:|---|---| | [ProseMirror](https://prosemirror.net/)                                                   | ✔ | [y-prosemirror](http://github.com/yjs/y-prosemirror) | [demo](https://yjs-demos.now.sh/prosemirror/) | -| [Quill](https://quilljs.com/) | | [y-quill](http://github.com/yjs/y-quill) | [demo](https://yjs-demos.now.sh/quill/) | +| [Quill](https://quilljs.com/) | ✔ | [y-quill](http://github.com/yjs/y-quill) | [demo](https://yjs-demos.now.sh/quill/) | | [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/yjs/y-codemirror) | [demo](https://yjs-demos.now.sh/codemirror/) | | [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/yjs/y-monaco) | [demo](https://yjs-demos.now.sh/monaco/) | | [Ace](https://ace.c9.io/) | | [y-ace](http://github.com/yjs/y-ace) | [demo](https://yjs-demos.now.sh/ace/) | From 66d500f08dad8ebb794dc18a8a916dff2c9243ba Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 30 Sep 2019 11:07:18 +0200 Subject: [PATCH 24/34] YEvent: consider case that item was added & removed in the same transaction --- src/utils/YEvent.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/YEvent.js b/src/utils/YEvent.js index 05429ee7..3a9d1dc7 100644 --- a/src/utils/YEvent.js +++ b/src/utils/YEvent.js @@ -56,6 +56,8 @@ export class YEvent { /** * Check if a struct is deleted by this event. * + * In contrast to change.deleted, this method also returns true if the struct was added and then deleted. + * * @param {AbstractStruct} struct * @return {boolean} */ @@ -66,6 +68,8 @@ export class YEvent { /** * Check if a struct is added by this event. * + * In contrast to change.deleted, this method also returns true if the struct was added and then deleted. + * * @param {AbstractStruct} struct * @return {boolean} */ @@ -106,7 +110,7 @@ export class YEvent { } for (let item = target._start; item !== null; item = item.right) { if (item.deleted) { - if (this.deletes(item)) { + if (this.deletes(item) && !this.adds(item)) { if (lastOp === null || lastOp.delete === undefined) { packOp() lastOp = { delete: 0 } From ece4841b5c503ae4bac89c3d451d182c9ba3d958 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 3 Oct 2019 22:06:07 +0200 Subject: [PATCH 25/34] update stackItem.meta doc --- src/utils/UndoManager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index 2066fa9e..8fe3badb 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -111,9 +111,9 @@ const popStackItem = (undoManager, stack, eventType) => { /** * Fires 'stack-item-added' event when a stack item was added to either the undo- or * the redo-stack. You may store additional stack information via the - * metadata property on `event.stackItem.metadata` (it is a `Map` of metadata properties). + * metadata property on `event.stackItem.meta` (it is a `Map` of metadata properties). * Fires 'stack-item-popped' event when a stack item was popped from either the - * undo- or the redo-stack. You may restore the saved stack information from `event.stackItem.metadata`. + * undo- or the redo-stack. You may restore the saved stack information from `event.stackItem.meta`. * * @extends {Observable<'stack-item-added'|'stack-item-popped'>} */ From 2150fa58f2c7de152b9eb232d8bbbe5f6beb7a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Koren?= Date: Sat, 5 Oct 2019 15:14:30 +0200 Subject: [PATCH 26/34] Fixing Y.Map's documentation of forEach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes #171 As always, it's an honor to submit a PR! 🐒 There was also a missing dot in the Y.XmlFragment title. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ca22fff..39d71354 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,8 @@ or any of its children. Copies the [key,value] pairs of this YMap to a new Object.It transforms all child types to JSON using their toJSON method. - forEach(function(key:string,value:object|boolean|Array|string|number|Uint8Array|Y.Type)) + forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, + key:string, map: Y.Map))
Execute the provided function once for every key-value pair.
@@ -343,7 +344,7 @@ or any of its children.
- YXmlFragment + Y.XmlFragment

A container that holds an Array of Y.XmlElements. From 303138f3093165adaf03e4865e7eb067f2ddedbf Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 8 Oct 2019 17:36:00 +0200 Subject: [PATCH 27/34] sanitize items before undoing. fixes #165 --- src/utils/UndoManager.js | 25 +++++++++++++++++++++---- tests/undo-redo.tests.js | 13 +++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index 8fe3badb..21c3b694 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -9,6 +9,7 @@ import { createID, followRedone, getItemCleanStart, + getState, Transaction, Doc, Item, GC, DeleteSet, AbstractType // eslint-disable-line } from '../internals.js' @@ -49,23 +50,39 @@ const popStackItem = (undoManager, stack, eventType) => { transact(doc, transaction => { while (stack.length > 0 && result === null) { const store = doc.store + const clientID = doc.clientID const stackItem = /** @type {StackItem} */ (stack.pop()) + const stackStartClock = stackItem.start + const stackEndClock = stackItem.start + stackItem.len const itemsToRedo = new Set() + // @todo iterateStructs should not need the structs parameter + const structs = /** @type {Array} */ (store.clients.get(clientID)) let performedChange = false + if (stackStartClock !== stackEndClock) { + // make sure structs don't overlap with the range of created operations [stackItem.start, stackItem.start + stackItem.end) + getItemCleanStart(transaction, createID(clientID, stackStartClock)) + if (stackEndClock < getState(doc.store, clientID)) { + getItemCleanStart(transaction, createID(clientID, stackEndClock)) + } + } iterateDeletedStructs(transaction, stackItem.ds, struct => { - if (struct instanceof Item && scope.some(type => isParentOf(type, struct))) { + if ( + struct instanceof Item && + scope.some(type => isParentOf(type, struct)) && + // Never redo structs in [stackItem.start, stackItem.start + stackItem.end) because they were created and deleted in the same capture interval. + !(struct.id.client === clientID && struct.id.clock >= stackStartClock && struct.id.clock < stackEndClock) + ) { itemsToRedo.add(struct) } }) itemsToRedo.forEach(item => { performedChange = redoItem(transaction, item, itemsToRedo) !== null || performedChange }) - const structs = /** @type {Array} */ (store.clients.get(doc.clientID)) /** * @type {Array} */ const itemsToDelete = [] - iterateStructs(transaction, structs, stackItem.start, stackItem.len, struct => { + iterateStructs(transaction, structs, stackStartClock, stackItem.len, struct => { if (struct instanceof Item && !struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) { if (struct.redone !== null) { let { item, diff } = followRedone(store, struct.id) @@ -73,7 +90,7 @@ const popStackItem = (undoManager, stack, eventType) => { item = getItemCleanStart(transaction, createID(item.id.client, item.id.clock + diff)) } if (item.length > stackItem.len) { - getItemCleanStart(transaction, createID(item.id.client, item.id.clock + stackItem.len)) + getItemCleanStart(transaction, createID(item.id.client, stackEndClock)) } struct = item } diff --git a/tests/undo-redo.tests.js b/tests/undo-redo.tests.js index cc4ee027..cb85233e 100644 --- a/tests/undo-redo.tests.js +++ b/tests/undo-redo.tests.js @@ -13,6 +13,10 @@ import * as t from 'lib0/testing.js' export const testUndoText = tc => { const { testConnector, text0, text1 } = init(tc, { users: 3 }) const undoManager = new UndoManager(text0) + text0.insert(0, 'test') + text0.delete(0, 4) + undoManager.undo() + t.assert(text0.toString() === '') text0.insert(0, 'abc') text1.insert(0, 'xyz') testConnector.syncAll() @@ -65,6 +69,15 @@ export const testUndoMap = tc => { t.assert(map0.get('a') === 44) undoManager.redo() t.assert(map0.get('a') === 44) + + // test setting value multiple times + map0.set('b', 'initial') + undoManager.stopCapturing() + map0.set('b', 'val1') + map0.set('b', 'val2') + undoManager.stopCapturing() + undoManager.undo() + t.assert(map0.get('b') === 'initial') } /** From 6d4f0c0cdde7f91b60a70418937c88de32a8f7e6 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 8 Oct 2019 17:40:32 +0200 Subject: [PATCH 28/34] 13.0.0-100 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1db1f772..0c43c9c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-99", + "version": "13.0.0-100", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f180f443..2fcfa12e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-99", + "version": "13.0.0-100", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From aeb23dbaa9effbd848601dd9ba4b1526ce35088b Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 8 Oct 2019 18:31:56 +0200 Subject: [PATCH 29/34] follow redone items to prevent some undo-redo issues. Fixes #162 --- src/structs/Item.js | 2 ++ src/utils/UndoManager.js | 10 ++++++---- tests/undo-redo.tests.js | 13 +++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/structs/Item.js b/src/structs/Item.js index 3abfa325..10e0e841 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -35,6 +35,8 @@ import * as set from 'lib0/set.js' import * as binary from 'lib0/binary.js' /** + * @todo This should return several items + * * @param {StructStore} store * @param {ID} id * @return {{item:Item, diff:number}} diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index 21c3b694..667cc9ac 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -75,15 +75,15 @@ const popStackItem = (undoManager, stack, eventType) => { itemsToRedo.add(struct) } }) - itemsToRedo.forEach(item => { - performedChange = redoItem(transaction, item, itemsToRedo) !== null || performedChange + itemsToRedo.forEach(struct => { + performedChange = redoItem(transaction, struct, itemsToRedo) !== null || performedChange }) /** * @type {Array} */ const itemsToDelete = [] iterateStructs(transaction, structs, stackStartClock, stackItem.len, struct => { - if (struct instanceof Item && !struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) { + if (struct instanceof Item) { if (struct.redone !== null) { let { item, diff } = followRedone(store, struct.id) if (diff > 0) { @@ -94,7 +94,9 @@ const popStackItem = (undoManager, stack, eventType) => { } struct = item } - itemsToDelete.push(struct) + if (!struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) { + itemsToDelete.push(struct) + } } }) // We want to delete in reverse order so that children are deleted before diff --git a/tests/undo-redo.tests.js b/tests/undo-redo.tests.js index cb85233e..349dcf7f 100644 --- a/tests/undo-redo.tests.js +++ b/tests/undo-redo.tests.js @@ -13,10 +13,23 @@ import * as t from 'lib0/testing.js' export const testUndoText = tc => { const { testConnector, text0, text1 } = init(tc, { users: 3 }) const undoManager = new UndoManager(text0) + + // items that are added & deleted in the same transaction won't be undo text0.insert(0, 'test') text0.delete(0, 4) undoManager.undo() t.assert(text0.toString() === '') + + // follow redone items + text0.insert(0, 'a') + undoManager.stopCapturing() + text0.delete(0, 1) + undoManager.stopCapturing() + undoManager.undo() + t.assert(text0.toString() === 'a') + undoManager.undo() + t.assert(text0.toString() === '') + text0.insert(0, 'abc') text1.insert(0, 'xyz') testConnector.syncAll() From f4c919d9ec70f6c7eadc35d6758f1b8b35459d8e Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 8 Oct 2019 18:33:50 +0200 Subject: [PATCH 30/34] 13.0.0-101 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0c43c9c7..2728f48d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-100", + "version": "13.0.0-101", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 2fcfa12e..0f0fcdd8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-100", + "version": "13.0.0-101", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From d1063ab70b8adfbf044157cecb05110773866720 Mon Sep 17 00:00:00 2001 From: Roeland Bosch Date: Tue, 15 Oct 2019 17:07:20 +0200 Subject: [PATCH 31/34] Fix attrs loop in yXmlText --- src/types/YXmlText.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/YXmlText.js b/src/types/YXmlText.js index 6c4088db..e2a51e1e 100644 --- a/src/types/YXmlText.js +++ b/src/types/YXmlText.js @@ -56,7 +56,7 @@ export class YXmlText extends YText { const node = nestedNodes[i] str += `<${node.nodeName}` for (let j = 0; j < node.attrs.length; j++) { - const attr = node.attrs[i] + const attr = node.attrs[j] str += ` ${attr.key}="${attr.value}"` } str += '>' From f53dff50431de597a5eb61a1f8d4a92599308c88 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 25 Oct 2019 23:44:09 +0200 Subject: [PATCH 32/34] delay errors in observe callbacks to throw after cleanup is done --- src/types/AbstractType.js | 3 +- src/utils/Transaction.js | 294 +++++++++++++++++++++----------------- tests/y-array.tests.js | 2 +- tests/y-map.tests.js | 50 +++++++ tests/y-text.tests.js | 6 +- tsconfig.json | 5 +- 6 files changed, 226 insertions(+), 134 deletions(-) diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 731e12d4..1ec7f76a 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -30,7 +30,7 @@ import * as encoding from 'lib0/encoding.js' // eslint-disable-line * @param {EventType} event */ export const callTypeObservers = (type, transaction, event) => { - callEventHandlerListeners(type._eH, event, transaction) + const changedType = type const changedParentTypes = transaction.changedParentTypes while (true) { // @ts-ignore @@ -40,6 +40,7 @@ export const callTypeObservers = (type, transaction, event) => { } type = type._item.parent } + callEventHandlerListeners(changedType._eH, event, transaction) } /** diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index abaa547d..42cc4af4 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -17,6 +17,7 @@ import * as encoding from 'lib0/encoding.js' import * as map from 'lib0/map.js' import * as math from 'lib0/math.js' import * as set from 'lib0/set.js' +import { callAll } from 'lib0/function.js' /** * A transaction is created for every change on the Yjs model. It is possible @@ -144,6 +145,164 @@ export const addChangedTypeToTransaction = (transaction, type, parentSub) => { } } +/** + * @param {Array} transactionCleanups + * @param {number} i + */ +const cleanupTransactions = (transactionCleanups, i) => { + if (i < transactionCleanups.length) { + const transaction = transactionCleanups[i] + const doc = transaction.doc + const store = doc.store + const ds = transaction.deleteSet + try { + sortAndMergeDeleteSet(ds) + transaction.afterState = getStateVector(transaction.doc.store) + doc._transaction = null + doc.emit('beforeObserverCalls', [transaction, doc]) + /** + * An array of event callbacks. + * + * Each callback is called even if the other ones throw errors. + * + * @type {Array} + */ + const fs = [] + // observe events on changed types + transaction.changed.forEach((subs, itemtype) => + fs.push(() => { + if (itemtype._item === null || !itemtype._item.deleted) { + itemtype._callObserver(transaction, subs) + } + }) + ) + fs.push(() => { + // deep observe events + transaction.changedParentTypes.forEach((events, type) => + fs.push(() => { + // We need to think about the possibility that the user transforms the + // Y.Doc in the event. + if (type._item === null || !type._item.deleted) { + events = events + .filter(event => + event.target._item === null || !event.target._item.deleted + ) + events + .forEach(event => { + event.currentTarget = type + }) + // We don't need to check for events.length + // because we know it has at least one element + callEventHandlerListeners(type._dEH, events, transaction) + } + }) + ) + fs.push(() => doc.emit('afterTransaction', [transaction, doc])) + }) + callAll(fs, []) + } finally { + /** + * @param {Array} structs + * @param {number} pos + */ + const tryToMergeWithLeft = (structs, pos) => { + const left = structs[pos - 1] + const right = structs[pos] + if (left.deleted === right.deleted && left.constructor === right.constructor) { + if (left.mergeWith(right)) { + structs.splice(pos, 1) + if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) { + right.parent._map.set(right.parentSub, /** @type {Item} */ (left)) + } + } + } + } + // Replace deleted items with ItemDeleted / GC. + // This is where content is actually remove from the Yjs Doc. + if (doc.gc) { + for (const [client, deleteItems] of ds.clients) { + const structs = /** @type {Array} */ (store.clients.get(client)) + for (let di = deleteItems.length - 1; di >= 0; di--) { + const deleteItem = deleteItems[di] + const endDeleteItemClock = deleteItem.clock + deleteItem.len + for ( + let si = findIndexSS(structs, deleteItem.clock), struct = structs[si]; + si < structs.length && struct.id.clock < endDeleteItemClock; + struct = structs[++si] + ) { + const struct = structs[si] + if (deleteItem.clock + deleteItem.len <= struct.id.clock) { + break + } + if (struct instanceof Item && struct.deleted && !struct.keep) { + struct.gc(store, false) + } + } + } + } + } + // try to merge deleted / gc'd items + // merge from right to left for better efficiecy and so we don't miss any merge targets + for (const [client, deleteItems] of ds.clients) { + const structs = /** @type {Array} */ (store.clients.get(client)) + for (let di = deleteItems.length - 1; di >= 0; di--) { + const deleteItem = deleteItems[di] + // start with merging the item next to the last deleted item + const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1)) + for ( + let si = mostRightIndexToCheck, struct = structs[si]; + si > 0 && struct.id.clock >= deleteItem.clock; + struct = structs[--si] + ) { + tryToMergeWithLeft(structs, si) + } + } + } + + // on all affected store.clients props, try to merge + for (const [client, clock] of transaction.afterState) { + const beforeClock = transaction.beforeState.get(client) || 0 + if (beforeClock !== clock) { + const structs = /** @type {Array} */ (store.clients.get(client)) + // we iterate from right to left so we can safely remove entries + const firstChangePos = math.max(findIndexSS(structs, beforeClock), 1) + for (let i = structs.length - 1; i >= firstChangePos; i--) { + tryToMergeWithLeft(structs, i) + } + } + } + // try to merge mergeStructs + // @todo: it makes more sense to transform mergeStructs to a DS, sort it, and merge from right to left + // but at the moment DS does not handle duplicates + for (const mid of transaction._mergeStructs) { + const client = mid.client + const clock = mid.clock + const structs = /** @type {Array} */ (store.clients.get(client)) + const replacedStructPos = findIndexSS(structs, clock) + if (replacedStructPos + 1 < structs.length) { + tryToMergeWithLeft(structs, replacedStructPos + 1) + } + if (replacedStructPos > 0) { + tryToMergeWithLeft(structs, replacedStructPos) + } + } + // @todo Merge all the transactions into one and provide send the data as a single update message + doc.emit('afterTransactionCleanup', [transaction, doc]) + if (doc._observers.has('update')) { + const updateMessage = computeUpdateMessageFromTransaction(transaction) + if (updateMessage !== null) { + doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc]) + } + } + if (transactionCleanups.length <= i + 1) { + doc._transactionCleanups = [] + } else { + cleanupTransactions(transactionCleanups, i + 1) + } + } + } +} + /** * Implements the functionality of `y.transact(()=>{..})` * @@ -169,134 +328,13 @@ export const transact = (doc, f, origin = null, local = true) => { if (initialCall && transactionCleanups[0] === doc._transaction) { // The first transaction ended, now process observer calls. // Observer call may create new transactions for which we need to call the observers and do cleanup. - // We don't want to nest these calls, so we execute these calls one after another - for (let i = 0; i < transactionCleanups.length; i++) { - const transaction = transactionCleanups[i] - const store = transaction.doc.store - const ds = transaction.deleteSet - sortAndMergeDeleteSet(ds) - transaction.afterState = getStateVector(transaction.doc.store) - doc._transaction = null - doc.emit('beforeObserverCalls', [transaction, doc]) - // emit change events on changed types - transaction.changed.forEach((subs, itemtype) => { - if (itemtype._item === null || !itemtype._item.deleted) { - itemtype._callObserver(transaction, subs) - } - }) - transaction.changedParentTypes.forEach((events, type) => { - // We need to think about the possibility that the user transforms the - // Y.Doc in the event. - if (type._item === null || !type._item.deleted) { - events = events - .filter(event => - event.target._item === null || !event.target._item.deleted - ) - events - .forEach(event => { - event.currentTarget = type - }) - // We don't need to check for events.length - // because we know it has at least one element - callEventHandlerListeners(type._dEH, events, transaction) - } - }) - doc.emit('afterTransaction', [transaction, doc]) - /** - * @param {Array} structs - * @param {number} pos - */ - const tryToMergeWithLeft = (structs, pos) => { - const left = structs[pos - 1] - const right = structs[pos] - if (left.deleted === right.deleted && left.constructor === right.constructor) { - if (left.mergeWith(right)) { - structs.splice(pos, 1) - if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) { - right.parent._map.set(right.parentSub, /** @type {Item} */ (left)) - } - } - } - } - // Replace deleted items with ItemDeleted / GC. - // This is where content is actually remove from the Yjs Doc. - if (doc.gc) { - for (const [client, deleteItems] of ds.clients) { - const structs = /** @type {Array} */ (store.clients.get(client)) - for (let di = deleteItems.length - 1; di >= 0; di--) { - const deleteItem = deleteItems[di] - const endDeleteItemClock = deleteItem.clock + deleteItem.len - for ( - let si = findIndexSS(structs, deleteItem.clock), struct = structs[si]; - si < structs.length && struct.id.clock < endDeleteItemClock; - struct = structs[++si] - ) { - const struct = structs[si] - if (deleteItem.clock + deleteItem.len <= struct.id.clock) { - break - } - if (struct instanceof Item && struct.deleted && !struct.keep) { - struct.gc(store, false) - } - } - } - } - } - // try to merge deleted / gc'd items - // merge from right to left for better efficiecy and so we don't miss any merge targets - for (const [client, deleteItems] of ds.clients) { - const structs = /** @type {Array} */ (store.clients.get(client)) - for (let di = deleteItems.length - 1; di >= 0; di--) { - const deleteItem = deleteItems[di] - // start with merging the item next to the last deleted item - const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1)) - for ( - let si = mostRightIndexToCheck, struct = structs[si]; - si > 0 && struct.id.clock >= deleteItem.clock; - struct = structs[--si] - ) { - tryToMergeWithLeft(structs, si) - } - } - } - - // on all affected store.clients props, try to merge - for (const [client, clock] of transaction.afterState) { - const beforeClock = transaction.beforeState.get(client) || 0 - if (beforeClock !== clock) { - const structs = /** @type {Array} */ (store.clients.get(client)) - // we iterate from right to left so we can safely remove entries - const firstChangePos = math.max(findIndexSS(structs, beforeClock), 1) - for (let i = structs.length - 1; i >= firstChangePos; i--) { - tryToMergeWithLeft(structs, i) - } - } - } - // try to merge mergeStructs - // @todo: it makes more sense to transform mergeStructs to a DS, sort it, and merge from right to left - // but at the moment DS does not handle duplicates - for (const mid of transaction._mergeStructs) { - const client = mid.client - const clock = mid.clock - const structs = /** @type {Array} */ (store.clients.get(client)) - const replacedStructPos = findIndexSS(structs, clock) - if (replacedStructPos + 1 < structs.length) { - tryToMergeWithLeft(structs, replacedStructPos + 1) - } - if (replacedStructPos > 0) { - tryToMergeWithLeft(structs, replacedStructPos) - } - } - // @todo Merge all the transactions into one and provide send the data as a single update message - doc.emit('afterTransactionCleanup', [transaction, doc]) - if (doc._observers.has('update')) { - const updateMessage = computeUpdateMessageFromTransaction(transaction) - if (updateMessage !== null) { - doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc]) - } - } - } - doc._transactionCleanups = [] + // We don't want to nest these calls, so we execute these calls one after + // another. + // Also we need to ensure that all cleanups are called, even if the + // observes throw errors. + // This file is full of hacky try {} finally {} blocks to ensure that an + // event can throw errors and also that the cleanup is called. + cleanupTransactions(transactionCleanups, 0) } } } diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index 27a7f147..085ac310 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -207,7 +207,7 @@ export const testChangeEvent = tc => { const newArr = new Y.Array() array0.insert(0, [newArr, 4, 'dtrn']) t.assert(changes !== null && changes.added.size === 2 && changes.deleted.size === 0) - t.compare(changes.delta, [{insert: [newArr, 4, 'dtrn']}]) + t.compare(changes.delta, [{ insert: [newArr, 4, 'dtrn'] }]) changes = null array0.delete(0, 2) t.assert(changes !== null && changes.added.size === 0 && changes.deleted.size === 2) diff --git a/tests/y-map.tests.js b/tests/y-map.tests.js index becfde34..5e01757e 100644 --- a/tests/y-map.tests.js +++ b/tests/y-map.tests.js @@ -340,6 +340,56 @@ export const testChangeEvent = tc => { compare(users) } +/** + * @param {t.TestCase} tc + */ +export const testYmapEventExceptionsShouldCompleteTransaction = tc => { + const doc = new Y.Doc() + const map = doc.getMap('map') + + let updateCalled = false + let throwingObserverCalled = false + let throwingDeepObserverCalled = false + doc.on('update', () => { + updateCalled = true + }) + + const throwingObserver = () => { + throwingObserverCalled = true + throw new Error('Failure') + } + + const throwingDeepObserver = () => { + throwingDeepObserverCalled = true + throw new Error('Failure') + } + + map.observe(throwingObserver) + map.observeDeep(throwingDeepObserver) + + t.fails(() => { + map.set('y', '2') + }) + + t.assert(updateCalled) + t.assert(throwingObserverCalled) + t.assert(throwingDeepObserverCalled) + + // check if it works again + updateCalled = false + throwingObserverCalled = false + throwingDeepObserverCalled = false + t.fails(() => { + map.set('z', '3') + }) + + t.assert(updateCalled) + t.assert(throwingObserverCalled) + t.assert(throwingDeepObserverCalled) + + t.assert(map.get('z') === '3') +} + /** * @param {t.TestCase} tc */ diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index f76ea659..86175479 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -81,10 +81,10 @@ export const testBasicFormat = tc => { export const testGetDeltaWithEmbeds = tc => { const { text0 } = init(tc, { users: 1 }) text0.applyDelta([{ - insert: {linebreak: 's'} + insert: { linebreak: 's' } }]) t.compare(text0.toDelta(), [{ - insert: {linebreak: 's'} + insert: { linebreak: 's' } }]) } @@ -127,7 +127,7 @@ export const testSnapshot = tc => { delete v.attributes.ychange.user } }) - t.compare(state2Diff, [{insert: 'a'}, {insert: 'x', attributes: {ychange: { type: 'added' }}}, {insert: 'b', attributes: {ychange: { type: 'removed' }}}, { insert: 'cd' }]) + t.compare(state2Diff, [{ insert: 'a' }, { insert: 'x', attributes: { ychange: { type: 'added' } } }, { insert: 'b', attributes: { ychange: { type: 'removed' } } }, { insert: 'cd' }]) } /** diff --git a/tsconfig.json b/tsconfig.json index 45859200..54eacaa0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -38,7 +38,10 @@ "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ "paths": { - "yjs": ["./src/index.js"] + "yjs": ["./src/index.js"], + "lib0/*": ["node_modules/lib0/*"], + "lib0/set.js": ["node_modules/lib0/set.js"], + "lib0/function.js": ["node_modules/lib0/function.js"] }, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ // "typeRoots": [], /* List of folders to include type definitions from. */ From 641dc250769a61da1afbf5a0a1a0f707e212c2e4 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 25 Oct 2019 23:47:23 +0200 Subject: [PATCH 33/34] 13.0.0-102 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2728f48d..9ed65fb6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-101", + "version": "13.0.0-102", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 0f0fcdd8..e17380d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-101", + "version": "13.0.0-102", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", From 755de18fd5597c357389489468490291c87d4a6f Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 7 Nov 2019 14:41:50 +0100 Subject: [PATCH 34/34] Create Funding.yml --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..3e8090b1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: dmonad +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']