From 6c2cf0f7694da05542ce7e83bd7116889ce0868c Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sun, 12 Jul 2020 18:25:45 +0200 Subject: [PATCH] Implement experimental new encoder :rocket: --- .jsdoc.json | 12 +- README.md | 5 +- package-lock.json | 40 ++-- package.json | 8 +- src/index.js | 6 + src/internals.js | 2 + src/structs/AbstractStruct.js | 5 +- src/structs/ContentAny.js | 17 +- src/structs/ContentBinary.js | 13 +- src/structs/ContentDeleted.js | 15 +- src/structs/ContentEmbed.js | 12 +- src/structs/ContentFormat.js | 14 +- src/structs/ContentJSON.js | 17 +- src/structs/ContentString.js | 13 +- src/structs/ContentType.js | 12 +- src/structs/GC.js | 10 +- src/structs/Item.js | 69 ++---- src/types/AbstractType.js | 5 +- src/types/YArray.js | 11 +- src/types/YMap.js | 10 +- src/types/YText.js | 10 +- src/types/YXmlElement.js | 15 +- src/types/YXmlFragment.js | 11 +- src/types/YXmlHook.js | 16 +- src/types/YXmlText.js | 15 +- src/utils/DeleteSet.js | 72 +++--- src/utils/PermanentUserData.js | 16 +- src/utils/Snapshot.js | 30 ++- src/utils/StructStore.js | 5 +- src/utils/Transaction.js | 28 ++- src/utils/UpdateDecoder.js | 392 +++++++++++++++++++++++++++++++ src/utils/UpdateEncoder.js | 408 +++++++++++++++++++++++++++++++++ src/utils/encoding.js | 203 ++++++++++++---- tests/testHelper.js | 8 +- tests/y-array.tests.js | 28 ++- tests/y-text.tests.js | 7 +- 36 files changed, 1224 insertions(+), 336 deletions(-) create mode 100644 src/utils/UpdateDecoder.js create mode 100644 src/utils/UpdateEncoder.js diff --git a/.jsdoc.json b/.jsdoc.json index 86397dac..2d92c47a 100644 --- a/.jsdoc.json +++ b/.jsdoc.json @@ -17,10 +17,13 @@ "useCollapsibles": true, "collapse": true, "resources": { - "yjs.dev": "Yjs website" + "yjs.dev": "Website", + "docs.yjs.dev": "Docs", + "discuss.yjs.dev": "Forum", + "https://gitter.im/Yjs/community": "Chat" }, "logo": { - "url": "https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png", + "url": "https://yjs.dev/images/logo/yjs-512x512.png", "width": "162px", "height": "162px", "link": "/" @@ -35,7 +38,7 @@ ], "default": { "staticFiles": { - "include": ["examples/"] + "include": [] } } }, @@ -44,7 +47,6 @@ "encoding": "utf8", "private": false, "recurse": true, - "template": "./node_modules/tui-jsdoc-template", - "tutorials": "./examples" + "template": "./node_modules/tui-jsdoc-template" } } diff --git a/README.md b/README.md index 1460671d..e482fe17 100644 --- a/README.md +++ b/README.md @@ -958,5 +958,6 @@ Yjs and all related projects are [**MIT licensed**](./LICENSE). Yjs is based on my research as a student at the [RWTH i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time. -Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or -hiring [me](https://github.com/dmonad) for professional support. +Fund this project by donating on [GitHub Sponsors](https://github.com/sponsors/dmonad) +or hiring [me](https://github.com/dmonad) as a contractor for your collaborative +app. diff --git a/package-lock.json b/package-lock.json index 8fd65748..edd7e2a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1429,9 +1429,9 @@ "dev": true }, "isomorphic.js": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.1.3.tgz", - "integrity": "sha512-pabBRLDwYefSsNS+qCazJ97o7P5xDTrNoxSYFTM09JlZTxPrOEPGKekwqUy3/Np4C4PHnVUXHYsZPOix0jELsA==" + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.1.4.tgz", + "integrity": "sha512-t9zbgkjE7f9f2M6OSW49YEq0lUrSdAllBbWFUZoeck/rnnFae6UlhmDtXWs48VJY3ZpryCoZsRiAiKD44hPIGQ==" }, "js-tokens": { "version": "4.0.0", @@ -1548,9 +1548,9 @@ } }, "lib0": { - "version": "0.2.29", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.29.tgz", - "integrity": "sha512-bcQqmh3bUDVXwZrAJnekTpNk0uaShRg1bHMK7uzBIDFAWWdMXXCUtQXO/d/XsIxCiskgHTqF4jiQmDdoPCMVIw==", + "version": "0.2.32", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.32.tgz", + "integrity": "sha512-cHHKhHTojtvFSsthTk+CKuD17jMHIxuZxYpTzXj9TeQLPNoGNDPl6ax+J6eFETVe3ZvPMh3V0nGfJgGo6QgSvA==", "requires": { "isomorphic.js": "^0.1.3" } @@ -1716,18 +1716,18 @@ "dev": true }, "markdownlint": { - "version": "0.20.3", - "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.20.3.tgz", - "integrity": "sha512-J93s59tGvSFvAPWVUtEgxqPI0CHayTx1Z8poj1/4UJAquHGPIruWRMurkRldiNbgBiaQ4OOt15rHZbFfU6u05A==", + "version": "0.20.4", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.20.4.tgz", + "integrity": "sha512-jpfaPgjT0OpeBbemjYNZbzGG3hCLcAIvrm/pEY3+q/szDScG6ZonDacqySVRJAv9glbo8y4wBPJ0wgW17+9GGA==", "dev": true, "requires": { "markdown-it": "10.0.0" } }, "markdownlint-cli": { - "version": "0.23.1", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.23.1.tgz", - "integrity": "sha512-UARWuPILksAcVLTosUv1F1tLognNYQ/qjLRIgWwQAYqdl3QQrTPurU/X9Z2jrdAJYlOim868QsufxjYJpH0K7Q==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.23.2.tgz", + "integrity": "sha512-OSl5OZ8xzGN6z355cqRkiq67zPi3reJimklaF72p0554q85Dng5ToOjjSB9tDKZebSt85jX8cp+ruoQlPqOsPA==", "dev": true, "requires": { "commander": "~2.9.0", @@ -1739,8 +1739,8 @@ "jsonc-parser": "~2.2.0", "lodash.differencewith": "~4.5.0", "lodash.flatten": "~4.4.0", - "markdownlint": "~0.20.3", - "markdownlint-rule-helpers": "~0.10.0", + "markdownlint": "~0.20.4", + "markdownlint-rule-helpers": "~0.11.0", "minimatch": "~3.0.4", "minimist": "~1.2.5", "rc": "~1.2.7" @@ -1770,9 +1770,9 @@ } }, "markdownlint-rule-helpers": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/markdownlint-rule-helpers/-/markdownlint-rule-helpers-0.10.0.tgz", - "integrity": "sha512-0e8VUTjNdQwS7hTyNan9oOLsy4a7KEsXo3fxKMDRFRk6Jn+pLB3iKZ3mj/m6ECrlOUCxPYYmgOmmyk3bSdbIvw==", + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/markdownlint-rule-helpers/-/markdownlint-rule-helpers-0.11.0.tgz", + "integrity": "sha512-PhGii9dOiDJDXxiRMpK8N0FM9powprvRPsXALgkjlSPTwLh6ymH+iF3iUe3nq8KGu26tclFBlLL5xAGy/zb7FA==", "dev": true }, "marked": { @@ -2786,9 +2786,9 @@ "dev": true }, "typescript": { - "version": "3.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.3.tgz", - "integrity": "sha512-D/wqnB2xzNFIcoBG9FG8cXRDjiqSTbG2wd8DMZeQyJlP1vfTkIxH4GKveWaEBYySKIg+USu+E+EDIR47SqnaMQ==", + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.6.tgz", + "integrity": "sha512-Pspx3oKAPJtjNwE92YS05HQoY7z2SFyOpHo9MqJor3BXAGNaPUs83CuVp9VISFkSjyRfiTpmKuAYGJB7S7hOxw==", "dev": true }, "uc.micro": { diff --git a/package.json b/package.json index bf2c055e..b6fb4562 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "lint": "markdownlint README.md && standard && tsc", "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true", "serve-docs": "npm run docs && http-server ./docs/", - "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000", + "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000 && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs", "debug": "concurrently 'http-server -o test.html' 'npm run watch'", "trace-deopt": "clear && rollup -c && node --trace-deopt dist/test.cjs", "trace-opt": "clear && rollup -c && node --trace-opt dist/test.cjs" @@ -60,7 +60,7 @@ }, "homepage": "https://yjs.dev", "dependencies": { - "lib0": "^0.2.29" + "lib0": "^0.2.32" }, "devDependencies": { "@rollup/plugin-commonjs": "^11.1.0", @@ -68,12 +68,12 @@ "concurrently": "^3.6.1", "http-server": "^0.12.3", "jsdoc": "^3.6.4", - "markdownlint-cli": "^0.23.1", + "markdownlint-cli": "^0.23.2", "rollup": "^1.32.1", "rollup-cli": "^1.0.9", "standard": "^14.3.4", "tui-jsdoc-template": "^1.2.2", - "typescript": "^3.9.3", + "typescript": "^3.9.6", "y-protocols": "^0.2.3" } } diff --git a/src/index.js b/src/index.js index 5d3ad00c..2e8e86a2 100644 --- a/src/index.js +++ b/src/index.js @@ -48,12 +48,18 @@ export { typeMapGetSnapshot, iterateDeletedStructs, applyUpdate, + applyUpdateV2, readUpdate, + readUpdateV2, encodeStateAsUpdate, + encodeStateAsUpdateV2, encodeStateVector, + encodeStateVectorV2, UndoManager, decodeSnapshot, encodeSnapshot, + decodeSnapshotV2, + encodeSnapshotV2, isDeleted, isParentOf, equalSnapshots, diff --git a/src/internals.js b/src/internals.js index 288cb404..9b1b22b8 100644 --- a/src/internals.js +++ b/src/internals.js @@ -1,6 +1,8 @@ export * from './utils/DeleteSet.js' export * from './utils/Doc.js' +export * from './utils/UpdateDecoder.js' +export * from './utils/UpdateEncoder.js' export * from './utils/encoding.js' export * from './utils/EventHandler.js' export * from './utils/ID.js' diff --git a/src/structs/AbstractStruct.js b/src/structs/AbstractStruct.js index fddd92c4..fc335a73 100644 --- a/src/structs/AbstractStruct.js +++ b/src/structs/AbstractStruct.js @@ -1,9 +1,8 @@ import { - StructStore, ID, Transaction // eslint-disable-line + AbstractUpdateEncoder, ID, Transaction // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' // eslint-disable-line import * as error from 'lib0/error.js' export class AbstractStruct { @@ -35,7 +34,7 @@ export class AbstractStruct { } /** - * @param {encoding.Encoder} encoder The encoder to write data to. + * @param {AbstractUpdateEncoder} encoder The encoder to write data to. * @param {number} offset * @param {number} encodingRef */ diff --git a/src/structs/ContentAny.js b/src/structs/ContentAny.js index bb2d6a34..f00128df 100644 --- a/src/structs/ContentAny.js +++ b/src/structs/ContentAny.js @@ -1,10 +1,7 @@ import { - Transaction, Item, StructStore // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' - export class ContentAny { /** * @param {Array} arr @@ -77,15 +74,15 @@ export class ContentAny { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { const len = this.arr.length - encoding.writeVarUint(encoder, len - offset) + encoder.writeLen(len - offset) for (let i = offset; i < len; i++) { const c = this.arr[i] - encoding.writeAny(encoder, c) + encoder.writeAny(c) } } @@ -98,14 +95,14 @@ export class ContentAny { } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentAny} */ export const readContentAny = decoder => { - const len = decoding.readVarUint(decoder) + const len = decoder.readLen() const cs = [] for (let i = 0; i < len; i++) { - cs.push(decoding.readAny(decoder)) + cs.push(decoder.readAny()) } return new ContentAny(cs) } diff --git a/src/structs/ContentBinary.js b/src/structs/ContentBinary.js index e597a64e..15d92aa2 100644 --- a/src/structs/ContentBinary.js +++ b/src/structs/ContentBinary.js @@ -1,10 +1,7 @@ import { - StructStore, Item, Transaction // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' -import * as buffer from 'lib0/buffer.js' import * as error from 'lib0/error.js' export class ContentBinary { @@ -73,11 +70,11 @@ export class ContentBinary { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { - encoding.writeVarUint8Array(encoder, this.content) + encoder.writeBuf(this.content) } /** @@ -89,7 +86,7 @@ export class ContentBinary { } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentBinary} */ -export const readContentBinary = decoder => new ContentBinary(buffer.copyUint8Array(decoding.readVarUint8Array(decoder))) +export const readContentBinary = decoder => new ContentBinary(decoder.readBuf()) diff --git a/src/structs/ContentDeleted.js b/src/structs/ContentDeleted.js index 63e2f4f4..7e00bebe 100644 --- a/src/structs/ContentDeleted.js +++ b/src/structs/ContentDeleted.js @@ -1,12 +1,9 @@ import { addToDeleteSet, - StructStore, Item, Transaction // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' - export class ContentDeleted { /** * @param {number} len @@ -67,7 +64,7 @@ export class ContentDeleted { * @param {Item} item */ integrate (transaction, item) { - addToDeleteSet(transaction.deleteSet, item.id, this.len) + addToDeleteSet(transaction.deleteSet, item.id.client, item.id.clock, this.len) item.markDeleted() } @@ -80,11 +77,11 @@ export class ContentDeleted { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { - encoding.writeVarUint(encoder, this.len - offset) + encoder.writeLen(this.len - offset) } /** @@ -98,7 +95,7 @@ export class ContentDeleted { /** * @private * - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentDeleted} */ -export const readContentDeleted = decoder => new ContentDeleted(decoding.readVarUint(decoder)) +export const readContentDeleted = decoder => new ContentDeleted(decoder.readLen()) diff --git a/src/structs/ContentEmbed.js b/src/structs/ContentEmbed.js index e2c30043..66b922e3 100644 --- a/src/structs/ContentEmbed.js +++ b/src/structs/ContentEmbed.js @@ -1,10 +1,8 @@ import { - StructStore, Item, Transaction // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' import * as error from 'lib0/error.js' /** @@ -76,11 +74,11 @@ export class ContentEmbed { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { - encoding.writeVarString(encoder, JSON.stringify(this.embed)) + encoder.writeJSON(this.embed) } /** @@ -94,7 +92,7 @@ export class ContentEmbed { /** * @private * - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentEmbed} */ -export const readContentEmbed = decoder => new ContentEmbed(JSON.parse(decoding.readVarString(decoder))) +export const readContentEmbed = decoder => new ContentEmbed(decoder.readJSON()) diff --git a/src/structs/ContentFormat.js b/src/structs/ContentFormat.js index c7990c61..7cfc85d7 100644 --- a/src/structs/ContentFormat.js +++ b/src/structs/ContentFormat.js @@ -1,10 +1,8 @@ import { - Item, StructStore, Transaction // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' import * as error from 'lib0/error.js' /** @@ -78,12 +76,12 @@ export class ContentFormat { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { - encoding.writeVarString(encoder, this.key) - encoding.writeVarString(encoder, JSON.stringify(this.value)) + encoder.writeKey(this.key) + encoder.writeJSON(this.value) } /** @@ -95,7 +93,7 @@ export class ContentFormat { } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentFormat} */ -export const readContentFormat = decoder => new ContentFormat(decoding.readVarString(decoder), JSON.parse(decoding.readVarString(decoder))) +export const readContentFormat = decoder => new ContentFormat(decoder.readString(), decoder.readJSON()) diff --git a/src/structs/ContentJSON.js b/src/structs/ContentJSON.js index 70c4d770..baf4c46c 100644 --- a/src/structs/ContentJSON.js +++ b/src/structs/ContentJSON.js @@ -1,10 +1,7 @@ import { - Transaction, Item, StructStore // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' - /** * @private */ @@ -80,15 +77,15 @@ export class ContentJSON { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { const len = this.arr.length - encoding.writeVarUint(encoder, len - offset) + encoder.writeLen(len - offset) for (let i = offset; i < len; i++) { const c = this.arr[i] - encoding.writeVarString(encoder, c === undefined ? 'undefined' : JSON.stringify(c)) + encoder.writeString(c === undefined ? 'undefined' : JSON.stringify(c)) } } @@ -103,14 +100,14 @@ export class ContentJSON { /** * @private * - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentJSON} */ export const readContentJSON = decoder => { - const len = decoding.readVarUint(decoder) + const len = decoder.readLen() const cs = [] for (let i = 0; i < len; i++) { - const c = decoding.readVarString(decoder) + const c = decoder.readString() if (c === 'undefined') { cs.push(undefined) } else { diff --git a/src/structs/ContentString.js b/src/structs/ContentString.js index 2b3b7fdd..5c338a99 100644 --- a/src/structs/ContentString.js +++ b/src/structs/ContentString.js @@ -1,10 +1,7 @@ import { - Transaction, Item, StructStore // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' - /** * @private */ @@ -80,11 +77,11 @@ export class ContentString { */ gc (store) {} /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { - encoding.writeVarString(encoder, offset === 0 ? this.str : this.str.slice(offset)) + encoder.writeString(offset === 0 ? this.str : this.str.slice(offset)) } /** @@ -98,7 +95,7 @@ export class ContentString { /** * @private * - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentString} */ -export const readContentString = decoder => new ContentString(decoding.readVarString(decoder)) +export const readContentString = decoder => new ContentString(decoder.readString()) diff --git a/src/structs/ContentType.js b/src/structs/ContentType.js index e1dd7483..7fd45570 100644 --- a/src/structs/ContentType.js +++ b/src/structs/ContentType.js @@ -7,15 +7,13 @@ import { readYXmlFragment, readYXmlHook, readYXmlText, - ID, StructStore, Transaction, Item, YEvent, AbstractType // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Transaction, Item, YEvent, AbstractType // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' // eslint-disable-line -import * as decoding from 'lib0/decoding.js' import * as error from 'lib0/error.js' /** - * @type {Array>} + * @type {Array>} * @private */ export const typeRefs = [ @@ -150,7 +148,7 @@ export class ContentType { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { @@ -168,7 +166,7 @@ export class ContentType { /** * @private * - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {ContentType} */ -export const readContentType = decoder => new ContentType(typeRefs[decoding.readVarUint(decoder)](decoder)) +export const readContentType = decoder => new ContentType(typeRefs[decoder.readTypeRef()](decoder)) diff --git a/src/structs/GC.js b/src/structs/GC.js index 8f8d1908..0b9e4244 100644 --- a/src/structs/GC.js +++ b/src/structs/GC.js @@ -2,11 +2,9 @@ import { AbstractStruct, addStruct, - StructStore, Transaction, ID // eslint-disable-line + AbstractUpdateEncoder, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' - export const structGCRefNumber = 0 /** @@ -41,12 +39,12 @@ export class GC extends AbstractStruct { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { - encoding.writeUint8(encoder, structGCRefNumber) - encoding.writeVarUint(encoder, this.length - offset) + encoder.writeInfo(structGCRefNumber) + encoder.writeLen(this.length - offset) } /** diff --git a/src/structs/Item.js b/src/structs/Item.js index 95acec56..a22002c2 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -1,7 +1,5 @@ import { - readID, - writeID, GC, getState, AbstractStruct, @@ -23,12 +21,10 @@ import { readContentFormat, readContentType, addChangedTypeToTransaction, - Doc, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line } from '../internals.js' import * as error from 'lib0/error.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' import * as maplib from 'lib0/map.js' import * as set from 'lib0/set.js' import * as binary from 'lib0/binary.js' @@ -574,7 +570,7 @@ export class Item extends AbstractStruct { parent._length -= this.length } this.markDeleted() - addToDeleteSet(transaction.deleteSet, this.id, this.length) + addToDeleteSet(transaction.deleteSet, this.id.client, this.id.clock, this.length) maplib.setIfUndefined(transaction.changed, parent, set.create).add(this.parentSub) this.content.delete(transaction) } @@ -602,7 +598,7 @@ export class Item extends AbstractStruct { * * This is called when this Item is sent to a remote peer. * - * @param {encoding.Encoder} encoder The encoder to write data to. + * @param {AbstractUpdateEncoder} encoder The encoder to write data to. * @param {number} offset */ write (encoder, offset) { @@ -613,12 +609,12 @@ export class Item extends AbstractStruct { (origin === null ? 0 : binary.BIT8) | // origin is defined (rightOrigin === null ? 0 : binary.BIT7) | // right origin is defined (parentSub === null ? 0 : binary.BIT6) // parentSub is non-null - encoding.writeUint8(encoder, info) + encoder.writeInfo(info) if (origin !== null) { - writeID(encoder, origin) + encoder.writeLeftID(origin) } if (rightOrigin !== null) { - writeID(encoder, rightOrigin) + encoder.writeRightID(rightOrigin) } if (origin === null && rightOrigin === null) { const parent = /** @type {AbstractType} */ (this.parent) @@ -627,14 +623,14 @@ export class Item extends AbstractStruct { // parent type on y._map // find the correct key const ykey = findRootTypeKey(parent) - encoding.writeVarUint(encoder, 1) // write parentYKey - encoding.writeVarString(encoder, ykey) + encoder.writeParentInfo(true) // write parentYKey + encoder.writeString(ykey) } else { - encoding.writeVarUint(encoder, 0) // write parent id - writeID(encoder, parentItem.id) + encoder.writeParentInfo(false) // write parent id + encoder.writeLeftID(parentItem.id) } if (parentSub !== null) { - encoding.writeVarString(encoder, parentSub) + encoder.writeString(parentSub) } } this.content.write(encoder, offset) @@ -642,15 +638,15 @@ export class Item extends AbstractStruct { } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @param {number} info */ -const readItemContent = (decoder, info) => contentRefs[info & binary.BITS5](decoder) +export const readItemContent = (decoder, info) => contentRefs[info & binary.BITS5](decoder) /** * A lookup map for reading Item content. * - * @type {Array} + * @type {Array} */ export const contentRefs = [ () => { throw error.unexpectedCase() }, // GC is not ItemContent @@ -741,7 +737,7 @@ export class AbstractContent { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {number} offset */ write (encoder, offset) { @@ -755,38 +751,3 @@ export class AbstractContent { throw error.methodUnimplemented() } } - -/** - * @param {decoding.Decoder} decoder - * @param {ID} id - * @param {number} info - * @param {Doc} doc - */ -export const readItem = (decoder, id, info, doc) => { - /** - * The item that was originally to the left of this item. - * @type {ID | null} - */ - const origin = (info & binary.BIT8) === binary.BIT8 ? readID(decoder) : null - /** - * The item that was originally to the right of this item. - * @type {ID | null} - */ - const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? readID(decoder) : null - const canCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0 - const hasParentYKey = canCopyParentInfo ? decoding.readVarUint(decoder) === 1 : false - /** - * If parent = null and neither left nor right are defined, then we know that `parent` is child of `y` - * and we read the next string as parentYKey. - * It indicates how we store/retrieve parent from `y.share` - * @type {string|null} - */ - const parentYKey = canCopyParentInfo && hasParentYKey ? decoding.readVarString(decoder) : null - - return new Item( - id, null, origin, null, rightOrigin, - canCopyParentInfo && !hasParentYKey ? readID(decoder) : (parentYKey ? doc.get(parentYKey) : null), // parent - canCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoding.readVarString(decoder) : null, // parentSub - /** @type {AbstractContent} */ (readItemContent(decoder, info)) // item content - ) -} diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index 7f167edf..a292c9b3 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -11,13 +11,12 @@ import { ContentAny, ContentBinary, getItemCleanStart, - ID, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line + AbstractUpdateEncoder, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line } from '../internals.js' import * as map from 'lib0/map.js' import * as iterator from 'lib0/iterator.js' import * as error from 'lib0/error.js' -import * as encoding from 'lib0/encoding.js' // eslint-disable-line /** * Accumulate all (list) children of a type and return them as an Array. @@ -116,7 +115,7 @@ export class AbstractType { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder */ _write (encoder) { } diff --git a/src/types/YArray.js b/src/types/YArray.js index 02ad42f0..b6bd9252 100644 --- a/src/types/YArray.js +++ b/src/types/YArray.js @@ -15,12 +15,9 @@ import { YArrayRefID, callTypeObservers, transact, - Doc, Transaction, Item // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line } from '../internals.js' -import * as decoding from 'lib0/decoding.js' // eslint-disable-line -import * as encoding from 'lib0/encoding.js' - /** * Event that describes the changes on a YArray * @template T @@ -204,15 +201,15 @@ export class YArray extends AbstractType { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder */ _write (encoder) { - encoding.writeVarUint(encoder, YArrayRefID) + encoder.writeTypeRef(YArrayRefID) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * * @private * @function diff --git a/src/types/YMap.js b/src/types/YMap.js index 47df69ad..13a19453 100644 --- a/src/types/YMap.js +++ b/src/types/YMap.js @@ -14,11 +14,9 @@ import { YMapRefID, callTypeObservers, transact, - Doc, Transaction, Item // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' // eslint-disable-line import * as iterator from 'lib0/iterator.js' /** @@ -229,15 +227,15 @@ export class YMap extends AbstractType { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder */ _write (encoder) { - encoding.writeVarUint(encoder, YMapRefID) + encoder.writeTypeRef(YMapRefID) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * * @private * @function diff --git a/src/types/YText.js b/src/types/YText.js index 81e87a0a..90f9301e 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -20,11 +20,9 @@ import { splitSnapshotAffectedStructs, iterateDeletedStructs, iterateStructs, - ID, Doc, Item, Snapshot, Transaction // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line } from '../internals.js' -import * as decoding from 'lib0/decoding.js' // eslint-disable-line -import * as encoding from 'lib0/encoding.js' import * as object from 'lib0/object.js' import * as map from 'lib0/map.js' @@ -1096,15 +1094,15 @@ export class YText extends AbstractType { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder */ _write (encoder) { - encoding.writeVarUint(encoder, YTextRefID) + encoder.writeTypeRef(YTextRefID) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {YText} * * @private diff --git a/src/types/YXmlElement.js b/src/types/YXmlElement.js index c16e7abb..6d91da15 100644 --- a/src/types/YXmlElement.js +++ b/src/types/YXmlElement.js @@ -8,12 +8,9 @@ import { typeMapGetAll, typeListForEach, YXmlElementRefID, - Snapshot, Doc, Item // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Snapshot, Doc, Item // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' - /** * An YXmlElement imitates the behavior of a * {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}. @@ -181,18 +178,18 @@ export class YXmlElement extends YXmlFragment { * * This is called when this Item is sent to a remote peer. * - * @param {encoding.Encoder} encoder The encoder to write data to. + * @param {AbstractUpdateEncoder} encoder The encoder to write data to. */ _write (encoder) { - encoding.writeVarUint(encoder, YXmlElementRefID) - encoding.writeVarString(encoder, this.nodeName) + encoder.writeTypeRef(YXmlElementRefID) + encoder.writeKey(this.nodeName) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {YXmlElement} * * @function */ -export const readYXmlElement = decoder => new YXmlElement(decoding.readVarString(decoder)) +export const readYXmlElement = decoder => new YXmlElement(decoder.readKey()) diff --git a/src/types/YXmlFragment.js b/src/types/YXmlFragment.js index 36bdb2d9..4fcf09e7 100644 --- a/src/types/YXmlFragment.js +++ b/src/types/YXmlFragment.js @@ -14,12 +14,9 @@ import { YXmlFragmentRefID, callTypeObservers, transact, - Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line + AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' // eslint-disable-line - /** * Define the elements to which a set of CSS queries apply. * {@link https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors|CSS_Selectors} @@ -325,15 +322,15 @@ export class YXmlFragment extends AbstractType { * * This is called when this Item is sent to a remote peer. * - * @param {encoding.Encoder} encoder The encoder to write data to. + * @param {AbstractUpdateEncoder} encoder The encoder to write data to. */ _write (encoder) { - encoding.writeVarUint(encoder, YXmlFragmentRefID) + encoder.writeTypeRef(YXmlFragmentRefID) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {YXmlFragment} * * @private diff --git a/src/types/YXmlHook.js b/src/types/YXmlHook.js index 4bb17554..51a98366 100644 --- a/src/types/YXmlHook.js +++ b/src/types/YXmlHook.js @@ -1,10 +1,9 @@ import { YMap, - YXmlHookRefID + YXmlHookRefID, + AbstractUpdateDecoder, AbstractUpdateEncoder // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' /** * You can manage binding to a custom type with YXmlHook. @@ -66,21 +65,20 @@ export class YXmlHook extends YMap { * * This is called when this Item is sent to a remote peer. * - * @param {encoding.Encoder} encoder The encoder to write data to. + * @param {AbstractUpdateEncoder} encoder The encoder to write data to. */ _write (encoder) { - super._write(encoder) - encoding.writeVarUint(encoder, YXmlHookRefID) - encoding.writeVarString(encoder, this.hookName) + encoder.writeTypeRef(YXmlHookRefID) + encoder.writeKey(this.hookName) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {YXmlHook} * * @private * @function */ export const readYXmlHook = decoder => - new YXmlHook(decoding.readVarString(decoder)) + new YXmlHook(decoder.readKey()) diff --git a/src/types/YXmlText.js b/src/types/YXmlText.js index b7e03f18..cada1585 100644 --- a/src/types/YXmlText.js +++ b/src/types/YXmlText.js @@ -1,8 +1,9 @@ -import { YText, YXmlTextRefID } from '../internals.js' - -import * as encoding from 'lib0/encoding.js' -import * as decoding from 'lib0/decoding.js' // eslint-disable-line +import { + YText, + YXmlTextRefID, + AbstractUpdateDecoder, AbstractUpdateEncoder // eslint-disable-line +} from '../internals.js' /** * Represents text in a Dom Element. In the future this type will also handle @@ -78,15 +79,15 @@ export class YXmlText extends YText { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder */ _write (encoder) { - encoding.writeVarUint(encoder, YXmlTextRefID) + encoder.writeTypeRef(YXmlTextRefID) } } /** - * @param {decoding.Decoder} decoder + * @param {AbstractUpdateDecoder} decoder * @return {YXmlText} * * @private diff --git a/src/utils/DeleteSet.js b/src/utils/DeleteSet.js index 943fa031..658f5087 100644 --- a/src/utils/DeleteSet.js +++ b/src/utils/DeleteSet.js @@ -3,9 +3,8 @@ import { findIndexSS, getState, splitItem, - createID, iterateStructs, - Item, AbstractStruct, GC, StructStore, Transaction, ID // eslint-disable-line + AbstractUpdateDecoder, AbstractDSDecoder, AbstractDSEncoder, DSDecoderV2, DSEncoderV2, Item, GC, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' import * as array from 'lib0/array.js' @@ -163,14 +162,15 @@ export const mergeDeleteSets = dss => { /** * @param {DeleteSet} ds - * @param {ID} id + * @param {number} client + * @param {number} clock * @param {number} length * * @private * @function */ -export const addToDeleteSet = (ds, id, length) => { - map.setIfUndefined(ds.clients, id.client, () => []).push(new DeleteItem(id.clock, length)) +export const addToDeleteSet = (ds, client, clock, length) => { + map.setIfUndefined(ds.clients, client, () => []).push(new DeleteItem(clock, length)) } export const createDeleteSet = () => new DeleteSet() @@ -210,28 +210,29 @@ export const createDeleteSetFromStructStore = ss => { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractDSEncoder} encoder * @param {DeleteSet} ds * * @private * @function */ export const writeDeleteSet = (encoder, ds) => { - encoding.writeVarUint(encoder, ds.clients.size) + encoding.writeVarUint(encoder.restEncoder, ds.clients.size) ds.clients.forEach((dsitems, client) => { - encoding.writeVarUint(encoder, client) + encoder.resetDsCurVal() + encoding.writeVarUint(encoder.restEncoder, client) const len = dsitems.length - encoding.writeVarUint(encoder, len) + encoding.writeVarUint(encoder.restEncoder, len) for (let i = 0; i < len; i++) { const item = dsitems[i] - encoding.writeVarUint(encoder, item.clock) - encoding.writeVarUint(encoder, item.len) + encoder.writeDsClock(item.clock) + encoder.writeDsLen(item.len) } }) } /** - * @param {decoding.Decoder} decoder + * @param {AbstractDSDecoder} decoder * @return {DeleteSet} * * @private @@ -239,19 +240,27 @@ export const writeDeleteSet = (encoder, ds) => { */ export const readDeleteSet = decoder => { const ds = new DeleteSet() - const numClients = decoding.readVarUint(decoder) + const numClients = decoding.readVarUint(decoder.restDecoder) for (let i = 0; i < numClients; i++) { - const client = decoding.readVarUint(decoder) - const numberOfDeletes = decoding.readVarUint(decoder) - for (let i = 0; i < numberOfDeletes; i++) { - addToDeleteSet(ds, createID(client, decoding.readVarUint(decoder)), decoding.readVarUint(decoder)) + decoder.resetDsCurVal() + const client = decoding.readVarUint(decoder.restDecoder) + const numberOfDeletes = decoding.readVarUint(decoder.restDecoder) + if (numberOfDeletes > 0) { + const dsField = map.setIfUndefined(ds.clients, client, () => []) + for (let i = 0; i < numberOfDeletes; i++) { + dsField.push(new DeleteItem(decoder.readDsClock(), decoder.readDsLen())) + } } } return ds } /** - * @param {decoding.Decoder} decoder + * @todo YDecoder also contains references to String and other Decoders. Would make sense to exchange YDecoder.toUint8Array for YDecoder.DsToUint8Array().. + */ + +/** + * @param {AbstractDSDecoder} decoder * @param {Transaction} transaction * @param {StructStore} store * @@ -260,18 +269,19 @@ export const readDeleteSet = decoder => { */ export const readAndApplyDeleteSet = (decoder, transaction, store) => { const unappliedDS = new DeleteSet() - const numClients = decoding.readVarUint(decoder) + const numClients = decoding.readVarUint(decoder.restDecoder) for (let i = 0; i < numClients; i++) { - const client = decoding.readVarUint(decoder) - const numberOfDeletes = decoding.readVarUint(decoder) + decoder.resetDsCurVal() + const client = decoding.readVarUint(decoder.restDecoder) + const numberOfDeletes = decoding.readVarUint(decoder.restDecoder) const structs = store.clients.get(client) || [] const state = getState(store, client) for (let i = 0; i < numberOfDeletes; i++) { - const clock = decoding.readVarUint(decoder) - const len = decoding.readVarUint(decoder) + const clock = decoder.readDsClock() + const clockEnd = clock + decoder.readDsLen() if (clock < state) { - if (state < clock + len) { - addToDeleteSet(unappliedDS, createID(client, state), clock + len - state) + if (state < clockEnd) { + addToDeleteSet(unappliedDS, client, state, clockEnd - state) } let index = findIndexSS(structs, clock) /** @@ -288,10 +298,10 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => { while (index < structs.length) { // @ts-ignore struct = structs[index++] - if (struct.id.clock < clock + len) { + if (struct.id.clock < clockEnd) { if (!struct.deleted) { - if (clock + len < struct.id.clock + struct.length) { - structs.splice(index, 0, splitItem(transaction, struct, clock + len - struct.id.clock)) + if (clockEnd < struct.id.clock + struct.length) { + structs.splice(index, 0, splitItem(transaction, struct, clockEnd - struct.id.clock)) } struct.delete(transaction) } @@ -300,14 +310,14 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => { } } } else { - addToDeleteSet(unappliedDS, createID(client, clock), len) + addToDeleteSet(unappliedDS, client, clock, clockEnd - clock) } } } if (unappliedDS.clients.size > 0) { // TODO: no need for encoding+decoding ds anymore - const unappliedDSEncoder = encoding.createEncoder() + const unappliedDSEncoder = new DSEncoderV2() writeDeleteSet(unappliedDSEncoder, unappliedDS) - store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toUint8Array(unappliedDSEncoder))) + store.pendingDeleteReaders.push(new DSDecoderV2(decoding.createDecoder((unappliedDSEncoder.toUint8Array())))) } } diff --git a/src/utils/PermanentUserData.js b/src/utils/PermanentUserData.js index d4eee09b..aecd168b 100644 --- a/src/utils/PermanentUserData.js +++ b/src/utils/PermanentUserData.js @@ -5,11 +5,11 @@ import { readDeleteSet, writeDeleteSet, createDeleteSet, - ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line + DSEncoderV1, DSDecoderV1, ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line } from '../internals.js' import * as decoding from 'lib0/decoding.js' -import * as encoding from 'lib0/encoding.js' + import { mergeDeleteSets, isDeleted } from './DeleteSet.js' export class PermanentUserData { @@ -46,12 +46,12 @@ export class PermanentUserData { event.changes.added.forEach(item => { item.content.getContent().forEach(encodedDs => { if (encodedDs instanceof Uint8Array) { - this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(decoding.createDecoder(encodedDs))])) + this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(new DSDecoderV1(decoding.createDecoder(encodedDs)))])) } }) }) }) - this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(decoding.createDecoder(encodedDs))))) + this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(new DSDecoderV1(encodedDs))))) ids.observe(/** @param {YArrayEvent} event */ event => event.changes.added.forEach(item => item.content.getContent().forEach(addClientId)) ) @@ -97,11 +97,11 @@ export class PermanentUserData { user.get('ids').push([clientid]) } }) - const encoder = encoding.createEncoder() + const encoder = new DSEncoderV1() const ds = this.dss.get(userDescription) if (ds) { writeDeleteSet(encoder, ds) - user.get('ds').push([encoding.toUint8Array(encoder)]) + user.get('ds').push([encoder.toUint8Array()]) } } }, 0) @@ -111,9 +111,9 @@ export class PermanentUserData { const yds = user.get('ds') const ds = transaction.deleteSet if (transaction.local && ds.clients.size > 0 && filter(transaction, ds)) { - const encoder = encoding.createEncoder() + const encoder = new DSEncoderV1() writeDeleteSet(encoder, ds) - yds.push([encoding.toUint8Array(encoder)]) + yds.push([encoder.toUint8Array()]) } }) }) diff --git a/src/utils/Snapshot.js b/src/utils/Snapshot.js index 1c13cf6f..5132a554 100644 --- a/src/utils/Snapshot.js +++ b/src/utils/Snapshot.js @@ -12,13 +12,13 @@ import { createDeleteSet, createID, getState, - Transaction, Doc, DeleteSet, Item // eslint-disable-line + AbstractDSDecoder, AbstractDSEncoder, DSEncoderV1, DSEncoderV2, DSDecoderV1, DSDecoderV2, Transaction, Doc, DeleteSet, Item // eslint-disable-line } from '../internals.js' import * as map from 'lib0/map.js' import * as set from 'lib0/set.js' -import * as encoding from 'lib0/encoding.js' import * as decoding from 'lib0/decoding.js' +import { DefaultDSEncoder } from './encoding.js' export class Snapshot { /** @@ -74,23 +74,35 @@ export const equalSnapshots = (snap1, snap2) => { /** * @param {Snapshot} snapshot + * @param {AbstractDSEncoder} [encoder] * @return {Uint8Array} */ -export const encodeSnapshot = snapshot => { - const encoder = encoding.createEncoder() +export const encodeSnapshotV2 = (snapshot, encoder = new DSEncoderV2()) => { writeDeleteSet(encoder, snapshot.ds) writeStateVector(encoder, snapshot.sv) - return encoding.toUint8Array(encoder) + return encoder.toUint8Array() +} + +/** + * @param {Snapshot} snapshot + * @return {Uint8Array} + */ +export const encodeSnapshot = snapshot => encodeSnapshotV2(snapshot, new DefaultDSEncoder()) + +/** + * @param {Uint8Array} buf + * @param {AbstractDSDecoder} [decoder] + * @return {Snapshot} + */ +export const decodeSnapshotV2 = (buf, decoder = new DSDecoderV2(decoding.createDecoder(buf))) => { + return new Snapshot(readDeleteSet(decoder), readStateVector(decoder)) } /** * @param {Uint8Array} buf * @return {Snapshot} */ -export const decodeSnapshot = buf => { - const decoder = decoding.createDecoder(buf) - return new Snapshot(readDeleteSet(decoder), readStateVector(decoder)) -} +export const decodeSnapshot = buf => decodeSnapshotV2(buf, new DSDecoderV1(decoding.createDecoder(buf))) /** * @param {DeleteSet} ds diff --git a/src/utils/StructStore.js b/src/utils/StructStore.js index 572ee17d..f2c7fd8a 100644 --- a/src/utils/StructStore.js +++ b/src/utils/StructStore.js @@ -2,12 +2,11 @@ import { GC, splitItem, - AbstractStruct, Transaction, ID, Item // eslint-disable-line + Transaction, ID, Item, DSDecoderV2 // eslint-disable-line } from '../internals.js' import * as math from 'lib0/math.js' import * as error from 'lib0/error.js' -import * as decoding from 'lib0/decoding.js' // eslint-disable-line export class StructStore { constructor () { @@ -31,7 +30,7 @@ export class StructStore { */ this.pendingStack = [] /** - * @type {Array} + * @type {Array} */ this.pendingDeleteReaders = [] } diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index ea2919b6..6a698d4a 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -11,15 +11,16 @@ import { Item, generateNewClientId, createID, - GC, StructStore, ID, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line + AbstractUpdateEncoder, GC, StructStore, UpdateEncoderV1, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line } from '../internals.js' -import * as encoding from 'lib0/encoding.js' import * as map from 'lib0/map.js' import * as math from 'lib0/math.js' import * as set from 'lib0/set.js' import * as logging from 'lib0/logging.js' import { callAll } from 'lib0/function.js' +import { DefaultUpdateEncoder } from './encoding.js' +import { UpdateEncoderV2 } from './UpdateEncoder.js' /** * A transaction is created for every change on the Yjs model. It is possible @@ -107,17 +108,18 @@ export class Transaction { } /** + * @param {AbstractUpdateEncoder} encoder * @param {Transaction} transaction + * @return {boolean} Whether data was written. */ -export const computeUpdateMessageFromTransaction = transaction => { +export const writeUpdateMessageFromTransaction = (encoder, transaction) => { if (transaction.deleteSet.clients.size === 0 && !map.any(transaction.afterState, (clock, client) => transaction.beforeState.get(client) !== clock)) { - return null + return false } - const encoder = encoding.createEncoder() sortAndMergeDeleteSet(transaction.deleteSet) writeStructsFromTransaction(encoder, transaction) writeDeleteSet(encoder, transaction.deleteSet) - return encoder + return true } /** @@ -322,9 +324,17 @@ const cleanupTransactions = (transactionCleanups, i) => { // @todo Merge all the transactions into one and provide send the data as a single update message doc.emit('afterTransactionCleanup', [transaction, doc]) if (doc._observers.has('update')) { - const updateMessage = computeUpdateMessageFromTransaction(transaction) - if (updateMessage !== null) { - doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc]) + const encoder = new DefaultUpdateEncoder() + const hasContent = writeUpdateMessageFromTransaction(encoder, transaction) + if (hasContent) { + doc.emit('update', [encoder.toUint8Array(), transaction.origin, doc]) + } + } + if (doc._observers.has('updateV2')) { + const encoder = new UpdateEncoderV2() + const hasContent = writeUpdateMessageFromTransaction(encoder, transaction) + if (hasContent) { + doc.emit('updateV2', [encoder.toUint8Array(), transaction.origin, doc]) } } if (transactionCleanups.length <= i + 1) { diff --git a/src/utils/UpdateDecoder.js b/src/utils/UpdateDecoder.js new file mode 100644 index 00000000..372c7749 --- /dev/null +++ b/src/utils/UpdateDecoder.js @@ -0,0 +1,392 @@ +import * as buffer from 'lib0/buffer.js' +import * as error from 'lib0/error.js' +import * as decoding from 'lib0/decoding.js' +import { + ID, createID +} from '../internals.js' + +export class AbstractDSDecoder { + /** + * @param {decoding.Decoder} decoder + */ + constructor (decoder) { + this.restDecoder = decoder + error.methodUnimplemented() + } + + resetDsCurVal () { } + + /** + * @return {number} + */ + readDsClock () { + error.methodUnimplemented() + } + + /** + * @return {number} + */ + readDsLen () { + error.methodUnimplemented() + } +} + +export class AbstractUpdateDecoder extends AbstractDSDecoder { + /** + * @return {ID} + */ + readLeftID () { + error.methodUnimplemented() + } + + /** + * @return {ID} + */ + readRightID () { + error.methodUnimplemented() + } + + /** + * Read the next client id. + * Use this in favor of readID whenever possible to reduce the number of objects created. + * + * @return {number} + */ + readClient () { + error.methodUnimplemented() + } + + /** + * @return {number} info An unsigned 8-bit integer + */ + readInfo () { + error.methodUnimplemented() + } + + /** + * @return {string} + */ + readString () { + error.methodUnimplemented() + } + + /** + * @return {boolean} isKey + */ + readParentInfo () { + error.methodUnimplemented() + } + + /** + * @return {number} info An unsigned 8-bit integer + */ + readTypeRef () { + error.methodUnimplemented() + } + + /** + * Write len of a struct - well suited for Opt RLE encoder. + * + * @return {number} len + */ + readLen () { + error.methodUnimplemented() + } + + /** + * @return {any} + */ + readAny () { + error.methodUnimplemented() + } + + /** + * @return {Uint8Array} + */ + readBuf () { + error.methodUnimplemented() + } + + /** + * Legacy implementation uses JSON parse. We use any-decoding in v2. + * + * @return {any} + */ + readJSON () { + error.methodUnimplemented() + } + + /** + * @return {string} + */ + readKey () { + error.methodUnimplemented() + } +} + +export class DSDecoderV1 { + /** + * @param {decoding.Decoder} decoder + */ + constructor (decoder) { + this.restDecoder = decoder + } + + resetDsCurVal () { + // nop + } + + /** + * @return {number} + */ + readDsClock () { + return decoding.readVarUint(this.restDecoder) + } + + /** + * @return {number} + */ + readDsLen () { + return decoding.readVarUint(this.restDecoder) + } +} + +export class UpdateDecoderV1 extends DSDecoderV1 { + /** + * @return {ID} + */ + readLeftID () { + return createID(decoding.readVarUint(this.restDecoder), decoding.readVarUint(this.restDecoder)) + } + + /** + * @return {ID} + */ + readRightID () { + return createID(decoding.readVarUint(this.restDecoder), decoding.readVarUint(this.restDecoder)) + } + + /** + * Read the next client id. + * Use this in favor of readID whenever possible to reduce the number of objects created. + */ + readClient () { + return decoding.readVarUint(this.restDecoder) + } + + /** + * @return {number} info An unsigned 8-bit integer + */ + readInfo () { + return decoding.readUint8(this.restDecoder) + } + + /** + * @return {string} + */ + readString () { + return decoding.readVarString(this.restDecoder) + } + + /** + * @return {boolean} isKey + */ + readParentInfo () { + return decoding.readVarUint(this.restDecoder) === 1 + } + + /** + * @return {number} info An unsigned 8-bit integer + */ + readTypeRef () { + return decoding.readVarUint(this.restDecoder) + } + + /** + * Write len of a struct - well suited for Opt RLE encoder. + * + * @return {number} len + */ + readLen () { + return decoding.readVarUint(this.restDecoder) + } + + /** + * @return {any} + */ + readAny () { + return decoding.readAny(this.restDecoder) + } + + /** + * @return {Uint8Array} + */ + readBuf () { + return buffer.copyUint8Array(decoding.readVarUint8Array(this.restDecoder)) + } + + /** + * Legacy implementation uses JSON parse. We use any-decoding in v2. + * + * @return {any} + */ + readJSON () { + return JSON.parse(decoding.readVarString(this.restDecoder)) + } + + /** + * @return {string} + */ + readKey () { + return decoding.readVarString(this.restDecoder) + } +} + +export class DSDecoderV2 { + /** + * @param {decoding.Decoder} decoder + */ + constructor (decoder) { + this.dsCurrVal = 0 + this.restDecoder = decoder + } + + resetDsCurVal () { + this.dsCurrVal = 0 + } + + readDsClock () { + this.dsCurrVal += decoding.readVarUint(this.restDecoder) + return this.dsCurrVal + } + + readDsLen () { + const diff = decoding.readVarUint(this.restDecoder) + 1 + this.dsCurrVal += diff + return diff + } +} + +export class UpdateDecoderV2 extends DSDecoderV2 { + /** + * @param {decoding.Decoder} decoder + */ + constructor (decoder) { + super(decoder) + /** + * List of cached keys. If the keys[id] does not exist, we read a new key + * from stringEncoder and push it to keys. + * + * @type {Array} + */ + this.keys = [] + decoding.readUint8(decoder) // read feature flag - currently unused + this.keyClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder)) + this.clientDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder)) + this.leftClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder)) + this.rightClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder)) + this.infoDecoder = new decoding.RleDecoder(decoding.readVarUint8Array(decoder), decoding.readUint8) + this.stringDecoder = new decoding.StringDecoder(decoding.readVarUint8Array(decoder)) + this.parentInfoDecoder = new decoding.RleDecoder(decoding.readVarUint8Array(decoder), decoding.readUint8) + this.typeRefDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder)) + this.lenDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder)) + } + + /** + * @return {ID} + */ + readLeftID () { + return new ID(this.clientDecoder.read(), this.leftClockDecoder.read()) + } + + /** + * @return {ID} + */ + readRightID () { + return new ID(this.clientDecoder.read(), this.rightClockDecoder.read()) + } + + /** + * Read the next client id. + * Use this in favor of readID whenever possible to reduce the number of objects created. + */ + readClient () { + return this.clientDecoder.read() + } + + /** + * @return {number} info An unsigned 8-bit integer + */ + readInfo () { + return /** @type {number} */ (this.infoDecoder.read()) + } + + /** + * @return {string} + */ + readString () { + return this.stringDecoder.read() + } + + /** + * @return {boolean} + */ + readParentInfo () { + return this.parentInfoDecoder.read() === 1 + } + + /** + * @return {number} An unsigned 8-bit integer + */ + readTypeRef () { + return this.typeRefDecoder.read() + } + + /** + * Write len of a struct - well suited for Opt RLE encoder. + * + * @return {number} + */ + readLen () { + return this.lenDecoder.read() + } + + /** + * @return {any} + */ + readAny () { + return decoding.readAny(this.restDecoder) + } + + /** + * @return {Uint8Array} + */ + readBuf () { + return decoding.readVarUint8Array(this.restDecoder) + } + + /** + * This is mainly here for legacy purposes. + * + * Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder. + * + * @return {any} + */ + readJSON () { + return decoding.readAny(this.restDecoder) + } + + /** + * @return {string} + */ + readKey () { + const keyClock = this.keyClockDecoder.read() + if (keyClock < this.keys.length) { + return this.keys[keyClock] + } else { + const key = this.stringDecoder.read() + this.keys.push(key) + return key + } + } +} diff --git a/src/utils/UpdateEncoder.js b/src/utils/UpdateEncoder.js new file mode 100644 index 00000000..eb13fdc3 --- /dev/null +++ b/src/utils/UpdateEncoder.js @@ -0,0 +1,408 @@ + +import * as error from 'lib0/error.js' +import * as encoding from 'lib0/encoding.js' + +import { + ID // eslint-disable-line +} from '../internals.js' + +export class AbstractDSEncoder { + constructor () { + this.restEncoder = encoding.createEncoder() + } + + /** + * @return {Uint8Array} + */ + toUint8Array () { + error.methodUnimplemented() + } + + /** + * Resets the ds value to 0. + * The v2 encoder uses this information to reset the initial diff value. + */ + resetDsCurVal () { } + + /** + * @param {number} clock + */ + writeDsClock (clock) { } + + /** + * @param {number} len + */ + writeDsLen (len) { } +} + +export class AbstractUpdateEncoder extends AbstractDSEncoder { + /** + * @return {Uint8Array} + */ + toUint8Array () { + error.methodUnimplemented() + } + + /** + * @param {ID} id + */ + writeLeftID (id) { } + + /** + * @param {ID} id + */ + writeRightID (id) { } + + /** + * Use writeClient and writeClock instead of writeID if possible. + * @param {number} client + */ + writeClient (client) { } + + /** + * @param {number} info An unsigned 8-bit integer + */ + writeInfo (info) { } + + /** + * @param {string} s + */ + writeString (s) { } + + /** + * @param {boolean} isYKey + */ + writeParentInfo (isYKey) { } + + /** + * @param {number} info An unsigned 8-bit integer + */ + writeTypeRef (info) { } + + /** + * Write len of a struct - well suited for Opt RLE encoder. + * + * @param {number} len + */ + writeLen (len) { } + + /** + * @param {any} any + */ + writeAny (any) { } + + /** + * @param {Uint8Array} buf + */ + writeBuf (buf) { } + + /** + * @param {any} embed + */ + writeJSON (embed) { } + + /** + * @param {string} key + */ + writeKey (key) { } +} + +export class DSEncoderV1 { + constructor () { + this.restEncoder = new encoding.Encoder() + } + + toUint8Array () { + return encoding.toUint8Array(this.restEncoder) + } + + resetDsCurVal () { + // nop + } + + /** + * @param {number} clock + */ + writeDsClock (clock) { + encoding.writeVarUint(this.restEncoder, clock) + } + + /** + * @param {number} len + */ + writeDsLen (len) { + encoding.writeVarUint(this.restEncoder, len) + } +} + +export class UpdateEncoderV1 extends DSEncoderV1 { + /** + * @param {ID} id + */ + writeLeftID (id) { + encoding.writeVarUint(this.restEncoder, id.client) + encoding.writeVarUint(this.restEncoder, id.clock) + } + + /** + * @param {ID} id + */ + writeRightID (id) { + encoding.writeVarUint(this.restEncoder, id.client) + encoding.writeVarUint(this.restEncoder, id.clock) + } + + /** + * Use writeClient and writeClock instead of writeID if possible. + * @param {number} client + */ + writeClient (client) { + encoding.writeVarUint(this.restEncoder, client) + } + + /** + * @param {number} info An unsigned 8-bit integer + */ + writeInfo (info) { + encoding.writeUint8(this.restEncoder, info) + } + + /** + * @param {string} s + */ + writeString (s) { + encoding.writeVarString(this.restEncoder, s) + } + + /** + * @param {boolean} isYKey + */ + writeParentInfo (isYKey) { + encoding.writeVarUint(this.restEncoder, isYKey ? 1 : 0) + } + + /** + * @param {number} info An unsigned 8-bit integer + */ + writeTypeRef (info) { + encoding.writeVarUint(this.restEncoder, info) + } + + /** + * Write len of a struct - well suited for Opt RLE encoder. + * + * @param {number} len + */ + writeLen (len) { + encoding.writeVarUint(this.restEncoder, len) + } + + /** + * @param {any} any + */ + writeAny (any) { + encoding.writeAny(this.restEncoder, any) + } + + /** + * @param {Uint8Array} buf + */ + writeBuf (buf) { + encoding.writeVarUint8Array(this.restEncoder, buf) + } + + /** + * @param {any} embed + */ + writeJSON (embed) { + encoding.writeVarString(this.restEncoder, JSON.stringify(embed)) + } + + /** + * @param {string} key + */ + writeKey (key) { + encoding.writeVarString(this.restEncoder, key) + } +} + +export class DSEncoderV2 { + constructor () { + this.restEncoder = new encoding.Encoder() // encodes all the rest / non-optimized + this.dsCurrVal = 0 + } + + toUint8Array () { + return encoding.toUint8Array(this.restEncoder) + } + + resetDsCurVal () { + this.dsCurrVal = 0 + } + + /** + * @param {number} clock + */ + writeDsClock (clock) { + const diff = clock - this.dsCurrVal + this.dsCurrVal = clock + encoding.writeVarUint(this.restEncoder, diff) + } + + /** + * @param {number} len + */ + writeDsLen (len) { + if (len === 0) { + error.unexpectedCase() + } + encoding.writeVarUint(this.restEncoder, len - 1) + this.dsCurrVal += len + } +} + +export class UpdateEncoderV2 extends DSEncoderV2 { + constructor () { + super() + /** + * @type {Map} + */ + this.keyMap = new Map() + /** + * Refers to the next uniqe key-identifier to me used. + * See writeKey method for more information. + * + * @type {number} + */ + this.keyClock = 0 + this.keyClockEncoder = new encoding.IntDiffOptRleEncoder() + this.clientEncoder = new encoding.UintOptRleEncoder() + this.leftClockEncoder = new encoding.IntDiffOptRleEncoder() + this.rightClockEncoder = new encoding.IntDiffOptRleEncoder() + this.infoEncoder = new encoding.RleEncoder(encoding.writeUint8) + this.stringEncoder = new encoding.StringEncoder() + this.parentInfoEncoder = new encoding.RleEncoder(encoding.writeUint8) + this.typeRefEncoder = new encoding.UintOptRleEncoder() + this.lenEncoder = new encoding.UintOptRleEncoder() + } + + toUint8Array () { + const encoder = encoding.createEncoder() + encoding.writeUint8(encoder, 0) // this is a feature flag that we might use in the future + encoding.writeVarUint8Array(encoder, this.keyClockEncoder.toUint8Array()) + encoding.writeVarUint8Array(encoder, this.clientEncoder.toUint8Array()) + encoding.writeVarUint8Array(encoder, this.leftClockEncoder.toUint8Array()) + encoding.writeVarUint8Array(encoder, this.rightClockEncoder.toUint8Array()) + encoding.writeVarUint8Array(encoder, encoding.toUint8Array(this.infoEncoder)) + encoding.writeVarUint8Array(encoder, this.stringEncoder.toUint8Array()) + encoding.writeVarUint8Array(encoder, encoding.toUint8Array(this.parentInfoEncoder)) + encoding.writeVarUint8Array(encoder, this.typeRefEncoder.toUint8Array()) + encoding.writeVarUint8Array(encoder, this.lenEncoder.toUint8Array()) + // @note The rest encoder is appended! (note the missing var) + encoding.writeUint8Array(encoder, encoding.toUint8Array(this.restEncoder)) + return encoding.toUint8Array(encoder) + } + + /** + * @param {ID} id + */ + writeLeftID (id) { + this.clientEncoder.write(id.client) + this.leftClockEncoder.write(id.clock) + } + + /** + * @param {ID} id + */ + writeRightID (id) { + this.clientEncoder.write(id.client) + this.rightClockEncoder.write(id.clock) + } + + /** + * @param {number} client + */ + writeClient (client) { + this.clientEncoder.write(client) + } + + /** + * @param {number} info An unsigned 8-bit integer + */ + writeInfo (info) { + this.infoEncoder.write(info) + } + + /** + * @param {string} s + */ + writeString (s) { + this.stringEncoder.write(s) + } + + /** + * @param {boolean} isYKey + */ + writeParentInfo (isYKey) { + this.parentInfoEncoder.write(isYKey ? 1 : 0) + } + + /** + * @param {number} info An unsigned 8-bit integer + */ + writeTypeRef (info) { + this.typeRefEncoder.write(info) + } + + /** + * Write len of a struct - well suited for Opt RLE encoder. + * + * @param {number} len + */ + writeLen (len) { + this.lenEncoder.write(len) + } + + /** + * @param {any} any + */ + writeAny (any) { + encoding.writeAny(this.restEncoder, any) + } + + /** + * @param {Uint8Array} buf + */ + writeBuf (buf) { + encoding.writeVarUint8Array(this.restEncoder, buf) + } + + /** + * This is mainly here for legacy purposes. + * + * Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder. + * + * @param {any} embed + */ + writeJSON (embed) { + encoding.writeAny(this.restEncoder, embed) + } + + /** + * Property keys are often reused. For example, in y-prosemirror the key `bold` might + * occur very often. For a 3d application, the key `position` might occur very often. + * + * We cache these keys in a Map and refer to them via a unique number. + * + * @param {string} key + */ + writeKey (key) { + const clock = this.keyMap.get(key) + if (clock === undefined) { + this.keyClockEncoder.write(this.keyClock++) + this.stringEncoder.write(key) + } else { + this.keyClockEncoder.write(this.keyClock++) + } + } +} diff --git a/src/utils/encoding.js b/src/utils/encoding.js index ff9df452..bc3fed33 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -1,7 +1,8 @@ /** * @module encoding - * + */ +/* * We use the first five bits in the info flag for determining the type of the struct. * * 0: GC @@ -16,8 +17,6 @@ import { findIndexSS, - writeID, - readID, getState, createID, getStateVector, @@ -25,16 +24,36 @@ import { writeDeleteSet, createDeleteSetFromStructStore, transact, - readItem, - Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line + readItemContent, + UpdateDecoderV1, + UpdateDecoderV2, + UpdateEncoderV1, + UpdateEncoderV2, + DSDecoderV2, + DSEncoderV2, + DSDecoderV1, + DSEncoderV1, + AbstractDSEncoder, AbstractDSDecoder, AbstractUpdateEncoder, AbstractUpdateDecoder, AbstractContent, Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line } from '../internals.js' import * as encoding from 'lib0/encoding.js' import * as decoding from 'lib0/decoding.js' import * as binary from 'lib0/binary.js' +export let DefaultDSEncoder = DSEncoderV1 +export let DefaultDSDecoder = DSDecoderV1 +export let DefaultUpdateEncoder = UpdateEncoderV1 +export let DefaultUpdateDecoder = UpdateDecoderV1 + +export const useV2Encoding = () => { + DefaultDSEncoder = DSEncoderV2 + DefaultDSDecoder = DSDecoderV2 + DefaultUpdateEncoder = UpdateEncoderV2 + DefaultUpdateDecoder = UpdateDecoderV2 +} + /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {Array} structs All structs by `client` * @param {number} client * @param {number} clock write structs starting with `ID(client,clock)` @@ -45,8 +64,9 @@ const writeStructs = (encoder, structs, client, clock) => { // write first id const startNewStructs = findIndexSS(structs, clock) // write # encoded structs - encoding.writeVarUint(encoder, structs.length - startNewStructs) - writeID(encoder, createID(client, clock)) + encoding.writeVarUint(encoder.restEncoder, structs.length - startNewStructs) + encoder.writeClient(client) + encoding.writeVarUint(encoder.restEncoder, clock) const firstStruct = structs[startNewStructs] // write first struct with an offset firstStruct.write(encoder, clock - firstStruct.id.clock) @@ -56,7 +76,7 @@ const writeStructs = (encoder, structs, client, clock) => { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {StructStore} store * @param {Map} _sm * @@ -78,7 +98,7 @@ export const writeClientsStructs = (encoder, store, _sm) => { } }) // write # states that were updated - encoding.writeVarUint(encoder, sm.size) + encoding.writeVarUint(encoder.restEncoder, sm.size) // Write items with higher client ids first // This heavily improves the conflict algorithm. Array.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => { @@ -88,7 +108,7 @@ export const writeClientsStructs = (encoder, store, _sm) => { } /** - * @param {decoding.Decoder} decoder The decoder object to read data from. + * @param {AbstractUpdateDecoder} decoder The decoder object to read data from. * @param {Map>} clientRefs * @param {Doc} doc * @return {Map>} @@ -97,21 +117,52 @@ export const writeClientsStructs = (encoder, store, _sm) => { * @function */ export const readClientsStructRefs = (decoder, clientRefs, doc) => { - const numOfStateUpdates = decoding.readVarUint(decoder) + const numOfStateUpdates = decoding.readVarUint(decoder.restDecoder) for (let i = 0; i < numOfStateUpdates; i++) { - const numberOfStructs = decoding.readVarUint(decoder) + const numberOfStructs = decoding.readVarUint(decoder.restDecoder) /** * @type {Array} */ const refs = [] - let { client, clock } = readID(decoder) - let info, struct + const client = decoder.readClient() + let clock = decoding.readVarUint(decoder.restDecoder) clientRefs.set(client, refs) for (let i = 0; i < numberOfStructs; i++) { - info = decoding.readUint8(decoder) - struct = (binary.BITS5 & info) === 0 ? new GC(createID(client, clock), decoding.readVarUint(decoder)) : readItem(decoder, createID(client, clock), info, doc) - refs.push(struct) - clock += struct.length + const info = decoder.readInfo() + if ((binary.BITS5 & info) !== 0) { + /** + * The item that was originally to the left of this item. + * @type {ID | null} + */ + const origin = (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null + /** + * The item that was originally to the right of this item. + * @type {ID | null} + */ + const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null + const canCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0 + const hasParentYKey = canCopyParentInfo ? decoder.readParentInfo() : false + /** + * If parent = null and neither left nor right are defined, then we know that `parent` is child of `y` + * and we read the next string as parentYKey. + * It indicates how we store/retrieve parent from `y.share` + * @type {string|null} + */ + const parentYKey = canCopyParentInfo && hasParentYKey ? decoder.readString() : null + + const struct = new Item( + createID(client, clock), null, origin, null, rightOrigin, + canCopyParentInfo && !hasParentYKey ? decoder.readLeftID() : (parentYKey ? doc.get(parentYKey) : null), // parent + canCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub + /** @type {AbstractContent} */ (readItemContent(decoder, info)) // item content + ) + refs.push(struct) + clock += struct.length + } else { + const len = decoder.readLen() + refs.push(new GC(createID(client, clock), len)) + clock += len + } } } return clientRefs @@ -222,7 +273,7 @@ export const tryResumePendingDeleteReaders = (transaction, store) => { } /** - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {Transaction} transaction * * @private @@ -275,7 +326,7 @@ const cleanupPendingStructs = pendingClientsStructRefs => { * * This is called when data is received from a remote peer. * - * @param {decoding.Decoder} decoder The decoder object to read data from. + * @param {AbstractUpdateDecoder} decoder The decoder object to read data from. * @param {Transaction} transaction * @param {StructStore} store * @@ -299,15 +350,46 @@ export const readStructs = (decoder, transaction, store) => { * @param {decoding.Decoder} decoder * @param {Doc} ydoc * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))` + * @param {AbstractUpdateDecoder} [structDecoder] * * @function */ -export const readUpdate = (decoder, ydoc, transactionOrigin) => +export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = new UpdateDecoderV2(decoder)) => transact(ydoc, transaction => { - readStructs(decoder, transaction, ydoc.store) - readAndApplyDeleteSet(decoder, transaction, ydoc.store) + readStructs(structDecoder, transaction, ydoc.store) + readAndApplyDeleteSet(structDecoder, transaction, ydoc.store) }, transactionOrigin, false) +/** + * Read and apply a document update. + * + * This function has the same effect as `applyUpdate` but accepts an decoder. + * + * @param {decoding.Decoder} decoder + * @param {Doc} ydoc + * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))` + * + * @function + */ +export const readUpdate = (decoder, ydoc, transactionOrigin) => readUpdateV2(decoder, ydoc, transactionOrigin, new DefaultUpdateDecoder(decoder)) + +/** + * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`. + * + * This function has the same effect as `readUpdate` but accepts an Uint8Array instead of a Decoder. + * + * @param {Doc} ydoc + * @param {Uint8Array} update + * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))` + * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder] + * + * @function + */ +export const applyUpdateV2 = (ydoc, update, transactionOrigin, YDecoder = UpdateDecoderV2) => { + const decoder = decoding.createDecoder(update) + readUpdateV2(decoder, ydoc, transactionOrigin, new YDecoder(decoder)) +} + /** * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`. * @@ -319,14 +401,13 @@ export const readUpdate = (decoder, ydoc, transactionOrigin) => * * @function */ -export const applyUpdate = (ydoc, update, transactionOrigin) => - readUpdate(decoding.createDecoder(update), ydoc, transactionOrigin) +export const applyUpdate = (ydoc, update, transactionOrigin) => applyUpdateV2(ydoc, update, transactionOrigin, DefaultUpdateDecoder) /** * Write all the document as a single update message. If you specify the state of the remote client (`targetStateVector`) it will * only write the operations that are missing. * - * @param {encoding.Encoder} encoder + * @param {AbstractUpdateEncoder} encoder * @param {Doc} doc * @param {Map} [targetStateVector] The state of the target that receives the update. Leave empty to write all known structs * @@ -345,31 +426,45 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map()) * * @param {Doc} doc * @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs + * @param {AbstractUpdateEncoder} [encoder] * @return {Uint8Array} * * @function */ -export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => { - const encoder = encoding.createEncoder() +export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => { const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector) writeStateAsUpdate(encoder, doc, targetStateVector) - return encoding.toUint8Array(encoder) + return encoder.toUint8Array() } +/** + * Write all the document as a single update message that can be applied on the remote document. If you specify the state of the remote client (`targetState`) it will + * only write the operations that are missing. + * + * Use `writeStateAsUpdate` instead if you are working with lib0/encoding.js#Encoder + * + * @param {Doc} doc + * @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs + * @return {Uint8Array} + * + * @function + */ +export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => encodeStateAsUpdateV2(doc, encodedTargetStateVector, new DefaultUpdateEncoder()) + /** * Read state vector from Decoder and return as Map * - * @param {decoding.Decoder} decoder + * @param {AbstractDSDecoder} decoder * @return {Map} Maps `client` to the number next expected `clock` from that client. * * @function */ export const readStateVector = decoder => { const ss = new Map() - const ssLength = decoding.readVarUint(decoder) + const ssLength = decoding.readVarUint(decoder.restDecoder) for (let i = 0; i < ssLength; i++) { - const client = decoding.readVarUint(decoder) - const clock = decoding.readVarUint(decoder) + const client = decoding.readVarUint(decoder.restDecoder) + const clock = decoding.readVarUint(decoder.restDecoder) ss.set(client, clock) } return ss @@ -383,28 +478,34 @@ export const readStateVector = decoder => { * * @function */ -export const decodeStateVector = decodedState => readStateVector(decoding.createDecoder(decodedState)) +export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState))) /** - * Write State Vector to `lib0/encoding.js#Encoder`. + * Read decodedState and return State as Map. * - * @param {encoding.Encoder} encoder + * @param {Uint8Array} decodedState + * @return {Map} Maps `client` to the number next expected `clock` from that client. + * + * @function + */ +export const decodeStateVector = decodedState => readStateVector(new DefaultDSDecoder(decoding.createDecoder(decodedState))) + +/** + * @param {AbstractDSEncoder} encoder * @param {Map} sv * @function */ export const writeStateVector = (encoder, sv) => { - encoding.writeVarUint(encoder, sv.size) + encoding.writeVarUint(encoder.restEncoder, sv.size) sv.forEach((clock, client) => { - encoding.writeVarUint(encoder, client) - encoding.writeVarUint(encoder, clock) + encoding.writeVarUint(encoder.restEncoder, client) // @todo use a special client decoder that is based on mapping + encoding.writeVarUint(encoder.restEncoder, clock) }) return encoder } /** - * Write State Vector to `lib0/encoding.js#Encoder`. - * - * @param {encoding.Encoder} encoder + * @param {AbstractDSEncoder} encoder * @param {Doc} doc * * @function @@ -415,12 +516,22 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod * Encode State as Uint8Array. * * @param {Doc} doc + * @param {AbstractDSEncoder} [encoder] * @return {Uint8Array} * * @function */ -export const encodeStateVector = doc => { - const encoder = encoding.createEncoder() +export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => { writeDocumentStateVector(encoder, doc) - return encoding.toUint8Array(encoder) + return encoder.toUint8Array() } + +/** + * Encode State as Uint8Array. + * + * @param {Doc} doc + * @return {Uint8Array} + * + * @function + */ +export const encodeStateVector = doc => encodeStateVectorV2(doc, new DefaultDSEncoder()) diff --git a/tests/testHelper.js b/tests/testHelper.js index 3e669b5f..2058fd6a 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -394,21 +394,21 @@ export const applyRandomTests = (tc, mods, iterations, initTestObject) => { const result = init(tc, { users: 5 }, initTestObject) const { testConnector, users } = result for (let i = 0; i < iterations; i++) { - if (prng.int31(gen, 0, 100) <= 2) { + if (prng.int32(gen, 0, 100) <= 2) { // 2% chance to disconnect/reconnect a random user if (prng.bool(gen)) { testConnector.disconnectRandom() } else { testConnector.reconnectRandom() } - } else if (prng.int31(gen, 0, 100) <= 1) { + } else if (prng.int32(gen, 0, 100) <= 1) { // 1% chance to flush all testConnector.flushAllMessages() - } else if (prng.int31(gen, 0, 100) <= 50) { + } else if (prng.int32(gen, 0, 100) <= 50) { // 50% chance to flush a random message testConnector.flushRandomMessage() } - const user = prng.int31(gen, 0, users.length - 1) + const user = prng.int32(gen, 0, users.length - 1) const test = prng.oneOf(gen, mods) test(users[user], gen, result.testObjects[user]) } diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index 1822e104..e06a8a60 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -5,6 +5,18 @@ import * as t from 'lib0/testing.js' import * as prng from 'lib0/prng.js' import * as math from 'lib0/math.js' +/** + * @param {t.TestCase} tc + */ +export const testBasicUpdate = tc => { + const doc1 = new Y.Doc() + const doc2 = new Y.Doc() + doc1.getArray('array').insert(0, ['hi']) + const update = Y.encodeStateAsUpdate(doc1) + Y.applyUpdate(doc2, update) + t.compare(doc2.getArray('array').toArray(), ['hi']) +} + /** * @param {t.TestCase} tc */ @@ -335,23 +347,23 @@ const arrayTransactions = [ const yarray = user.getArray('array') var uniqueNumber = getUniqueNumber() var content = [] - var len = prng.int31(gen, 1, 4) + var len = prng.int32(gen, 1, 4) for (var i = 0; i < len; i++) { content.push(uniqueNumber) } - var pos = prng.int31(gen, 0, yarray.length) + var pos = prng.int32(gen, 0, yarray.length) yarray.insert(pos, content) }, function insertTypeArray (user, gen) { const yarray = user.getArray('array') - var pos = prng.int31(gen, 0, yarray.length) + var pos = prng.int32(gen, 0, yarray.length) yarray.insert(pos, [new Y.Array()]) var array2 = yarray.get(pos) array2.insert(0, [1, 2, 3, 4]) }, function insertTypeMap (user, gen) { const yarray = user.getArray('array') - var pos = prng.int31(gen, 0, yarray.length) + var pos = prng.int32(gen, 0, yarray.length) yarray.insert(pos, [new Y.Map()]) var map = yarray.get(pos) map.set('someprop', 42) @@ -362,13 +374,13 @@ const arrayTransactions = [ const yarray = user.getArray('array') var length = yarray.length if (length > 0) { - var somePos = prng.int31(gen, 0, length - 1) - var delLength = prng.int31(gen, 1, math.min(2, length - somePos)) + var somePos = prng.int32(gen, 0, length - 1) + var delLength = prng.int32(gen, 1, math.min(2, length - somePos)) if (prng.bool(gen)) { var type = yarray.get(somePos) if (type.length > 0) { - somePos = prng.int31(gen, 0, type.length - 1) - delLength = prng.int31(gen, 0, math.min(2, type.length - somePos)) + somePos = prng.int32(gen, 0, type.length - 1) + delLength = prng.int32(gen, 0, math.min(2, type.length - somePos)) type.delete(somePos, delLength) } } else { diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index 2230131b..5e7ea573 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -215,7 +215,7 @@ const tryGc = () => { * @param {t.TestCase} tc */ export const testLargeFragmentedDocument = tc => { - const itemsToInsert = 2000000 + const itemsToInsert = 1000000 let update = /** @type {any} */ (null) ;(() => { const doc1 = new Y.Doc() @@ -230,14 +230,15 @@ export const testLargeFragmentedDocument = tc => { }) tryGc() t.measureTime('time to encode document', () => { - update = Y.encodeStateAsUpdate(doc1) + update = Y.encodeStateAsUpdateV2(doc1) }) + t.describe('Document size:', update.byteLength) })() ;(() => { const doc2 = new Y.Doc() tryGc() t.measureTime(`time to apply ${itemsToInsert} updates`, () => { - Y.applyUpdate(doc2, update) + Y.applyUpdateV2(doc2, update) }) })() }