diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..3e8090b1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: dmonad +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.jsdoc.json b/.jsdoc.json index 1cca0d93..86397dac 100644 --- a/.jsdoc.json +++ b/.jsdoc.json @@ -17,7 +17,7 @@ "useCollapsibles": true, "collapse": true, "resources": { - "y-js.org": "yjs.website" + "yjs.dev": "Yjs website" }, "logo": { "url": "https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png", diff --git a/README.md b/README.md index 42e54230..39d71354 100644 --- a/README.md +++ b/README.md @@ -1,305 +1,856 @@ -#  +#  -Yjs is a framework for offline-first p2p shared editing on structured data like -text, richtext, json, or XML. It is fairly easy to get started, as Yjs hides -most of the complexity of concurrent editing. For additional information, demos, -and tutorials visit [y-js.org](http://y-js.org/). +> A CRDT framework with a powerful abstraction of shared data -:warning: Checkout the [v13 docs](./README.v13.md) for the upcoming release :warning: +Yjs is a [CRDT implementation](#Yjs-CRDT-Algorithm) that exposes its internal +data structure as *shared types*. Shared types are common data types like `Map` +or `Array` with superpowers: changes are automatically distributed to other +peers and merged without merge conflicts. -### Extensions -Yjs only knows how to resolve conflicts on shared data. You have to choose a .. +Yjs is **network agnostic** (p2p!), supports many existing **rich text +editors**, **offline editing**, **version snapshots**, **undo/redo** and +**shared cursors**. It scales well with an unlimited number of users and is well +suited for even large documents. -* *Connector* - a communication protocol that propagates changes to the clients -* *Database* - a database to store your changes -* one or more *Types* - that represent the shared data +* Demos: [https://github.com/yjs/yjs-demos](https://github.com/yjs/yjs-demos) +* Discuss: [https://discuss.yjs.dev](https://discuss.yjs.dev) +* Benchmarks: + [https://github.com/dmonad/crdt-benchmarks](https://github.com/dmonad/crdt-benchmarks) -Connectors, Databases, and Types are available as modules that extend Yjs. Here -is a list of the modules we know of: +:warning: This is the documentation for v13 (still in alpha). For the stable v12 +release checkout the [v12 docs](./README.v12.md) :warning: -##### Connectors +## Table of Contents -|Name | Description | -|----------------|-----------------------------------| -|[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC| -|[websockets](https://github.com/y-js/y-websockets-client) | Set up [a central server](https://github.com/y-js/y-websockets-client), and connect to it via websockets | -|[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))| -|[ipfs](https://github.com/ipfs-labs/y-ipfs-connector) | Connector for the [Interplanetary File System](https://ipfs.io/)!| -|[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios| +* [Overview](#Overview) + * [Bindings](#Bindings) + * [Providers](#Providers) +* [Getting Started](#Getting-Started) +* [API](#API) + * [Shared Types](#Shared-Types) + * [Y.Doc](#YDoc) + * [Document Updates](#Document-Updates) + * [Relative Positions](#Relative-Positions) + * [Y.UndoManager](#YUndoManager) +* [Miscellaneous](#Miscellaneous) + * [Typescript Declarations](#Typescript-Declarations) +* [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm) +* [Evaluation](#Evaluation) + * [Existing shared editing libraries](#Exisisting-Javascript-Libraries) + * [CRDT Algorithms](#CRDT-Algorithms) + * [Comparison of CRDT with OT](#Comparing-CRDT-with-OT) + * [Comparison of CRDT Algorithms](#Comparing-CRDT-Algorithms) + * [Comparison of Yjs with other Implementations](#Comparing-Yjs-with-other-Implementations) +* [License and Author](#License-and-Author) -##### Database adapters +## Overview -|Name | Description | -|----------------|-----------------------------------| -|[memory](https://github.com/y-js/y-memory) | In-memory storage. | -|[indexeddb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | -|[leveldb](https://github.com/y-js/y-leveldb) | Persistent storage for node apps | +This repository contains a collection of shared types that can be observed for +changes and manipulated concurrently. Network functionality and two-way-bindings +are implemented in separate modules. -##### Types +### Bindings -| Name | Description | -|----------|-------------------| -|[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | -|[array](https://github.com/y-js/y-array) | A shared Array implementation | -|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | -|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to the [Ace Editor](https://ace.c9.io), [CodeMirror](https://codemirror.net/), [Monaco](https://github.com/Microsoft/monaco-editor), textareas, input elements, and HTML elements (e.g. <*h1*>, or <*p*>) | -|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| +| Name | Cursors | Binding | Demo | +|---|:-:|---|---| +| [ProseMirror](https://prosemirror.net/) | ✔ | [y-prosemirror](http://github.com/yjs/y-prosemirror) | [demo](https://yjs-demos.now.sh/prosemirror/) | +| [Quill](https://quilljs.com/) | ✔ | [y-quill](http://github.com/yjs/y-quill) | [demo](https://yjs-demos.now.sh/quill/) | +| [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/yjs/y-codemirror) | [demo](https://yjs-demos.now.sh/codemirror/) | +| [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/yjs/y-monaco) | [demo](https://yjs-demos.now.sh/monaco/) | +| [Ace](https://ace.c9.io/) | | [y-ace](http://github.com/yjs/y-ace) | [demo](https://yjs-demos.now.sh/ace/) | +| [Textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) | | [y-textarea](http://github.com/yjs/y-textarea) | [demo](https://yjs-demos.now.sh/textarea/) | +| [DOM](https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model) | | [y-dom](http://github.com/yjs/y-dom) | [demo](https://yjs-demos.now.sh/dom/) | -##### Other +### Providers -| Name | Description | -|-----------|-------------------| -|[y-element](http://y-js.org/y-element/) | Yjs Polymer Element | +Setting up the communication between clients, managing awareness information, +and storing shared data for offline usage is quite a hassle. **Providers** +manage all that for you and are the perfect starting point for your +collaborative app. -## Use it! -Install Yjs, and its modules with [bower](http://bower.io/), or -[npm](https://www.npmjs.org/package/yjs). +<dl> + <dt><a href="http://github.com/yjs/y-websocket">y-websocket</a></dt> + <dd> +A module that contains a simple websocket backend and a websocket client that +connects to that backend. The backend can be extended to persist updates in a +leveldb database. + </dd> + <dt><a href="http://github.com/yjs/y-mesh">y-mesh</a></dt> + <dd> +[WIP] Creates a connected graph of webrtc connections with a high +<a href="https://en.wikipedia.org/wiki/Strength_of_a_graph">strength</a>. It +requires a signalling server that connects a client to the first peer. But after +that the network manages itself. It is well suited for large and small networks. + </dd> + <dt><a href="http://github.com/yjs/y-dat">y-dat</a></dt> + <dd> +[WIP] Write document updates effinciently to the dat network using +<a href="https://github.com/kappa-db/multifeed">multifeed</a>. Each client has +an append-only log of CRDT local updates (hypercore). Multifeed manages and sync +hypercores and y-dat listens to changes and applies them to the Yjs document. +</dd> +</dl> -### Bower +## Getting Started -``` -bower install --save yjs y-array % add all y-* modules you want to use -``` -You only need to include the `y.js` file. Yjs is able to automatically require -missing modules. -``` -<script src="./bower_components/yjs/y.js"></script> -``` +Install Yjs and a provider with your favorite package manager: -### CDN - -``` -<script src="https://cdn.jsdelivr.net/npm/yjs@12/dist/y.js"></script> -<script src="https://cdn.jsdelivr.net/npm/y-array@10/dist/y-array.js"></script> -<script src="https://cdn.jsdelivr.net/npm/y-websockets-client@8/dist/y-websockets-client.js"></script> -<script src="https://cdn.jsdelivr.net/npm/y-memory@8/dist/y-memory.js"></script> -<script src="https://cdn.jsdelivr.net/npm/y-map@10/dist/y-map.js"></script> -<script src="https://cdn.jsdelivr.net/npm/y-text@9/dist/y-text.js"></script> -// .. -// do the same for all modules you want to use -``` - -### Npm - -``` -npm install --save yjs % add all y-* modules you want to use -``` - -If you don't include via script tag, you have to explicitly include all modules! -(Same goes for other module systems) -``` -var Y = require('yjs') -require('y-array')(Y) // add the y-array type to Yjs -require('y-websockets-client')(Y) -require('y-memory')(Y) -require('y-map')(Y) -require('y-text')(Y) -// .. -// do the same for all modules you want to use -``` - -### ES6 Syntax - -``` -import Y from 'yjs' -import yArray from 'y-array' -import yWebsocketsClient from 'y-webrtc' -import yMemory from 'y-memory' -import yMap from 'y-map' -import yText from 'y-text' -// .. -Y.extend(yArray, yWebsocketsClient, yMemory, yArray, yMap, yText /*, .. */) -``` - -# Text editing example - -Install dependencies -``` -bower i yjs y-memory y-webrtc y-array y-text -``` - -Here is a simple example of a shared textarea -```HTML - <!DOCTYPE html> - <html> - <body> - <script src="./bower_components/yjs/y.js"></script> - <!-- Yjs automatically includes all missing dependencies (browser only) --> - <script> - Y({ - db: { - name: 'memory' // use memory database adapter. - // name: 'indexeddb' // use indexeddb database adapter instead for offline apps - }, - connector: { - name: 'webrtc', // use webrtc connector - // name: 'websockets-client' - // name: 'xmpp' - room: 'my-room' // clients connecting to the same room share data - }, - sourceDir: './bower_components', // location of the y-* modules (browser only) - share: { - textarea: 'Text' // y.share.textarea is of type y-text - } - }).then(function (y) { - // The Yjs instance `y` is available - // y.share.* contains the shared types - - // Bind `y.share.textarea` to `<textarea/>` - y.share.textarea.bind(document.querySelector('textarea')) - }) - </script> - <textarea></textarea> - </body> - </html> -``` - -## Get Help & Give Help -There are some friendly people on [](https://gitter.im/y-js/yjs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) who are eager to help, and answer questions. Please join! - -Report _any_ issues to the -[Github issue page](https://github.com/y-js/yjs/issues)! I try to fix them very -soon, if possible. - -# API - -### Y(options) -* Y.extend(module1, module2, ..) - * Add extensions to Y - * `Y.extend(require('y-webrtc'))` has the same semantics as - `require('y-webrtc')(Y)` -* options.db - * Will be forwarded to the database adapter. Specify the database adaper on - `options.db.name`. - * Have a look at the used database adapter repository to see all available - options. -* options.connector - * Will be forwarded to the connector adapter. Specify the connector adaper on - `options.connector.name`. - * All our connectors implement a `room` property. Clients that specify the - same room share the same data. - * All of our connectors specify an `url` property that defines the connection - endpoint of the used connector. - * All of our connectors also have a default connection endpoint that you can - use for development. - * Set `options.connector.generateUserId = true` in order to genenerate a - userid, instead of receiving one from the server. This way the `Y(..)` is - immediately going to be resolved, without waiting for any confirmation from - the server. Use with caution. - * Have a look at the used connector repository to see all available options. - * *Only if you know what you are doing:* Set - `options.connector.preferUntransformed = true` in order receive the shared - data untransformed. This is very efficient as the database content is simply - copied to this client. This does only work if this client receives content - from only one client. -* options.sourceDir (browser only) - * Path where all y-* modules are stored - * Defaults to `/bower_components` - * Not required when running on `nodejs` / `iojs` - * When using nodejs you need to manually extend Yjs: -``` -var Y = require('yjs') -// you have to require a db, connector, and *all* types you use! -require('y-memory')(Y) -require('y-webrtc')(Y) -require('y-map')(Y) -// .. -``` -* options.share - * Specify on `options.share[arbitraryName]` types that are shared among all - users. - * E.g. Specify `options.share[arbitraryName] = 'Array'` to require y-array and - create an y-array type on `y.share[arbitraryName]`. - * If userA doesn't specify `options.share[arbitraryName]`, it won't be - available for userA. - * If userB specifies `options.share[arbitraryName]`, it still won't be - available for userA. But all the updates are send from userB to userA. - * In contrast to y-map, types on `y.share.*` cannot be overwritten or deleted. - Instead, they are merged among all users. This feature is only available on - `y.share.*` - * Weird behavior: It is supported that two users specify different types with - the same property name. - E.g. userA specifies `options.share.x = 'Array'`, and userB specifies - `options.share.x = 'Text'`. But they only share data if they specified the - same type with the same property name -* options.type (browser only) - * Array of modules that Yjs needs to require, before instantiating a shared - type. - * By default Yjs requires the specified database adapter, the specified - connector, and all modules that are used in `options.share.*` - * Put all types here that you intend to use, but are not used in y.share.* - -### Instantiated Y object (y) -`Y(options)` returns a promise that is fulfilled when.. - -* All modules are loaded - * The specified database adapter is loaded - * The specified connector is loaded - * All types are included -* The connector is initialized, and a unique user id is set (received from the - server) - * Note: When using y-indexeddb, a retrieved user id is stored on `localStorage` - -The promise returns an instance of Y. We denote it with a lower case `y`. - -* y.share.* - * Instances of the types you specified on options.share.* - * y.share.* can only be defined once when you instantiate Y! -* y.connector is an instance of Y.AbstractConnector -* y.connector.onUserEvent(function (event) {..}) - * Observe user events (event.action is either 'userLeft' or 'userJoined') -* y.connector.whenSynced(listener) - * `listener` is executed when y synced with at least one user. - * `listener` is not called when no other user is in the same room. - * y-websockets-client aways waits to sync with the server -* y.connector.disconnect() - * Force to disconnect this instance from the other instances -* y.connector.connect() - * Try to reconnect to the other instances (needs to be supported by the - connector) - * Not supported by y-xmpp -* y.close() - * Destroy this object. - * Destroys all types (they will throw weird errors if you still use them) - * Disconnects from the other instances (via connector) - * Returns a promise -* y.destroy() - * calls y.close() - * Removes all data from the database - * Returns a promise -* y.db.stopGarbageCollector() - * Stop the garbage collector. Call y.db.garbageCollect() to continue garbage - collection -* y.db.gc :: Boolean - * Whether gc is turned on -* y.db.gcTimeout :: Number (defaults to 50000 ms) - * Time interval between two garbage collect cycles - * It is required that all instances exchanged all messages after two garbage - collect cycles (after 100000 ms per default) -* y.db.userId :: String - * The used user id for this client. **Never overwrite this** - -### Logging -Yjs uses [debug](https://github.com/visionmedia/debug) for logging. The flag -`y*` enables logging for all y-* components. You can selectively remove -components you are not interested in: E.g. The flag `y*,-y:connector-message` -will not log the long `y:connector-message` messages. - -##### Enable logging in Node.js ```sh -DEBUG=y* node app.js +npm i yjs@13.0.0-97 y-websocket@1.0.0-6 ``` -Remove the colors in order to log to a file: +Start the y-websocket server: + ```sh -DEBUG_COLORS=0 DEBUG=y* node app.js > log +PORT=1234 node ./node_modules/y-websocket/bin/server.js ``` -##### Enable logging in the browser +### Example: Observe types + ```js -localStorage.debug = 'y*' +const yarray = doc.getArray('my-array') +yarray.observe(event => { + console.log('yarray was modified') +}) +// every time a local or remote client modifies yarray, the observer is called +yarray.insert(0, ['val']) // => "yarray was modified" ``` -## License -Yjs is licensed under the [MIT License](./LICENSE). +### Example: Nest types + +Remember, shared types are just plain old data types. The only limitation is +that a shared type must exist only once in the shared document. + +```js +const ymap = doc.getMap('map') +const foodArray = new Y.Array() +foodArray.insert(0, ['apple', 'banana']) +ymap.set('food', foodArray) +ymap.get('food') === foodArray // => true +ymap.set('fruit', foodArray) // => Error! foodArray is already defined +``` + +Now you understand how types are defined on a shared document. Next you can jump +to the [demo repository](https://github.com/yjs/yjs-demos) or continue reading +the API docs. + +## API + +```js +import * as Y from 'yjs' +``` + +### Shared Types + +<details> + <summary><b>Y.Array</b></summary> + <br> + <p> +A shareable Array-like type that supports efficient insert/delete of elements +at any position. Internally it uses a linked list of Arrays that is split when +necessary. + </p> + <pre>const yarray = new Y.Array()</pre> + <dl> + <b><code>insert(index:number, content:Array<object|boolean|Array|string|number|Uint8Array|Y.Type>)</code></b> + <dd> +Insert content at <var>index</var>. Note that content is an array of elements. +I.e. <code>array.insert(0, [1]</code> splices the list and inserts 1 at +position 0. + </dd> + <b><code>push(Array<Object|boolean|Array|string|number|Uint8Array|Y.Type>)</code></b> + <dd></dd> + <b><code>delete(index:number, length:number)</code></b> + <dd></dd> + <b><code>get(index:number)</code></b> + <dd></dd> + <b><code>length:number</code></b> + <dd></dd> + <b> + <code> +forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, + index:number, array: Y.Array)) + </code> + </b> + <dd></dd> + <b><code>map(function(T, number, YArray):M):Array<M></code></b> + <dd></dd> + <b><code>toArray():Array<object|boolean|Array|string|number|Uint8Array|Y.Type></code></b> + <dd>Copies the content of this YArray to a new Array.</dd> + <b><code>toJSON():Array<Object|boolean|Array|string|number></code></b> + <dd> +Copies the content of this YArray to a new Array. It transforms all child types +to JSON using their <code>toJSON</code> method. + </dd> + <b><code>[Symbol.Iterator]</code></b> + <dd> + Returns an YArray Iterator that contains the values for each index in the array. + <pre>for (let value of yarray) { .. }</pre> + </dd> + <b><code>observe(function(YArrayEvent, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. + </dd> + <b><code>unobserve(function(YArrayEvent, Transaction):void)</code></b> + <dd> + Removes an <code>observe</code> event listener from this type. + </dd> + <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. + </dd> + <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> + Removes an <code>observeDeep</code> event listener from this type. + </dd> + </dl> +</details> +<details> + <summary><b>Y.Map</b></summary> + <br> + <p> + A shareable Map type. + </p> + <pre><code>const ymap = new Y.Map()</code></pre> + <dl> + <b><code>get(key:string):object|boolean|string|number|Uint8Array|Y.Type</code></b> + <dd></dd> + <b><code>set(key:string, value:object|boolean|string|number|Uint8Array|Y.Type)</code></b> + <dd></dd> + <b><code>delete(key:string)</code></b> + <dd></dd> + <b><code>has(key:string):boolean</code></b> + <dd></dd> + <b><code>get(index:number)</code></b> + <dd></dd> + <b><code>toJSON():Object<string, Object|boolean|Array|string|number|Uint8Array></code></b> + <dd> +Copies the <code>[key,value]</code> pairs of this YMap to a new Object.It +transforms all child types to JSON using their <code>toJSON</code> method. + </dd> + <b><code>forEach(function(value:object|boolean|Array|string|number|Uint8Array|Y.Type, + key:string, map: Y.Map))</code></b> + <dd> + Execute the provided function once for every key-value pair. + </dd> + <b><code>[Symbol.Iterator]</code></b> + <dd> + Returns an Iterator of <code>[key, value]</code> pairs. + <pre>for (let [key, value] of ymap) { .. }</pre> + </dd> + <b><code>entries()</code></b> + <dd> + Returns an Iterator of <code>[key, value]</code> pairs. + </dd> + <b><code>values()</code></b> + <dd> + Returns an Iterator of all values. + </dd> + <b><code>keys()</code></b> + <dd> + Returns an Iterator of all keys. + </dd> + <b><code>observe(function(YMapEvent, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. + </dd> + <b><code>unobserve(function(YMapEvent, Transaction):void)</code></b> + <dd> + Removes an <code>observe</code> event listener from this type. + </dd> + <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. + </dd> + <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> + Removes an <code>observeDeep</code> event listener from this type. + </dd> + </dl> +</details> + +<details> + <summary><b>Y.Text</b></summary> + <br> + <p> +A shareable type that is optimized for shared editing on text. It allows to +assign properties to ranges in the text. This makes it possible to implement +rich-text bindings to this type. + </p> + <p> +This type can also be transformed to the +<a href="https://quilljs.com/docs/delta">delta format</a>. Similarly the +YTextEvents compute changes as deltas. + </p> + <pre>const ytext = new Y.Text()</pre> + <dl> + <b><code>insert(index:number, content:string, [formattingAttributes:Object<string,string>])</code></b> + <dd> + Insert a string at <var>index</var> and assign formatting attributes to it. + <pre>ytext.insert(0, 'bold text', { bold: true })</pre> + </dd> + <b><code>delete(index:number, length:number)</code></b> + <dd></dd> + <b><code>format(index:number, length:number, formattingAttributes:Object<string,string>)</code></b> + <dd>Assign formatting attributes to a range in the text</dd> + <b><code>applyDelta(delta)</code></b> + <dd>See <a href="https://quilljs.com/docs/delta/">Quill Delta</a></dd> + <b><code>length:number</code></b> + <dd></dd> + <b><code>toString():string</code></b> + <dd>Transforms this type, without formatting options, into a string.</dd> + <b><code>toJSON():string</code></b> + <dd>See <code>toString</code></dd> + <b><code>toDelta():Delta</code></b> + <dd> +Transforms this type to a <a href="https://quilljs.com/docs/delta/">Quill Delta</a> + </dd> + <b><code>observe(function(YTextEvent, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. + </dd> + <b><code>unobserve(function(YTextEvent, Transaction):void)</code></b> + <dd> + Removes an <code>observe</code> event listener from this type. + </dd> + <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. + </dd> + <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> + Removes an <code>observeDeep</code> event listener from this type. + </dd> + </dl> +</details> + +<details> + <summary><b>Y.XmlFragment</b></summary> + <br> + <p> + A container that holds an Array of Y.XmlElements. + </p> + <pre><code>const yxml = new Y.XmlFragment()</code></pre> + <dl> + <b><code>insert(index:number, content:Array<Y.XmlElement|Y.XmlText>)</code></b> + <dd></dd> + <b><code>delete(index:number, length:number)</code></b> + <dd></dd> + <b><code>get(index:number)</code></b> + <dd></dd> + <b><code>length:number</code></b> + <dd></dd> + <b><code>toArray():Array<Y.XmlElement|Y.XmlText></code></b> + <dd>Copies the children to a new Array.</dd> + <b><code>toDOM():DocumentFragment</code></b> + <dd>Transforms this type and all children to new DOM elements.</dd> + <b><code>toString():string</code></b> + <dd>Get the XML serialization of all descendants.</dd> + <b><code>toJSON():string</code></b> + <dd>See <code>toString</code>.</dd> + <b><code>observe(function(YXmlEvent, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type is modified. In the case this type is modified in the event listener, +the event listener will be called again after the current event listener returns. + </dd> + <b><code>unobserve(function(YXmlEvent, Transaction):void)</code></b> + <dd> + Removes an <code>observe</code> event listener from this type. + </dd> + <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. + </dd> + <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> + Removes an <code>observeDeep</code> event listener from this type. + </dd> + </dl> +</details> + +<details> + <summary><b>Y.XmlElement</b></summary> + <br> + <p> +A shareable type that represents an XML Element. It has a <code>nodeName</code>, +attributes, and a list of children. But it makes no effort to validate its +content and be actually XML compliant. + </p> + <pre><code>const yxml = new Y.XmlElement()</code></pre> + <dl> + <b><code>insert(index:number, content:Array<Y.XmlElement|Y.XmlText>)</code></b> + <dd></dd> + <b><code>delete(index:number, length:number)</code></b> + <dd></dd> + <b><code>get(index:number)</code></b> + <dd></dd> + <b><code>length:number</code></b> + <dd></dd> + <b><code>setAttribute(attributeName:string, attributeValue:string)</code></b> + <dd></dd> + <b><code>removeAttribute(attributeName:string)</code></b> + <dd></dd> + <b><code>getAttribute(attributeName:string):string</code></b> + <dd></dd> + <b><code>getAttributes(attributeName:string):Object<string,string></code></b> + <dd></dd> + <b><code>toArray():Array<Y.XmlElement|Y.XmlText></code></b> + <dd>Copies the children to a new Array.</dd> + <b><code>toDOM():Element</code></b> + <dd>Transforms this type and all children to a new DOM element.</dd> + <b><code>toString():string</code></b> + <dd>Get the XML serialization of all descendants.</dd> + <b><code>toJSON():string</code></b> + <dd>See <code>toString</code>.</dd> + <b><code>observe(function(YXmlEvent, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every +time this type is modified. In the case this type is modified in the event +listener, the event listener will be called again after the current event +listener returns. + </dd> + <b><code>unobserve(function(YXmlEvent, Transaction):void)</code></b> + <dd> + Removes an <code>observe</code> event listener from this type. + </dd> + <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> +Adds an event listener to this type that will be called synchronously every time +this type or any of its children is modified. In the case this type is modified +in the event listener, the event listener will be called again after the current +event listener returns. The event listener receives all Events created by itself +or any of its children. + </dd> + <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> + <dd> + Removes an <code>observeDeep</code> event listener from this type. + </dd> + </dl> +</details> + +### Y.Doc + +```js +const doc = new Y.Doc() +``` + +<dl> + <b><code>clientID</code></b> + <dd>A unique id that identifies this client. (readonly)</dd> + <b><code>transact(function(Transaction):void [, origin:any])</code></b> + <dd> +Every change on the shared document happens in a transaction. Observer calls and +the <code>update</code> event are called after each transaction. You should +<i>bundle</i> changes into a single transaction to reduce the amount of event +calls. I.e. <code>doc.transact(() => { yarray.insert(..); ymap.set(..) })</code> +triggers a single change event. <br>You can specify an optional <code>origin</code> +parameter that is stored on <code>transaction.origin</code> and +<code>on('update', (update, origin) => ..)</code>. + </dd> + <b><code>get(string, Y.[TypeClass]):[Type]</code></b> + <dd>Define a shared type.</dd> + <b><code>getArray(string):Y.Array</code></b> + <dd>Define a shared Y.Array type. Is equivalent to <code>y.get(string, Y.Array)</code>.</dd> + <b><code>getMap(string):Y.Map</code></b> + <dd>Define a shared Y.Map type. Is equivalent to <code>y.get(string, Y.Map)</code>.</dd> + <b><code>getXmlFragment(string):Y.XmlFragment</code></b> + <dd>Define a shared Y.XmlFragment type. Is equivalent to <code>y.get(string, Y.XmlFragment)</code>.</dd> + <b><code>on(string, function)</code></b> + <dd>Register an event listener on the shared type</dd> + <b><code>off(string, function)</code></b> + <dd>Unregister an event listener from the shared type</dd> +</dl> + +#### Y.Doc Events + +<dl> + <b><code>on('update', function(updateMessage:Uint8Array, origin:any, Y.Doc):void)</code></b> + <dd> +Listen to document updates. Document updates must be transmitted to all other +peers. You can apply document updates in any order and multiple times. + </dd> + <b><code>on('beforeTransaction', function(Y.Transaction, Y.Doc):void)</code></b> + <dd>Emitted before each transaction.</dd> + <b><code>on('afterTransaction', function(Y.Transaction, Y.Doc):void)</code></b> + <dd>Emitted after each transaction.</dd> +</dl> + +### Document Updates + +Changes on the shared document are encoded into *document updates*. Document +updates are *commutative* and *idempotent*. This means that they can be applied +in any order and multiple times. + +#### Example: Listen to update events and apply them on remote client + +```js +const doc1 = new Y.Doc() +const doc2 = new Y.Doc() + +doc1.on('update', update => { + Y.applyUpdate(doc2, update) +}) + +doc2.on('update', update => { + Y.applyUpdate(doc1, update) +}) + +// All changes are also applied to the other document +doc1.getArray('myarray').insert(0, ['Hello doc2, you got this?']) +doc2.getArray('myarray').get(0) // => 'Hello doc2, you got this?' +``` + +Yjs internally maintains a [state vector](#State-Vector) that denotes the next +expected clock from each client. In a different interpretation it holds the +number of structs created by each client. When two clients sync, you can either +exchange the complete document structure or only the differences by sending the +state vector to compute the differences. + +#### Example: Sync two clients by exchanging the complete document structure + +```js +const state1 = Y.encodeStateAsUpdate(ydoc1) +const state2 = Y.encodeStateAsUpdate(ydoc2) +Y.applyUpdate(ydoc1, state2) +Y.applyUpdate(ydoc2, state1) +``` + +#### Example: Sync two clients by computing the differences + +This example shows how to sync two clients with the minimal amount of exchanged +data by computing only the differences using the state vector of the remote +client. Syncing clients using the state vector requires another roundtrip, but +can safe a lot of bandwidth. + +```js +const stateVector1 = Y.encodeStateVector(ydoc1) +const stateVector2 = Y.encodeStateVector(ydoc2) +const diff1 = Y.encodeStateAsUpdate(ydoc1, stateVector2) +const diff2 = Y.encodeStateAsUpdate(ydoc2, stateVector1) +Y.applyUpdate(ydoc1, diff2) +Y.applyUpdate(ydoc2, diff1) +``` + +<dl> + <b><code>Y.applyUpdate(Y.Doc, update:Uint8Array, [transactionOrigin:any])</code></b> + <dd> +Apply a document update on the shared document. Optionally you can specify +<code>transactionOrigin</code> that will be stored on +<code>transaction.origin</code> +and <code>ydoc.on('update', (update, origin) => ..)</code>. + </dd> + <b><code>Y.encodeStateAsUpdate(Y.Doc, [encodedTargetStateVector:Uint8Array]):Uint8Array</code></b> + <dd> +Encode the document state as a single update message that can be applied on the +remote document. Optionally specify the target state vector to only write the +differences to the update message. + </dd> + <b><code>Y.encodeStateVector(Y.Doc):Uint8Array</code></b> + <dd>Computes the state vector and encodes it into an Uint8Array.</dd> +</dl> + +### Relative Positions + +> This API is not stable yet + +This feature is intended for managing selections / cursors. When working with +other users that manipulate the shared document, you can't trust that an index +position (an integer) will stay at the intended location. A *relative position* +is fixated to an element in the shared document and is not affected by remote +changes. I.e. given the document `"a|c"`, the relative position is attached to +`c`. When a remote user modifies the document by inserting a character before +the cursor, the cursor will stay attached to the character `c`. `insert(1, +'x')("a|c") = "ax|c"`. When the *relative position* is set to the end of the +document, it will stay attached to the end of the document. + +#### Example: Transform to RelativePosition and back + +```js +const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) +const pos = Y.createAbsolutePositionFromRelativePosition(relPos, doc) +pos.type === ytext // => true +pos.index === 2 // => true +``` + +#### Example: Send relative position to remote client (json) + +```js +const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) +const encodedRelPos = JSON.stringify(relPos) +// send encodedRelPos to remote client.. +const parsedRelPos = JSON.parse(encodedRelPos) +const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) +pos.type === remoteytext // => true +pos.index === 2 // => true +``` + +#### Example: Send relative position to remote client (Uint8Array) + +```js +const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) +const encodedRelPos = Y.encodeRelativePosition(relPos) +// send encodedRelPos to remote client.. +const parsedRelPos = Y.decodeRelativePosition(encodedRelPos) +const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) +pos.type === remoteytext // => true +pos.index === 2 // => true +``` + +<dl> + <b><code>Y.createRelativePositionFromTypeIndex(Uint8Array|Y.Type, number)</code></b> + <dd></dd> + <b><code>Y.createAbsolutePositionFromRelativePosition(RelativePosition, Y.Doc)</code></b> + <dd></dd> + <b><code>Y.encodeRelativePosition(RelativePosition):Uint8Array</code></b> + <dd></dd> + <b><code>Y.decodeRelativePosition(Uint8Array):RelativePosition</code></b> + <dd></dd> +</dl> + +### Y.UndoManager + +Yjs ships with an Undo/Redo manager for selective undo/redo of of changes on a +Yjs type. The changes can be optionally scoped to transaction origins. + +```js +const ytext = doc.getArray('array') +const undoManager = new Y.UndoManager(ytext) + +ytext.insert(0, 'abc') +undoManager.undo() +ytext.toString() // => '' +undoManager.redo() +ytext.toString() // => 'abc' +``` + +<dl> + <b><code>constructor(scope:Y.AbstractType|Array<Y.AbstractType>, + [[{captureTimeout:number,trackedOrigins:Set<any>,deleteFilter:function(item):boolean}]])</code></b> + <dd>Accepts either single type as scope or an array of types.</dd> + <b><code>undo()</code></b> + <dd></dd> + <b><code>redo()</code></b> + <dd></dd> + <b><code>stopCapturing()</code></b> + <dd></dd> + <b> + <code> +on('stack-item-added', { stackItem: { meta: Map<any,any> }, type: 'undo' +| 'redo' }) + </code> + </b> + <dd> +Register an event that is called when a <code>StackItem</code> is added to the +undo- or the redo-stack. + </dd> + <b> + <code> +on('stack-item-popped', { stackItem: { meta: Map<any,any> }, type: 'undo' +| 'redo' }) + </code> + </b> + <dd> +Register an event that is called when a <code>StackItem</code> is popped from +the undo- or the redo-stack. + </dd> +</dl> + +#### Example: Stop Capturing + +UndoManager merges Undo-StackItems if they are created within time-gap +smaller than `options.captureTimeout`. Call `um.stopCapturing()` so that the next +StackItem won't be merged. + +```js +// without stopCapturing +ytext.insert(0, 'a') +ytext.insert(1, 'b') +um.undo() +ytext.toString() // => '' (note that 'ab' was removed) +// with stopCapturing +ytext.insert(0, 'a') +um.stopCapturing() +ytext.insert(0, 'b') +um.undo() +ytext.toString() // => 'a' (note that only 'b' was removed) +``` + +#### Example: Specify tracked origins + +Every change on the shared document has an origin. If no origin was specified, +it defaults to `null`. By specifying `trackedTransactionOrigins` you can +selectively specify which changes should be tracked by `UndoManager`. The +UndoManager instance is always added to `trackedTransactionOrigins`. + +```js +class CustomBinding {} + +const ytext = doc.getArray('array') +const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) + +ytext.insert(0, 'abc') +undoManager.undo() +ytext.toString() // => 'abc' (does not track because origin `null` and not part + // of `trackedTransactionOrigins`) +ytext.delete(0, 3) // revert change + +doc.transact(() => { + ytext.insert(0, 'abc') +}, 42) +undoManager.undo() +ytext.toString() // => '' (tracked because origin is an instance of `trackedTransactionorigins`) + +doc.transact(() => { + ytext.insert(0, 'abc') +}, 41) +undoManager.undo() +ytext.toString() // => '' (not tracked because 41 is not an instance of + // `trackedTransactionorigins`) +ytext.delete(0, 3) // revert change + +doc.transact(() => { + ytext.insert(0, 'abc') +}, new CustomBinding()) +undoManager.undo() +ytext.toString() // => '' (tracked because origin is a `CustomBinding` and + // `CustomBinding` is in `trackedTransactionorigins`) +``` + +#### Example: Add additional information to the StackItems + +When undoing or redoing a previous action, it is often expected to restore +additional meta information like the cursor location or the view on the +document. You can assign meta-information to Undo-/Redo-StackItems. + +```js +const ytext = doc.getArray('array') +const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) + +undoManager.on('stack-item-added', event => { + // save the current cursor location on the stack-item + event.stackItem.meta.set('cursor-location', getRelativeCursorLocation()) +}) + +undoManager.on('stack-item-popped', event => { + // restore the current cursor location on the stack-item + restoreCursorLocation(event.stackItem.meta.get('cursor-location')) +}) +``` + +## Miscellaneous + +### Typescript Declarations + +Yjs has type descriptions. But until [this +ticket](https://github.com/Microsoft/TypeScript/issues/7546) is fixed, this is +how you can make use of Yjs type declarations. + +```json +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + }, + "maxNodeModuleJsDepth": 5 +} +``` + +## Yjs CRDT Algorithm + +*Conflict-free replicated data types* (CRDT) for collaborative editing are an +alternative approach to *operational transformation* (OT). A very simple +differenciation between the two approaches is that OT attempts to transform +index positions to ensure convergence (all clients end up with the same +content), while CRDTs use mathematical models that usually do not involve index +transformations, like linked lists. OT is currently the de-facto standard for +shared editing on text. OT approaches that support shared editing without a +central source of truth (a central server) require too much bookkeeping to be +viable in practice. CRDTs are better suited for distributed systems, provide +additional guarantees that the document can be synced with remote clients, and +do not require a central source of truth. + +Yjs implements a modified version of the algorithm described in [this +paper](https://www.researchgate.net/publication/310212186_Near_Real-Time_Peer-to-Peer_Shared_Editing_on_Extensible_Data_Types). +I will eventually publish a paper that describes why this approach works so well +in practice. Note: Since operations make up the document structure, we prefer +the term *struct* now. + +CRDTs suitable for shared text editing suffer from the fact that they only grow +in size. There are CRDTs that do not grow in size, but they do not have the +characteristics that are benificial for shared text editing (like intention +preservation). Yjs implements many improvements to the original algorithm that +diminish the trade-off that the document only grows in size. We can't garbage +collect deleted structs (tombstones) while ensuring a unique order of the +structs. But we can 1. merge preceeding structs into a single struct to reduce +the amount of meta information, 2. we can delete content from the struct if it +is deleted, and 3. we can garbage collect tombstones if we don't care about the +order of the structs anymore (e.g. if the parent was deleted). + +**Examples:** + +1. If a user inserts elements in sequence, the struct will be merged into a + single struct. E.g. `array.insert(0, ['a']), array.insert(0, ['b']);` is + first represented as two structs (`[{id: {client, clock: 0}, content: 'a'}, + {id: {client, clock: 1}, content: 'b'}`) and then merged into a single + struct: `[{id: {client, clock: 0}, content: 'ab'}]`. +2. When a struct that contains content (e.g. `ItemString`) is deleted, the + struct will be replaced with an `ItemDeleted` that does not contain content + anymore. +3. When a type is deleted, all child elements are transformed to `GC` structs. A + `GC` struct only denotes the existence of a struct and that it is deleted. + `GC` structs can always be merged with other `GC` structs if the id's are + adjacent. + +Especially when working on structured content (e.g. shared editing on +ProseMirror), these improvements yield very good results when +[benchmarking](https://github.com/dmonad/crdt-benchmarks) random document edits. +In practice they show even better results, because users usually edit text in +sequence, resulting in structs that can easily be merged. The benchmarks show +that even in the worst case scenario that a user edits text from right to left, +Yjs achieves good performance even for huge documents. + +### State Vector + +Yjs has the ability to exchange only the differences when syncing two clients. +We use lamport timestamps to identify structs and to track in which order a +client created them. Each struct has an `struct.id = { client: number, clock: +number}` that uniquely identifies a struct. We define the next expected `clock` +by each client as the *state vector*. This data structure is similar to the +[version vectors](https://en.wikipedia.org/wiki/Version_vector) data structure. +But we use state vectors only to describe the state of the local document, so we +can compute the missing struct of the remote client. We do not use it to track +causality. + +## License and Author + +Yjs and all related projects are [**MIT licensed**](./LICENSE). + +Yjs is based on my research as a student at the [RWTH +i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time. + +Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or +hiring [me](https://github.com/dmonad) for professional support. diff --git a/README.v12.md b/README.v12.md new file mode 100644 index 00000000..f750f721 --- /dev/null +++ b/README.v12.md @@ -0,0 +1,305 @@ + +#  + +Yjs is a framework for offline-first p2p shared editing on structured data like +text, richtext, json, or XML. It is fairly easy to get started, as Yjs hides +most of the complexity of concurrent editing. For additional information, demos, +and tutorials visit [y-js.org](http://y-js.org/). + +:warning: Checkout the [v13 docs](./README.md) for the upcoming release :warning: + +### Extensions +Yjs only knows how to resolve conflicts on shared data. You have to choose a .. + +* *Connector* - a communication protocol that propagates changes to the clients +* *Database* - a database to store your changes +* one or more *Types* - that represent the shared data + +Connectors, Databases, and Types are available as modules that extend Yjs. Here +is a list of the modules we know of: + +##### Connectors + +|Name | Description | +|----------------|-----------------------------------| +|[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC| +|[websockets](https://github.com/y-js/y-websockets-client) | Set up [a central server](https://github.com/y-js/y-websockets-client), and connect to it via websockets | +|[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))| +|[ipfs](https://github.com/ipfs-labs/y-ipfs-connector) | Connector for the [Interplanetary File System](https://ipfs.io/)!| +|[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios| + +##### Database adapters + +|Name | Description | +|----------------|-----------------------------------| +|[memory](https://github.com/y-js/y-memory) | In-memory storage. | +|[indexeddb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | +|[leveldb](https://github.com/y-js/y-leveldb) | Persistent storage for node apps | + +##### Types + +| Name | Description | +|----------|-------------------| +|[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | +|[array](https://github.com/y-js/y-array) | A shared Array implementation | +|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | +|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to the [Ace Editor](https://ace.c9.io), [CodeMirror](https://codemirror.net/), [Monaco](https://github.com/Microsoft/monaco-editor), textareas, input elements, and HTML elements (e.g. <*h1*>, or <*p*>) | +|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| + +##### Other + +| Name | Description | +|-----------|-------------------| +|[y-element](http://y-js.org/y-element/) | Yjs Polymer Element | + +## Use it! +Install Yjs, and its modules with [bower](http://bower.io/), or +[npm](https://www.npmjs.org/package/yjs). + +### Bower + +``` +bower install --save yjs y-array % add all y-* modules you want to use +``` +You only need to include the `y.js` file. Yjs is able to automatically require +missing modules. +``` +<script src="./bower_components/yjs/y.js"></script> +``` + +### CDN + +``` +<script src="https://cdn.jsdelivr.net/npm/yjs@12/dist/y.js"></script> +<script src="https://cdn.jsdelivr.net/npm/y-array@10/dist/y-array.js"></script> +<script src="https://cdn.jsdelivr.net/npm/y-websockets-client@8/dist/y-websockets-client.js"></script> +<script src="https://cdn.jsdelivr.net/npm/y-memory@8/dist/y-memory.js"></script> +<script src="https://cdn.jsdelivr.net/npm/y-map@10/dist/y-map.js"></script> +<script src="https://cdn.jsdelivr.net/npm/y-text@9/dist/y-text.js"></script> +// .. +// do the same for all modules you want to use +``` + +### Npm + +``` +npm install --save yjs % add all y-* modules you want to use +``` + +If you don't include via script tag, you have to explicitly include all modules! +(Same goes for other module systems) +``` +var Y = require('yjs') +require('y-array')(Y) // add the y-array type to Yjs +require('y-websockets-client')(Y) +require('y-memory')(Y) +require('y-map')(Y) +require('y-text')(Y) +// .. +// do the same for all modules you want to use +``` + +### ES6 Syntax + +``` +import Y from 'yjs' +import yArray from 'y-array' +import yWebsocketsClient from 'y-webrtc' +import yMemory from 'y-memory' +import yMap from 'y-map' +import yText from 'y-text' +// .. +Y.extend(yArray, yWebsocketsClient, yMemory, yArray, yMap, yText /*, .. */) +``` + +# Text editing example + +Install dependencies +``` +bower i yjs y-memory y-webrtc y-array y-text +``` + +Here is a simple example of a shared textarea +```HTML + <!DOCTYPE html> + <html> + <body> + <script src="./bower_components/yjs/y.js"></script> + <!-- Yjs automatically includes all missing dependencies (browser only) --> + <script> + Y({ + db: { + name: 'memory' // use memory database adapter. + // name: 'indexeddb' // use indexeddb database adapter instead for offline apps + }, + connector: { + name: 'webrtc', // use webrtc connector + // name: 'websockets-client' + // name: 'xmpp' + room: 'my-room' // clients connecting to the same room share data + }, + sourceDir: './bower_components', // location of the y-* modules (browser only) + share: { + textarea: 'Text' // y.share.textarea is of type y-text + } + }).then(function (y) { + // The Yjs instance `y` is available + // y.share.* contains the shared types + + // Bind `y.share.textarea` to `<textarea/>` + y.share.textarea.bind(document.querySelector('textarea')) + }) + </script> + <textarea></textarea> + </body> + </html> +``` + +## Get Help & Give Help +There are some friendly people on [](https://gitter.im/y-js/yjs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) who are eager to help, and answer questions. Please join! + +Report _any_ issues to the +[Github issue page](https://github.com/y-js/yjs/issues)! I try to fix them very +soon, if possible. + +# API + +### Y(options) +* Y.extend(module1, module2, ..) + * Add extensions to Y + * `Y.extend(require('y-webrtc'))` has the same semantics as + `require('y-webrtc')(Y)` +* options.db + * Will be forwarded to the database adapter. Specify the database adaper on + `options.db.name`. + * Have a look at the used database adapter repository to see all available + options. +* options.connector + * Will be forwarded to the connector adapter. Specify the connector adaper on + `options.connector.name`. + * All our connectors implement a `room` property. Clients that specify the + same room share the same data. + * All of our connectors specify an `url` property that defines the connection + endpoint of the used connector. + * All of our connectors also have a default connection endpoint that you can + use for development. + * Set `options.connector.generateUserId = true` in order to genenerate a + userid, instead of receiving one from the server. This way the `Y(..)` is + immediately going to be resolved, without waiting for any confirmation from + the server. Use with caution. + * Have a look at the used connector repository to see all available options. + * *Only if you know what you are doing:* Set + `options.connector.preferUntransformed = true` in order receive the shared + data untransformed. This is very efficient as the database content is simply + copied to this client. This does only work if this client receives content + from only one client. +* options.sourceDir (browser only) + * Path where all y-* modules are stored + * Defaults to `/bower_components` + * Not required when running on `nodejs` / `iojs` + * When using nodejs you need to manually extend Yjs: +``` +var Y = require('yjs') +// you have to require a db, connector, and *all* types you use! +require('y-memory')(Y) +require('y-webrtc')(Y) +require('y-map')(Y) +// .. +``` +* options.share + * Specify on `options.share[arbitraryName]` types that are shared among all + users. + * E.g. Specify `options.share[arbitraryName] = 'Array'` to require y-array and + create an y-array type on `y.share[arbitraryName]`. + * If userA doesn't specify `options.share[arbitraryName]`, it won't be + available for userA. + * If userB specifies `options.share[arbitraryName]`, it still won't be + available for userA. But all the updates are send from userB to userA. + * In contrast to y-map, types on `y.share.*` cannot be overwritten or deleted. + Instead, they are merged among all users. This feature is only available on + `y.share.*` + * Weird behavior: It is supported that two users specify different types with + the same property name. + E.g. userA specifies `options.share.x = 'Array'`, and userB specifies + `options.share.x = 'Text'`. But they only share data if they specified the + same type with the same property name +* options.type (browser only) + * Array of modules that Yjs needs to require, before instantiating a shared + type. + * By default Yjs requires the specified database adapter, the specified + connector, and all modules that are used in `options.share.*` + * Put all types here that you intend to use, but are not used in y.share.* + +### Instantiated Y object (y) +`Y(options)` returns a promise that is fulfilled when.. + +* All modules are loaded + * The specified database adapter is loaded + * The specified connector is loaded + * All types are included +* The connector is initialized, and a unique user id is set (received from the + server) + * Note: When using y-indexeddb, a retrieved user id is stored on `localStorage` + +The promise returns an instance of Y. We denote it with a lower case `y`. + +* y.share.* + * Instances of the types you specified on options.share.* + * y.share.* can only be defined once when you instantiate Y! +* y.connector is an instance of Y.AbstractConnector +* y.connector.onUserEvent(function (event) {..}) + * Observe user events (event.action is either 'userLeft' or 'userJoined') +* y.connector.whenSynced(listener) + * `listener` is executed when y synced with at least one user. + * `listener` is not called when no other user is in the same room. + * y-websockets-client aways waits to sync with the server +* y.connector.disconnect() + * Force to disconnect this instance from the other instances +* y.connector.connect() + * Try to reconnect to the other instances (needs to be supported by the + connector) + * Not supported by y-xmpp +* y.close() + * Destroy this object. + * Destroys all types (they will throw weird errors if you still use them) + * Disconnects from the other instances (via connector) + * Returns a promise +* y.destroy() + * calls y.close() + * Removes all data from the database + * Returns a promise +* y.db.stopGarbageCollector() + * Stop the garbage collector. Call y.db.garbageCollect() to continue garbage + collection +* y.db.gc :: Boolean + * Whether gc is turned on +* y.db.gcTimeout :: Number (defaults to 50000 ms) + * Time interval between two garbage collect cycles + * It is required that all instances exchanged all messages after two garbage + collect cycles (after 100000 ms per default) +* y.db.userId :: String + * The used user id for this client. **Never overwrite this** + +### Logging +Yjs uses [debug](https://github.com/visionmedia/debug) for logging. The flag +`y*` enables logging for all y-* components. You can selectively remove +components you are not interested in: E.g. The flag `y*,-y:connector-message` +will not log the long `y:connector-message` messages. + +##### Enable logging in Node.js +```sh +DEBUG=y* node app.js +``` + +Remove the colors in order to log to a file: +```sh +DEBUG_COLORS=0 DEBUG=y* node app.js > log +``` + +##### Enable logging in the browser +```js +localStorage.debug = 'y*' +``` + +## License +Yjs is licensed under the [MIT License](./LICENSE). diff --git a/README.v13.md b/README.v13.md deleted file mode 100644 index 8adcab73..00000000 --- a/README.v13.md +++ /dev/null @@ -1,869 +0,0 @@ - -#  - -> A CRDT framework with a powerful abstraction of shared data - -Yjs is a [CRDT implementation](#Yjs-CRDT-Algorithm) that exposes its internal -data structure as *shared types*. Shared types are common data types like `Map` -or `Array` with superpowers: changes are automatically distributed to other -peers and merged without merge conflicts. - -Yjs is **network agnostic** (p2p!), supports many existing **rich text -editors**, **offline editing**, **version snapshots**, **undo/redo** and -**shared cursors**. It scales well with an unlimited number of users and is well -suited for even large documents. - -* Chat: [https://gitter.im/y-js/yjs](https://gitter.im/y-js/yjs) -* Demos: [https://github.com/y-js/yjs-demos](https://github.com/y-js/yjs-demos) -* Benchmarks: [https://github.com/dmonad/crdt-benchmarks](https://github.com/dmonad/crdt-benchmarks) - -## Table of Contents - -* [Overview](#Overview) - * [Bindings](#Bindings) - * [Providers](#Providers) -* [Getting Started](#Getting-Started) -* [API](#API) - * [Shared Types](#Shared-Types) - * [Y.Doc](#YDoc) - * [Document Updates](#Document-Updates) - * [Relative Positions](#Relative-Positions) - * [Y.UndoManager](#YUndoManager) -* [Miscellaneous](#Miscellaneous) - * [Typescript Declarations](#Typescript-Declarations) -* [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm) -* [Evaluation](#Evaluation) - * [Existing shared editing libraries](#Exisisting-Javascript-Libraries) - * [CRDT Algorithms](#CRDT-Algorithms) - * [Comparison of CRDT with OT](#Comparing-CRDT-with-OT) - * [Comparison of CRDT Algorithms](#Comparing-CRDT-Algorithms) - * [Comparison of Yjs with other Implementations](#Comparing-Yjs-with-other-Implementations) -* [License and Author](#License-and-Author) - -## Overview - -This repository contains a collection of shared types that can be observed for -changes and manipulated concurrently. Network functionality and two-way-bindings -are implemented in separate modules. - -### Bindings - -| Name | Cursors | Binding | Demo | -|---|:-:|---|---| -| [ProseMirror](https://prosemirror.net/) | ✔ | [y-prosemirror](http://github.com/y-js/y-prosemirror) | [demo](https://yjs-demos.now.sh/prosemirror/) | -| [Quill](https://quilljs.com/) | | [y-quill](http://github.com/y-js/y-quill) | [demo](https://yjs-demos.now.sh/quill/) | -| [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/y-js/y-codemirror) | [demo](https://yjs-demos.now.sh/codemirror/) | -| [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/y-js/y-monaco) | [demo](https://yjs-demos.now.sh/monaco/) | -| [Ace](https://ace.c9.io/) | | [y-ace](http://github.com/y-js/y-ace) | [demo](https://yjs-demos.now.sh/ace/) | -| [Textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) | | [y-textarea](http://github.com/y-js/y-textarea) | [demo](https://yjs-demos.now.sh/textarea/) | -| [DOM](https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model) | | [y-dom](http://github.com/y-js/y-dom) | [demo](https://yjs-demos.now.sh/dom/) | - -### Providers - -Setting up the communication between clients, managing awareness information, -and storing shared data for offline usage is quite a hassle. **Providers** -manage all that for you and are the perfect starting point for your -collaborative app. - -<dl> - <dt><a href="http://github.com/y-js/y-websocket">y-websocket</a></dt> - <dd> -A module that contains a simple websocket backend and a websocket client that -connects to that backend. The backend can be extended to persist updates in a -leveldb database. - </dd> - <dt><a href="http://github.com/y-js/y-mesh">y-mesh</a></dt> - <dd> -[WIP] Creates a connected graph of webrtc connections with a high -<a href="https://en.wikipedia.org/wiki/Strength_of_a_graph">strength</a>. It -requires a signalling server that connects a client to the first peer. But after -that the network manages itself. It is well suited for large and small networks. - </dd> - <dt><a href="http://github.com/y-js/y-dat">y-dat</a></dt> - <dd> -[WIP] Write document updates effinciently to the dat network using -<a href="https://github.com/kappa-db/multifeed">multifeed</a>. Each client has -an append-only log of CRDT local updates (hypercore). Multifeed manages and sync -hypercores and y-dat listens to changes and applies them to the Yjs document. -</dd> -</dl> - -## Getting Started - -Install Yjs and a provider with your favorite package manager: - -```sh -npm i yjs@13.0.0-82 y-websocket@1.0.0-3 y-textarea -``` - -Start the y-websocket server: - -```sh -PORT=1234 node ./node_modules/y-websocket/bin/server.js -``` - -### Example: Textarea Binding - -This is a complete example on how to create a connection to a -[y-websocket](https://github.com/y-js/y-websocket) server instance, sync the -shared document to all clients in a *room*, and bind a Y.Text type to a dom -textarea. All changes to the textarea are automatically shared with everyone in -the same room. - -```js -import * as Y from 'yjs' -import { WebsocketProvider } from 'y-websocket' -import { TextareaBinding } from 'y-textarea' - -const doc = Y.Doc() -const provider = new WebsocketProvider('ws://localhost:1234', 'roomname', doc) - -// Define a shared type on the document. -const ytext = doc.getText('my resume') - -// use data bindings to bind types to editors -const binding = new TextareaBinding(ytext, document.querySelector('textarea')) -``` - -#### Example: Observe types - -```js -const yarray = doc.getArray('my-array') -yarray.observe(event => { - console.log('yarray was modified') -}) -// every time a local or remote client modifies yarray, the observer is called -yarray.insert(0, ['val']) // => "yarray was modified" -``` - -#### Example: Nest types - -Remember, shared types are just plain old data types. The only limitation is -that a shared type must exist only once in the shared document. - -```js -const ymap = doc.getMap('map') -const foodArray = new Y.Array() -foodArray.insert(0, ['apple', 'banana']) -ymap.set('food', foodArray) -ymap.get('food') === foodArray // => true -ymap.set('fruit', foodArray) // => Error! foodArray is already defined -``` - -Now you understand how types are defined on a shared document. Next you can jump -to the [demo repository](https://github.com/y-js/yjs-demos) or continue reading -the API docs. - -## API - -```js -import * as Y from 'yjs' -``` - -### Shared Types - -<details> - <summary><b>Y.Array</b></summary> - <br> - <p> -A shareable Array-like type that supports efficient insert/delete of elements -at any position. Internally it uses a linked list of Arrays that is split when -necessary. - </p> - <pre>const yarray = new Y.Array()</pre> - <dl> - <b><code>insert(index:number, content:Array<object|boolean|Array|string|number|Uint8Array|Y.Type>)</code></b> - <dd> -Insert content at <var>index</var>. Note that content is an array of elements. -I.e. <code>array.insert(0, [1]</code> splices the list and inserts 1 at -position 0. - </dd> - <b><code>push(Array<Object|boolean|Array|string|number|Uint8Array|Y.Type>)</code></b> - <dd></dd> - <b><code>delete(index:number, length:number)</code></b> - <dd></dd> - <b><code>get(index:number)</code></b> - <dd></dd> - <b><code>length:number</code></b> - <dd></dd> - <b><code>forEach(function(index:number,value:object|boolean|Array|string|number|Uint8Array|Y.Type))</code></b> - <dd></dd> - <b><code>map(function(T, number, YArray):M):Array<M></code></b> - <dd></dd> - <b><code>toArray():Array<object|boolean|Array|string|number|Uint8Array|Y.Type></code></b> - <dd>Copies the content of this YArray to a new Array.</dd> - <b><code>toJSON():Array<Object|boolean|Array|string|number></code></b> - <dd> -Copies the content of this YArray to a new Array. It transforms all child types -to JSON using their <code>toJSON</code> method. - </dd> - <b><code>[Symbol.Iterator]</code></b> - <dd> - Returns an YArray Iterator that contains the values for each index in the array. - <pre>for (let value of yarray) { .. }</pre> - </dd> - <b><code>observe(function(YArrayEvent, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. - </dd> - <b><code>unobserve(function(YArrayEvent, Transaction):void)</code></b> - <dd> - Removes an <code>observe</code> event listener from this type. - </dd> - <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. - </dd> - <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> - Removes an <code>observeDeep</code> event listener from this type. - </dd> - </dl> -</details> -<details> - <summary><b>Y.Map</b></summary> - <br> - <p> - A shareable Map type. - </p> - <pre><code>const ymap = new Y.Map()</code></pre> - <dl> - <b><code>get(key:string):object|boolean|string|number|Uint8Array|Y.Type</code></b> - <dd></dd> - <b><code>set(key:string, value:object|boolean|string|number|Uint8Array|Y.Type)</code></b> - <dd></dd> - <b><code>delete(key:string)</code></b> - <dd></dd> - <b><code>has(key:string):boolean</code></b> - <dd></dd> - <b><code>get(index:number)</code></b> - <dd></dd> - <b><code>toJSON():Object<string, Object|boolean|Array|string|number|Uint8Array></code></b> - <dd> -Copies the <code>[key,value]</code> pairs of this YMap to a new Object.It -transforms all child types to JSON using their <code>toJSON</code> method. - </dd> - <b><code>forEach(function(key:string,value:object|boolean|Array|string|number|Uint8Array|Y.Type))</code></b> - <dd> - Execute the provided function once for every key-value pair. - </dd> - <b><code>[Symbol.Iterator]</code></b> - <dd> - Returns an Iterator of <code>[key, value]</code> pairs. - <pre>for (let [key, value] of ymap) { .. }</pre> - </dd> - <b><code>entries()</code></b> - <dd> - Returns an Iterator of <code>[key, value]</code> pairs. - </dd> - <b><code>values()</code></b> - <dd> - Returns an Iterator of all values. - </dd> - <b><code>keys()</code></b> - <dd> - Returns an Iterator of all keys. - </dd> - <b><code>observe(function(YMapEvent, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. - </dd> - <b><code>unobserve(function(YMapEvent, Transaction):void)</code></b> - <dd> - Removes an <code>observe</code> event listener from this type. - </dd> - <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. - </dd> - <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> - Removes an <code>observeDeep</code> event listener from this type. - </dd> - </dl> -</details> - -<details> - <summary><b>Y.Text</b></summary> - <br> - <p> -A shareable type that is optimized for shared editing on text. It allows to -assign properties to ranges in the text. This makes it possible to implement -rich-text bindings to this type. - </p> - <p> -This type can also be transformed to the -<a href="https://quilljs.com/docs/delta">delta format</a>. Similarly the -YTextEvents compute changes as deltas. - </p> - <pre>const ytext = new Y.Text()</pre> - <dl> - <b><code>insert(index:number, content:string, [formattingAttributes:Object<string,string>])</code></b> - <dd> - Insert a string at <var>index</var> and assign formatting attributes to it. - <pre>ytext.insert(0, 'bold text', { bold: true })</pre> - </dd> - <b><code>delete(index:number, length:number)</code></b> - <dd></dd> - <b><code>format(index:number, length:number, formattingAttributes:Object<string,string>)</code></b> - <dd>Assign formatting attributes to a range in the text</dd> - <b><code>applyDelta(delta)</code></b> - <dd>See <a href="https://quilljs.com/docs/delta/">Quill Delta</a></dd> - <b><code>length:number</code></b> - <dd></dd> - <b><code>toString():string</code></b> - <dd>Transforms this type, without formatting options, into a string.</dd> - <b><code>toJSON():string</code></b> - <dd>See <code>toString</code></dd> - <b><code>toDelta():Delta</code></b> - <dd> -Transforms this type to a <a href="https://quilljs.com/docs/delta/">Quill Delta</a> - </dd> - <b><code>observe(function(YTextEvent, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. - </dd> - <b><code>unobserve(function(YTextEvent, Transaction):void)</code></b> - <dd> - Removes an <code>observe</code> event listener from this type. - </dd> - <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. - </dd> - <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> - Removes an <code>observeDeep</code> event listener from this type. - </dd> - </dl> -</details> - -<details> - <summary><b>YXmlFragment</b></summary> - <br> - <p> - A container that holds an Array of Y.XmlElements. - </p> - <pre><code>const yxml = new Y.XmlFragment()</code></pre> - <dl> - <b><code>insert(index:number, content:Array<Y.XmlElement|Y.XmlText>)</code></b> - <dd></dd> - <b><code>delete(index:number, length:number)</code></b> - <dd></dd> - <b><code>get(index:number)</code></b> - <dd></dd> - <b><code>length:number</code></b> - <dd></dd> - <b><code>toArray():Array<Y.XmlElement|Y.XmlText></code></b> - <dd>Copies the children to a new Array.</dd> - <b><code>toDOM():DocumentFragment</code></b> - <dd>Transforms this type and all children to new DOM elements.</dd> - <b><code>toString():string</code></b> - <dd>Get the XML serialization of all descendants.</dd> - <b><code>toJSON():string</code></b> - <dd>See <code>toString</code>.</dd> - <b><code>observe(function(YXmlEvent, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type is modified. In the case this type is modified in the event listener, -the event listener will be called again after the current event listener returns. - </dd> - <b><code>unobserve(function(YXmlEvent, Transaction):void)</code></b> - <dd> - Removes an <code>observe</code> event listener from this type. - </dd> - <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. - </dd> - <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> - Removes an <code>observeDeep</code> event listener from this type. - </dd> - </dl> -</details> - -<details> - <summary><b>Y.XmlElement</b></summary> - <br> - <p> -A shareable type that represents an XML Element. It has a <code>nodeName</code>, -attributes, and a list of children. But it makes no effort to validate its -content and be actually XML compliant. - </p> - <pre><code>const yxml = new Y.XmlElement()</code></pre> - <dl> - <b><code>insert(index:number, content:Array<Y.XmlElement|Y.XmlText>)</code></b> - <dd></dd> - <b><code>delete(index:number, length:number)</code></b> - <dd></dd> - <b><code>get(index:number)</code></b> - <dd></dd> - <b><code>length:number</code></b> - <dd></dd> - <b><code>setAttribute(attributeName:string, attributeValue:string)</code></b> - <dd></dd> - <b><code>removeAttribute(attributeName:string)</code></b> - <dd></dd> - <b><code>getAttribute(attributeName:string):string</code></b> - <dd></dd> - <b><code>getAttributes(attributeName:string):Object<string,string></code></b> - <dd></dd> - <b><code>toArray():Array<Y.XmlElement|Y.XmlText></code></b> - <dd>Copies the children to a new Array.</dd> - <b><code>toDOM():Element</code></b> - <dd>Transforms this type and all children to a new DOM element.</dd> - <b><code>toString():string</code></b> - <dd>Get the XML serialization of all descendants.</dd> - <b><code>toJSON():string</code></b> - <dd>See <code>toString</code>.</dd> - <b><code>observe(function(YXmlEvent, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every -time this type is modified. In the case this type is modified in the event -listener, the event listener will be called again after the current event -listener returns. - </dd> - <b><code>unobserve(function(YXmlEvent, Transaction):void)</code></b> - <dd> - Removes an <code>observe</code> event listener from this type. - </dd> - <b><code>observeDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> -Adds an event listener to this type that will be called synchronously every time -this type or any of its children is modified. In the case this type is modified -in the event listener, the event listener will be called again after the current -event listener returns. The event listener receives all Events created by itself -or any of its children. - </dd> - <b><code>unobserveDeep(function(Array<YEvent>, Transaction):void)</code></b> - <dd> - Removes an <code>observeDeep</code> event listener from this type. - </dd> - </dl> -</details> - -### Y.Doc - -```js -const doc = new Y.Doc() -``` - -<dl> - <b><code>clientID</code></b> - <dd>A unique id that identifies this client. (readonly)</dd> - <b><code>transact(function(Transaction):void [, origin:any])</code></b> - <dd> -Every change on the shared document happens in a transaction. Observer calls and -the <code>update</code> event are called after each transaction. You should -<i>bundle</i> changes into a single transaction to reduce the amount of event -calls. I.e. <code>doc.transact(() => { yarray.insert(..); ymap.set(..) })</code> -triggers a single change event. <br>You can specify an optional <code>origin</code> -parameter that is stored on <code>transaction.origin</code> and -<code>on('update', (update, origin) => ..)</code>. - </dd> - <b><code>get(string, Y.[TypeClass]):[Type]</code></b> - <dd>Define a shared type.</dd> - <b><code>getArray(string):Y.Array</code></b> - <dd>Define a shared Y.Array type. Is equivalent to <code>y.get(string, Y.Array)</code>.</dd> - <b><code>getMap(string):Y.Map</code></b> - <dd>Define a shared Y.Map type. Is equivalent to <code>y.get(string, Y.Map)</code>.</dd> - <b><code>getXmlFragment(string):Y.XmlFragment</code></b> - <dd>Define a shared Y.XmlFragment type. Is equivalent to <code>y.get(string, Y.XmlFragment)</code>.</dd> - <b><code>on(string, function)</code></b> - <dd>Register an event listener on the shared type</dd> - <b><code>off(string, function)</code></b> - <dd>Unregister an event listener from the shared type</dd> -</dl> - -#### Y.Doc Events - -<dl> - <b><code>on('update', function(updateMessage:Uint8Array, origin:any, Y.Doc):void)</code></b> - <dd> -Listen to document updates. Document updates must be transmitted to all other -peers. You can apply document updates in any order and multiple times. - </dd> - <b><code>on('beforeTransaction', function(Y.Transaction, Y.Doc):void)</code></b> - <dd>Emitted before each transaction.</dd> - <b><code>on('afterTransaction', function(Y.Transaction, Y.Doc):void)</code></b> - <dd>Emitted after each transaction.</dd> -</dl> - -### Document Updates - -Changes on the shared document are encoded into *document updates*. Document -updates are *commutative* and *idempotent*. This means that they can be applied -in any order and multiple times. - -#### Example: Listen to update events and apply them on remote client - -```js -const doc1 = new Y.Doc() -const doc2 = new Y.Doc() - -doc1.on('update', update => { - Y.applyUpdate(doc2, update) -}) - -doc2.on('update', update => { - Y.applyUpdate(doc1, update) -}) - -// All changes are also applied to the other document -doc1.getArray('myarray').insert(0, ['Hello doc2, you got this?']) -doc2.getArray('myarray').get(0) // => 'Hello doc2, you got this?' -``` - -Yjs internally maintains a [state vector](#State-Vector) that denotes the next -expected clock from each client. In a different interpretation it holds the -number of structs created by each client. When two clients sync, you can either -exchange the complete document structure or only the differences by sending the -state vector to compute the differences. - -#### Example: Sync two clients by exchanging the complete document structure - -```js -const state1 = Y.encodeStateAsUpdate(ydoc1) -const state2 = Y.encodeStateAsUpdate(ydoc2) -Y.applyUpdate(ydoc1, state2) -Y.applyUpdate(ydoc2, state1) -``` - -#### Example: Sync two clients by computing the differences - -This example shows how to sync two clients with the minimal amount of exchanged -data by computing only the differences using the state vector of the remote -client. Syncing clients using the state vector requires another roundtrip, but -can safe a lot of bandwidth. - -```js -const stateVector1 = Y.encodeStateVector(ydoc1) -const stateVector2 = Y.encodeStateVector(ydoc2) -const diff1 = Y.encodeStateAsUpdate(ydoc1, stateVector2) -const diff2 = Y.encodeStateAsUpdate(ydoc2, stateVector1) -Y.applyUpdate(ydoc1, diff2) -Y.applyUpdate(ydoc2, diff1) -``` - -<dl> - <b><code>Y.applyUpdate(Y.Doc, update:Uint8Array, [transactionOrigin:any])</code></b> - <dd> -Apply a document update on the shared document. Optionally you can specify -<code>transactionOrigin</code> that will be stored on -<code>transaction.origin</code> -and <code>ydoc.on('update', (update, origin) => ..)</code>. - </dd> - <b><code>Y.encodeStateAsUpdate(Y.Doc, [encodedTargetStateVector:Uint8Array]):Uint8Array</code></b> - <dd> -Encode the document state as a single update message that can be applied on the -remote document. Optionally specify the target state vector to only write the -differences to the update message. - </dd> - <b><code>Y.encodeStateVector(Y.Doc):Uint8Array</code></b> - <dd>Computes the state vector and encodes it into an Uint8Array.</dd> -</dl> - -### Relative Positions - -> This API is not stable yet - -This feature is intended for managing selections / cursors. When working with -other users that manipulate the shared document, you can't trust that an index -position (an integer) will stay at the intended location. A *relative position* -is fixated to an element in the shared document and is not affected by remote -changes. I.e. given the document `"a|c"`, the relative position is attached to -`c`. When a remote user modifies the document by inserting a character before -the cursor, the cursor will stay attached to the character `c`. `insert(1, -'x')("a|c") = "ax|c"`. When the *relative position* is set to the end of the -document, it will stay attached to the end of the document. - -#### Example: Transform to RelativePosition and back - -```js -const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) -const pos = Y.createAbsolutePositionFromRelativePosition(relPos, doc) -pos.type === ytext // => true -pos.index === 2 // => true -``` - -#### Example: Send relative position to remote client (json) - -```js -const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) -const encodedRelPos = JSON.stringify(relPos) -// send encodedRelPos to remote client.. -const parsedRelPos = JSON.parse(encodedRelPos) -const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) -pos.type === remoteytext // => true -pos.index === 2 // => true -``` - -#### Example: Send relative position to remote client (Uint8Array) - -```js -const relPos = Y.createRelativePositionFromTypeIndex(ytext, 2) -const encodedRelPos = Y.encodeRelativePosition(relPos) -// send encodedRelPos to remote client.. -const parsedRelPos = Y.decodeRelativePosition(encodedRelPos) -const pos = Y.createAbsolutePositionFromRelativePosition(parsedRelPos, remoteDoc) -pos.type === remoteytext // => true -pos.index === 2 // => true -``` - -<dl> - <b><code>Y.createRelativePositionFromTypeIndex(Uint8Array|Y.Type, number)</code></b> - <dd></dd> - <b><code>Y.createAbsolutePositionFromRelativePosition(RelativePosition, Y.Doc)</code></b> - <dd></dd> - <b><code>Y.encodeRelativePosition(RelativePosition):Uint8Array</code></b> - <dd></dd> - <b><code>Y.decodeRelativePosition(Uint8Array):RelativePosition</code></b> - <dd></dd> -</dl> - -### Y.UndoManager - -Yjs ships with an Undo/Redo manager for selective undo/redo of of changes on a -Yjs type. The changes can be optionally scoped to transaction origins. - -```js -const ytext = doc.getArray('array') -const undoManager = new Y.UndoManager(ytext) - -ytext.insert(0, 'abc') -undoManager.undo() -ytext.toString() // => '' -undoManager.redo() -ytext.toString() // => 'abc' -``` - -<dl> - <b><code>constructor(scope:Y.AbstractType|Array<Y.AbstractType>, - [trackedTransactionOrigins:Set<any>, [{captureTimeout: number}]])</code></b> - <dd>Accepts either single type as scope or an array of types.</dd> - <b><code>undo()</code></b> - <dd></dd> - <b><code>redo()</code></b> - <dd></dd> - <b><code>stopCapturing()</code></b> - <dd></dd> - <b> - <code> -on('stack-item-added', { stackItem: { meta: Map<any,any> }, type: 'undo' -| 'redo' }) - </code> - </b> - <dd> -Register an event that is called when a <code>StackItem</code> is added to the -undo- or the redo-stack. - </dd> - <b> - <code> -on('stack-item-popped', { stackItem: { meta: Map<any,any> }, type: 'undo' -| 'redo' }) - </code> - </b> - <dd> -Register an event that is called when a <code>StackItem</code> is popped from -the undo- or the redo-stack. - </dd> -</dl> - -#### Example: Stop Capturing - -UndoManager merges Undo-StackItems if they are created within time-gap -smaller than `options.captureTimeout`. Call `um.stopCapturing()` so that the next -StackItem won't be merged. - -```js -// without stopCapturing -ytext.insert(0, 'a') -ytext.insert(1, 'b') -um.undo() -ytext.toString() // => '' (note that 'ab' was removed) -// with stopCapturing -ytext.insert(0, 'a') -um.stopCapturing() -ytext.insert(0, 'b') -um.undo() -ytext.toString() // => 'a' (note that only 'b' was removed) -``` - -#### Example: Specify tracked origins - -Every change on the shared document has an origin. If no origin was specified, -it defaults to `null`. By specifying `trackedTransactionOrigins` you can -selectively specify which changes should be tracked by `UndoManager`. The -UndoManager instance is always added to `trackedTransactionOrigins`. - -```js -class CustomBinding {} - -const ytext = doc.getArray('array') -const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) - -ytext.insert(0, 'abc') -undoManager.undo() -ytext.toString() // => 'abc' (does not track because origin `null` and not part - // of `trackedTransactionOrigins`) -ytext.delete(0, 3) // revert change - -doc.transact(() => { - ytext.insert(0, 'abc') -}, 42) -undoManager.undo() -ytext.toString() // => '' (tracked because origin is an instance of `trackedTransactionorigins`) - -doc.transact(() => { - ytext.insert(0, 'abc') -}, 41) -undoManager.undo() -ytext.toString() // => '' (not tracked because 41 is not an instance of - // `trackedTransactionorigins`) -ytext.delete(0, 3) // revert change - -doc.transact(() => { - ytext.insert(0, 'abc') -}, new CustomBinding()) -undoManager.undo() -ytext.toString() // => '' (tracked because origin is a `CustomBinding` and - // `CustomBinding` is in `trackedTransactionorigins`) -``` - -#### Example: Add additional information to the StackItems - -When undoing or redoing a previous action, it is often expected to restore -additional meta information like the cursor location or the view on the -document. You can assign meta-information to Undo-/Redo-StackItems. - -```js -const ytext = doc.getArray('array') -const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding])) - -undoManager.on('stack-item-added', event => { - // save the current cursor location on the stack-item - event.stackItem.meta.set('cursor-location', getRelativeCursorLocation()) -}) - -undoManager.on('stack-item-popped', event => { - // restore the current cursor location on the stack-item - restoreCursorLocation(event.stackItem.meta.get('cursor-location')) -}) -``` - -## Miscellaneous - -### Typescript Declarations - -Yjs has type descriptions. But until [this -ticket](https://github.com/Microsoft/TypeScript/issues/7546) is fixed, this is -how you can make use of Yjs type declarations. - -```json -{ - "compilerOptions": { - "allowJs": true, - "checkJs": true, - }, - "maxNodeModuleJsDepth": 5 -} -``` - -## Yjs CRDT Algorithm - -*Conflict-free replicated data types* (CRDT) for collaborative editing are an -alternative approach to *operational transformation* (OT). A very simple -differenciation between the two approaches is that OT attempts to transform -index positions to ensure convergence (all clients end up with the same -content), while CRDTs use mathematical models that usually do not involve index -transformations, like linked lists. OT is currently the de-facto standard for -shared editing on text. OT approaches that support shared editing without a -central source of truth (a central server) require too much bookkeeping to be -viable in practice. CRDTs are better suited for distributed systems, provide -additional guarantees that the document can be synced with remote clients, and -do not require a central source of truth. - -Yjs implements a modified version of the algorithm described in [this -paper](https://www.researchgate.net/publication/310212186_Near_Real-Time_Peer-to-Peer_Shared_Editing_on_Extensible_Data_Types). -I will eventually publish a paper that describes why this approach works so well -in practice. Note: Since operations make up the document structure, we prefer -the term *struct* now. - -CRDTs suitable for shared text editing suffer from the fact that they only grow -in size. There are CRDTs that do not grow in size, but they do not have the -characteristics that are benificial for shared text editing (like intention -preservation). Yjs implements many improvements to the original algorithm that -diminish the trade-off that the document only grows in size. We can't garbage -collect deleted structs (tombstones) while ensuring a unique order of the -structs. But we can 1. merge preceeding structs into a single struct to reduce -the amount of meta information, 2. we can delete content from the struct if it -is deleted, and 3. we can garbage collect tombstones if we don't care about the -order of the structs anymore (e.g. if the parent was deleted). - -**Examples:** - -1. If a user inserts elements in sequence, the struct will be merged into a - single struct. E.g. `array.insert(0, ['a']), array.insert(0, ['b']);` is - first represented as two structs (`[{id: {client, clock: 0}, content: 'a'}, - {id: {client, clock: 1}, content: 'b'}`) and then merged into a single - struct: `[{id: {client, clock: 0}, content: 'ab'}]`. -2. When a struct that contains content (e.g. `ItemString`) is deleted, the - struct will be replaced with an `ItemDeleted` that does not contain content - anymore. -3. When a type is deleted, all child elements are transformed to `GC` structs. A - `GC` struct only denotes the existence of a struct and that it is deleted. - `GC` structs can always be merged with other `GC` structs if the id's are - adjacent. - -Especially when working on structured content (e.g. shared editing on -ProseMirror), these improvements yield very good results when -[benchmarking](https://github.com/dmonad/crdt-benchmarks) random document edits. -In practice they show even better results, because users usually edit text in -sequence, resulting in structs that can easily be merged. The benchmarks show -that even in the worst case scenario that a user edits text from right to left, -Yjs achieves good performance even for huge documents. - -### State Vector - -Yjs has the ability to exchange only the differences when syncing two clients. -We use lamport timestamps to identify structs and to track in which order a -client created them. Each struct has an `struct.id = { client: number, clock: -number}` that uniquely identifies a struct. We define the next expected `clock` -by each client as the *state vector*. This data structure is similar to the -[version vectors](https://en.wikipedia.org/wiki/Version_vector) data structure. -But we use state vectors only to describe the state of the local document, so we -can compute the missing struct of the remote client. We do not use it to track -causality. - -## License and Author - -Yjs and all related projects are [**MIT licensed**](./LICENSE). - -Yjs is based on my research as a student at the [RWTH -i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time. - -Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or -hiring [me](https://github.com/dmonad) for professional support. diff --git a/package-lock.json b/package-lock.json index 20a9d4a6..9ed65fb6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,13 @@ { "name": "yjs", - "version": "13.0.0-94", + "version": "13.0.0-102", "lockfileVersion": 1, "requires": true, "dependencies": { "@babel/parser": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz", - "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.5.5.tgz", + "integrity": "sha512-E5BN68cqR7dhKan1SfqgPGhQ178bkVKpXTPEXnFJBrEt8/DKRZlybmy+IgYLTeN7tp1R5Ccmbm2rBk17sHYU3g==", "dev": true }, "@types/estree": { @@ -42,9 +42,9 @@ } }, "acorn": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", - "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.0.0.tgz", + "integrity": "sha512-PaF/MduxijYYt7unVGRuds1vBC9bFxbNf+VWqhOClfdgy7RlVkQqt610ig1/yxTgsDIfW1cWDel5EBbOy3jdtQ==", "dev": true }, "acorn-jsx": { @@ -431,12 +431,12 @@ "dev": true }, "catharsis": { - "version": "0.8.10", - "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.10.tgz", - "integrity": "sha512-l2OUaz/3PU3MZylspVFJvwHCVfWyvcduPq4lv3AzZ2pJzZCo7kNKFNyatwujD7XgvGkNAE/Jhhbh2uARNwNkfw==", + "version": "0.8.11", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.11.tgz", + "integrity": "sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g==", "dev": true, "requires": { - "lodash": "^4.17.11" + "lodash": "^4.17.14" } }, "chalk": { @@ -1490,7 +1490,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -1511,12 +1512,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1531,17 +1534,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -1658,7 +1664,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -1670,6 +1677,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -1684,6 +1692,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1691,12 +1700,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -1715,6 +1726,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -1795,7 +1807,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -1807,6 +1820,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -1892,7 +1906,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -1928,6 +1943,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -1947,6 +1963,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -1990,12 +2007,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -2495,22 +2514,22 @@ } }, "jsdoc": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.2.tgz", - "integrity": "sha512-S2vzg99C5+gb7FWlrK4TVdyzVPGGkdvpDkCEJH1JABi2PKzPeLu5/zZffcJUifgWUJqXWl41Hoc+MmuM2GukIg==", + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.3.tgz", + "integrity": "sha512-Yf1ZKA3r9nvtMWHO1kEuMZTlHOF8uoQ0vyo5eH7SQy5YeIiHM+B0DgKnn+X6y6KDYZcF7G2SPkKF+JORCXWE/A==", "dev": true, "requires": { "@babel/parser": "^7.4.4", "bluebird": "^3.5.4", - "catharsis": "^0.8.10", + "catharsis": "^0.8.11", "escape-string-regexp": "^2.0.0", "js2xmlparser": "^4.0.0", "klaw": "^3.0.0", "markdown-it": "^8.4.2", "markdown-it-anchor": "^5.0.2", - "marked": "^0.6.2", + "marked": "^0.7.0", "mkdirp": "^0.5.1", - "requizzle": "^0.2.2", + "requizzle": "^0.2.3", "strip-json-comments": "^3.0.1", "taffydb": "2.6.2", "underscore": "~1.9.1" @@ -2577,14 +2596,14 @@ } }, "lib0": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.0.5.tgz", - "integrity": "sha512-3ElV6/t5Lv0Eczlnh/05q+Uq3RxQ/Q0zdN6LVtaUERQIDDZsP/CUXEGLsV8KZTgZwVFNCPGXNWYE+3WTOo+SHw==" + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.1.1.tgz", + "integrity": "sha512-ghjoI4xL/xzVR1fRLYEOnJjYMguoI2dnDUf5HYOpTfD6R5GPKLml6xNKl4ZfBVmczkIOQPNthhukp6nlgbmDLw==" }, "linkify-it": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.1.0.tgz", - "integrity": "sha512-4REs8/062kV2DSHxNfq5183zrqXMl7WP0WzABH9IeJI+NLm429FgE1PDecltYfnOoFDFlZGh2T8PfZn0r+GTRg==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz", + "integrity": "sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==", "dev": true, "requires": { "uc.micro": "^1.0.1" @@ -2634,9 +2653,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "lodash.assignin": { @@ -2682,9 +2701,9 @@ "dev": true }, "lodash.merge": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", - "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, "lodash.pick": { @@ -2765,15 +2784,15 @@ } }, "markdown-it-anchor": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.1.0.tgz", - "integrity": "sha512-wJOmyXzDUxI8iuowEsaQAKMQBButhSw8j64SpgcaL75QZYC/OSZV66Fnr50lfMLYNGtV0rJdw2fmLwXCT6T+bw==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.2.4.tgz", + "integrity": "sha512-n8zCGjxA3T+Mx1pG8HEgbJbkB8JFUuRkeTZQuIM8iPY6oQ8sWOPRZJDFC9a/pNg2QkHEjjGkhBEl/RSyzaDZ3A==", "dev": true }, "marked": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.6.2.tgz", - "integrity": "sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==", + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-0.7.0.tgz", + "integrity": "sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==", "dev": true }, "mdurl": { @@ -2846,9 +2865,9 @@ "dev": true }, "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", "dev": true, "requires": { "for-in": "^1.0.2", @@ -3418,12 +3437,12 @@ } }, "requizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.2.tgz", - "integrity": "sha512-oJ6y7JcUJkblRGhMByGNcszeLgU0qDxNKFCiUZR1XyzHyVsev+Mxb1tyygxLd1ORsKee1SA5BInFdUwY64GE/A==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.3.tgz", + "integrity": "sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ==", "dev": true, "requires": { - "lodash": "^4.17.11" + "lodash": "^4.17.14" } }, "resolve": { @@ -3473,14 +3492,22 @@ } }, "rollup": { - "version": "1.12.4", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.12.4.tgz", - "integrity": "sha512-sHg0F05oTMJzM592MWU8irsPx8LIFMKSCnEkcp6vp/gnj+oJ9GJEBW9hl8jUqy2L6Q2uUxFzPgvoExLbfuSODA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.20.3.tgz", + "integrity": "sha512-/OMCkY0c6E8tleeVm4vQVDz24CkVgvueK3r8zTYu2AQNpjrcaPwO9hE+pWj5LTFrvvkaxt4MYIp2zha4y0lRvg==", "dev": true, "requires": { "@types/estree": "0.0.39", - "@types/node": "^12.0.2", - "acorn": "^6.1.1" + "@types/node": "^12.7.2", + "acorn": "^7.0.0" + }, + "dependencies": { + "@types/node": { + "version": "12.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.4.tgz", + "integrity": "sha512-W0+n1Y+gK/8G2P/piTkBBN38Qc5Q1ZSO6B5H3QmPCUewaiXOo2GCAWZ4ElZCcNhjJuBSUSLGFUJnmlCn5+nxOQ==", + "dev": true + } } }, "rollup-cli": { @@ -3629,9 +3656,9 @@ } }, "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", "dev": true, "requires": { "extend-shallow": "^2.0.1", @@ -4139,9 +4166,9 @@ "dev": true }, "typescript": { - "version": "3.4.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.5.tgz", - "integrity": "sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.2.tgz", + "integrity": "sha512-lmQ4L+J6mnu3xweP8+rOrUwzmN+MRAj7TgtJtDaXE5PMyX2kCrklhg3rvOsOIfNeAWMQWO2F1GPc1kMD2vLAfw==", "dev": true }, "uc.micro": { @@ -4157,38 +4184,15 @@ "dev": true }, "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", "dev": true, "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", "is-extendable": "^0.1.1", - "set-value": "^0.4.3" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "dev": true, - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } - } + "set-value": "^2.0.1" } }, "uniq": { @@ -4366,6 +4370,14 @@ "dev": true, "requires": { "lib0": "0.0.5" + }, + "dependencies": { + "lib0": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.0.5.tgz", + "integrity": "sha512-3ElV6/t5Lv0Eczlnh/05q+Uq3RxQ/Q0zdN6LVtaUERQIDDZsP/CUXEGLsV8KZTgZwVFNCPGXNWYE+3WTOo+SHw==", + "dev": true + } } }, "yallist": { diff --git a/package.json b/package.json index cc0e099f..e17380d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "13.0.0-94", + "version": "13.0.0-102", "description": "Shared Editing Library", "main": "./dist/yjs.js", "module": "./dist/yjs.mjs", @@ -10,8 +10,8 @@ "test-exhaustive": "npm run lint && npm run dist && node ./dist/tests.js --repitition-time 10000", "dist": "rm -rf dist && rollup -c", "watch": "rollup -wc", - "lint": "markdownlint README.v13.md && standard && tsc", - "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.v13.md --package ./package.json || true", + "lint": "markdownlint README.md && standard && tsc", + "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true", "serve-docs": "npm run docs && serve ./docs/", "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.js --repitition-time 1000", "postversion": "git push && git push --tags", @@ -38,31 +38,31 @@ }, "repository": { "type": "git", - "url": "https://github.com/y-js/yjs.git" + "url": "https://github.com/yjs/yjs.git" }, "keywords": [ "crdt" ], "author": "Kevin Jahns", - "email": "kevin.jahns@rwth-aachen.de", + "email": "kevin.jahns@protonmail.com", "license": "MIT", "bugs": { - "url": "https://github.com/y-js/yjs/issues" + "url": "https://github.com/yjs/yjs/issues" }, - "homepage": "http://y-js.org", + "homepage": "https://yjs.dev", "dependencies": { - "lib0": "0.0.5" + "lib0": "^0.1.1" }, "devDependencies": { "concurrently": "^3.6.1", - "jsdoc": "^3.6.2", + "jsdoc": "^3.6.3", "live-server": "^1.2.1", - "rollup": "^1.11.3", + "rollup": "^1.20.3", "rollup-cli": "^1.0.9", "rollup-plugin-node-resolve": "^4.2.4", "standard": "^11.0.1", "tui-jsdoc-template": "^1.2.2", - "typescript": "^3.4.5", + "typescript": "^3.6.2", "y-protocols": "0.0.6" } } diff --git a/src/index.js b/src/index.js index 73d4e76c..7ddcf46e 100644 --- a/src/index.js +++ b/src/index.js @@ -21,6 +21,7 @@ export { ContentEmbed, ContentFormat, ContentJSON, + ContentAny, ContentString, ContentType, AbstractType, @@ -36,6 +37,11 @@ export { compareIDs, getState, Snapshot, + createSnapshot, + createDeleteSet, + createDeleteSetFromStructStore, + snapshot, + emptySnapshot, findRootTypeKey, typeListToArraySnapshot, typeMapGetSnapshot, @@ -43,5 +49,10 @@ export { applyUpdate, encodeStateAsUpdate, encodeStateVector, - UndoManager + UndoManager, + decodeSnapshot, + encodeSnapshot, + isDeleted, + equalSnapshots, + PermanentUserData // @TODO experimental } from './internals.js' diff --git a/src/internals.js b/src/internals.js index 74ce0eba..288cb404 100644 --- a/src/internals.js +++ b/src/internals.js @@ -1,13 +1,16 @@ + export * from './utils/DeleteSet.js' +export * from './utils/Doc.js' +export * from './utils/encoding.js' export * from './utils/EventHandler.js' export * from './utils/ID.js' export * from './utils/isParentOf.js' +export * from './utils/PermanentUserData.js' export * from './utils/RelativePosition.js' export * from './utils/Snapshot.js' export * from './utils/StructStore.js' export * from './utils/Transaction.js' export * from './utils/UndoManager.js' -export * from './utils/Doc.js' export * from './utils/YEvent.js' export * from './types/AbstractType.js' @@ -27,8 +30,7 @@ export * from './structs/ContentDeleted.js' export * from './structs/ContentEmbed.js' export * from './structs/ContentFormat.js' export * from './structs/ContentJSON.js' +export * from './structs/ContentAny.js' export * from './structs/ContentString.js' export * from './structs/ContentType.js' export * from './structs/Item.js' - -export * from './utils/encoding.js' diff --git a/src/structs/ContentAny.js b/src/structs/ContentAny.js new file mode 100644 index 00000000..1367cbae --- /dev/null +++ b/src/structs/ContentAny.js @@ -0,0 +1,108 @@ +import { + Transaction, Item, StructStore // eslint-disable-line +} from '../internals.js' + +import * as encoding from 'lib0/encoding.js' +import * as decoding from 'lib0/decoding.js' + +/** + * @private + */ +export class ContentAny { + /** + * @param {Array<any>} arr + */ + constructor (arr) { + /** + * @type {Array<any>} + */ + this.arr = arr + } + /** + * @return {number} + */ + getLength () { + return this.arr.length + } + /** + * @return {Array<any>} + */ + getContent () { + return this.arr + } + /** + * @return {boolean} + */ + isCountable () { + return true + } + /** + * @return {ContentAny} + */ + copy () { + return new ContentAny(this.arr) + } + /** + * @param {number} offset + * @return {ContentAny} + */ + splice (offset) { + const right = new ContentAny(this.arr.slice(offset)) + this.arr = this.arr.slice(0, offset) + return right + } + /** + * @param {ContentAny} right + * @return {boolean} + */ + mergeWith (right) { + this.arr = this.arr.concat(right.arr) + return true + } + /** + * @param {Transaction} transaction + * @param {Item} item + */ + integrate (transaction, item) {} + /** + * @param {Transaction} transaction + */ + delete (transaction) {} + /** + * @param {StructStore} store + */ + gc (store) {} + /** + * @param {encoding.Encoder} encoder + * @param {number} offset + */ + write (encoder, offset) { + const len = this.arr.length + encoding.writeVarUint(encoder, len - offset) + for (let i = offset; i < len; i++) { + const c = this.arr[i] + encoding.writeAny(encoder, c) + } + } + /** + * @return {number} + */ + getRef () { + return 8 + } +} + +/** + * @private + * + * @param {decoding.Decoder} decoder + * @return {ContentAny} + */ +export const readContentAny = decoder => { + const len = decoding.readVarUint(decoder) + const cs = [] + for (let i = 0; i < len; i++) { + cs.push(decoding.readAny(decoder)) + } + return new ContentAny(cs) +} diff --git a/src/structs/Item.js b/src/structs/Item.js index 9d7e46fe..10e0e841 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -18,6 +18,7 @@ import { readContentDeleted, readContentBinary, readContentJSON, + readContentAny, readContentString, readContentEmbed, readContentFormat, @@ -34,6 +35,8 @@ import * as set from 'lib0/set.js' import * as binary from 'lib0/binary.js' /** + * @todo This should return several items + * * @param {StructStore} store * @param {ID} id * @return {{item:Item, diff:number}} @@ -134,7 +137,7 @@ export const splitItem = (transaction, leftItem, diff) => { */ export const redoItem = (transaction, item, redoitems) => { if (item.redone !== null) { - return getItemCleanStart(transaction, transaction.doc.store, item.redone) + return getItemCleanStart(transaction, item.redone) } let parentItem = item.parent._item /** @@ -174,7 +177,7 @@ export const redoItem = (transaction, item, redoitems) => { } if (parentItem !== null && parentItem.redone !== null) { while (parentItem.redone !== null) { - parentItem = getItemCleanStart(transaction, transaction.doc.store, parentItem.redone) + parentItem = getItemCleanStart(transaction, parentItem.redone) } // find next cloned_redo items while (left !== null) { @@ -184,7 +187,7 @@ export const redoItem = (transaction, item, redoitems) => { let leftTrace = left // trace redone until parent matches while (leftTrace !== null && leftTrace.parent._item !== parentItem) { - leftTrace = leftTrace.redone === null ? null : getItemCleanStart(transaction, transaction.doc.store, leftTrace.redone) + leftTrace = leftTrace.redone === null ? null : getItemCleanStart(transaction, leftTrace.redone) } if (leftTrace !== null && leftTrace.parent._item === parentItem) { left = leftTrace @@ -199,7 +202,7 @@ export const redoItem = (transaction, item, redoitems) => { let rightTrace = right // trace redone until parent matches while (rightTrace !== null && rightTrace.parent._item !== parentItem) { - rightTrace = rightTrace.redone === null ? null : getItemCleanStart(transaction, transaction.doc.store, rightTrace.redone) + rightTrace = rightTrace.redone === null ? null : getItemCleanStart(transaction, rightTrace.redone) } if (rightTrace !== null && rightTrace.parent._item === parentItem) { right = rightTrace @@ -561,7 +564,8 @@ export const contentRefs = [ readContentString, readContentEmbed, readContentFormat, - readContentType + readContentType, + readContentAny ] /** @@ -724,7 +728,7 @@ export class ItemRef extends AbstractStructRef { } const left = this.left === null ? null : getItemCleanEnd(transaction, store, this.left) - const right = this.right === null ? null : getItemCleanStart(transaction, store, this.right) + const right = this.right === null ? null : getItemCleanStart(transaction, this.right) let parent = null let parentSub = this.parentSub if (this.parent !== null) { diff --git a/src/types/AbstractType.js b/src/types/AbstractType.js index dd4ae2cd..1ec7f76a 100644 --- a/src/types/AbstractType.js +++ b/src/types/AbstractType.js @@ -7,7 +7,7 @@ import { nextID, isVisible, ContentType, - ContentJSON, + ContentAny, ContentBinary, createID, getItemCleanStart, @@ -30,7 +30,7 @@ import * as encoding from 'lib0/encoding.js' // eslint-disable-line * @param {EventType} event */ export const callTypeObservers = (type, transaction, event) => { - callEventHandlerListeners(type._eH, event, transaction) + const changedType = type const changedParentTypes = transaction.changedParentTypes while (true) { // @ts-ignore @@ -40,6 +40,7 @@ export const callTypeObservers = (type, transaction, event) => { } type = type._item.parent } + callEventHandlerListeners(changedType._eH, event, transaction) } /** @@ -374,7 +375,7 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem, let jsonContent = [] const packJsonContent = () => { if (jsonContent.length > 0) { - left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentJSON(jsonContent)) + left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentAny(jsonContent)) left.integrate(transaction) jsonContent = [] } @@ -428,7 +429,7 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => { if (index <= n.length) { if (index < n.length) { // insert in-between - getItemCleanStart(transaction, transaction.doc.store, createID(n.id.client, n.id.clock + index)) + getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index)) } break } @@ -454,7 +455,7 @@ export const typeListDelete = (transaction, parent, index, length) => { for (; n !== null && index > 0; n = n.right) { if (!n.deleted && n.countable) { if (index < n.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(n.id.client, n.id.clock + index)) + getItemCleanStart(transaction, createID(n.id.client, n.id.clock + index)) } index -= n.length } @@ -463,7 +464,7 @@ export const typeListDelete = (transaction, parent, index, length) => { while (length > 0 && n !== null) { if (!n.deleted) { if (length < n.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(n.id.client, n.id.clock + length)) + getItemCleanStart(transaction, createID(n.id.client, n.id.clock + length)) } n.delete(transaction) length -= n.length @@ -503,7 +504,7 @@ export const typeMapSet = (transaction, parent, key, value) => { const left = parent._map.get(key) || null let content if (value == null) { - content = new ContentJSON([value]) + content = new ContentAny([value]) } else { switch (value.constructor) { case Number: @@ -511,7 +512,7 @@ export const typeMapSet = (transaction, parent, key, value) => { case Boolean: case Array: case String: - content = new ContentJSON([value]) + content = new ContentAny([value]) break case Uint8Array: content = new ContentBinary(value) @@ -584,7 +585,7 @@ export const typeMapHas = (parent, key) => { */ export const typeMapGetSnapshot = (parent, key, snapshot) => { let v = parent._map.get(key) || null - while (v !== null && (!snapshot.sm.has(v.id.client) || v.id.clock >= (snapshot.sm.get(v.id.client) || 0))) { + while (v !== null && (!snapshot.sv.has(v.id.client) || v.id.clock >= (snapshot.sv.get(v.id.client) || 0))) { v = v.left } return v !== null && isVisible(v, snapshot) ? v.content.getContent()[v.length - 1] : undefined diff --git a/src/types/YText.js b/src/types/YText.js index 167841b9..2520f3ee 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -16,11 +16,20 @@ import { ContentEmbed, ContentFormat, ContentString, - Doc, Item, Snapshot, StructStore, Transaction // eslint-disable-line + splitSnapshotAffectedStructs, + ID, Doc, Item, Snapshot, Transaction // eslint-disable-line } from '../internals.js' import * as decoding from 'lib0/decoding.js' // eslint-disable-line import * as encoding from 'lib0/encoding.js' +import * as object from 'lib0/object.js' + +/** + * @param {any} a + * @param {any} b + * @return {boolean} + */ +const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b)) export class ItemListPosition { /** @@ -59,7 +68,6 @@ export class ItemInsertionResult extends ItemListPosition { /** * @param {Transaction} transaction - * @param {StructStore} store * @param {Map<string,any>} currentAttributes * @param {Item|null} left * @param {Item|null} right @@ -69,7 +77,7 @@ export class ItemInsertionResult extends ItemListPosition { * @private * @function */ -const findNextPosition = (transaction, store, currentAttributes, left, right, count) => { +const findNextPosition = (transaction, currentAttributes, left, right, count) => { while (right !== null && count > 0) { switch (right.content.constructor) { case ContentEmbed: @@ -77,7 +85,7 @@ const findNextPosition = (transaction, store, currentAttributes, left, right, co if (!right.deleted) { if (count < right.length) { // split right - getItemCleanStart(transaction, store, createID(right.id.client, right.id.clock + count)) + getItemCleanStart(transaction, createID(right.id.client, right.id.clock + count)) } count -= right.length } @@ -96,7 +104,6 @@ const findNextPosition = (transaction, store, currentAttributes, left, right, co /** * @param {Transaction} transaction - * @param {StructStore} store * @param {AbstractType<any>} parent * @param {number} index * @return {ItemTextListPosition} @@ -104,11 +111,11 @@ const findNextPosition = (transaction, store, currentAttributes, left, right, co * @private * @function */ -const findPosition = (transaction, store, parent, index) => { +const findPosition = (transaction, parent, index) => { let currentAttributes = new Map() let left = null let right = parent._start - return findNextPosition(transaction, store, currentAttributes, left, right, index) + return findNextPosition(transaction, currentAttributes, left, right, index) } /** @@ -130,7 +137,7 @@ const insertNegatedAttributes = (transaction, parent, left, right, negatedAttrib right !== null && ( right.deleted === true || ( right.content.constructor === ContentFormat && - (negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key) === /** @type {ContentFormat} */ (right.content).value) + equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key), /** @type {ContentFormat} */ (right.content).value) ) ) ) { @@ -180,7 +187,7 @@ const minimizeAttributeChanges = (left, right, currentAttributes, attributes) => break } else if (right.deleted) { // continue - } else if (right.content.constructor === ContentFormat && (attributes[(/** @type {ContentFormat} */ (right.content)).key] || null) === /** @type {ContentFormat} */ (right.content).value) { + } else if (right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (right.content)).key] || null, /** @type {ContentFormat} */ (right.content).value)) { // found a format, update currentAttributes and continue updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content)) } else { @@ -210,7 +217,7 @@ const insertAttributes = (transaction, parent, left, right, currentAttributes, a for (let key in attributes) { const val = attributes[key] const currentVal = currentAttributes.get(key) || null - if (currentVal !== val) { + if (!equalAttrs(currentVal, val)) { // save negated attribute (set null if currentVal undefined) negatedAttributes.set(key, currentVal) left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentFormat(key, val)) @@ -272,13 +279,13 @@ const formatText = (transaction, parent, left, right, currentAttributes, length, // iterate until first non-format or null is found // delete all formats with attributes[format.key] != null while (length > 0 && right !== null) { - if (right.deleted === false) { + if (!right.deleted) { switch (right.content.constructor) { case ContentFormat: const { key, value } = /** @type {ContentFormat} */ (right.content) const attr = attributes[key] if (attr !== undefined) { - if (attr === value) { + if (equalAttrs(attr, value)) { negatedAttributes.delete(key) } else { negatedAttributes.set(key, value) @@ -290,7 +297,7 @@ const formatText = (transaction, parent, left, right, currentAttributes, length, case ContentEmbed: case ContentString: if (length < right.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(right.id.client, right.id.clock + length)) + getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length)) } length -= right.length break @@ -334,7 +341,7 @@ const deleteText = (transaction, left, right, currentAttributes, length) => { case ContentEmbed: case ContentString: if (length < right.length) { - getItemCleanStart(transaction, transaction.doc.store, createID(right.id.client, right.id.clock + length)) + getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length)) } length -= right.length right.delete(transaction) @@ -516,11 +523,11 @@ export class YTextEvent extends YEvent { if (this.adds(item)) { if (!this.deletes(item)) { const curVal = currentAttributes.get(key) || null - if (curVal !== value) { + if (!equalAttrs(curVal, value)) { if (action === 'retain') { addOp() } - if (value === (oldAttributes.get(key) || null)) { + if (equalAttrs(value, (oldAttributes.get(key) || null))) { delete attributes[key] } else { attributes[key] = value @@ -532,7 +539,7 @@ export class YTextEvent extends YEvent { } else if (this.deletes(item)) { oldAttributes.set(key, value) const curVal = currentAttributes.get(key) || null - if (curVal !== value) { + if (!equalAttrs(curVal, value)) { if (action === 'retain') { addOp() } @@ -542,7 +549,7 @@ export class YTextEvent extends YEvent { oldAttributes.set(key, value) const attr = attributes[key] if (attr !== undefined) { - if (attr !== value) { + if (!equalAttrs(attr, value)) { if (action === 'retain') { addOp() } @@ -705,16 +712,18 @@ export class YText extends AbstractType { * * @param {Snapshot} [snapshot] * @param {Snapshot} [prevSnapshot] + * @param {function('removed' | 'added', ID):any} [computeYChange] * @return {any} The Delta representation of this type. * * @public */ - toDelta (snapshot, prevSnapshot) { + toDelta (snapshot, prevSnapshot, computeYChange) { /** * @type{Array<any>} */ const ops = [] const currentAttributes = new Map() + const doc = /** @type {Doc} */ (this.doc) let str = '' let n = this._start function packStr () { @@ -740,42 +749,54 @@ export class YText extends AbstractType { str = '' } } - while (n !== null) { - if (isVisible(n, snapshot) || (prevSnapshot !== undefined && isVisible(n, prevSnapshot))) { - switch (n.content.constructor) { - case ContentString: - const cur = currentAttributes.get('ychange') - if (snapshot !== undefined && !isVisible(n, snapshot)) { - if (cur === undefined || cur.user !== n.id.client || cur.state !== 'removed') { - packStr() - currentAttributes.set('ychange', { user: n.id.client, state: 'removed' }) - } - } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { - if (cur === undefined || cur.user !== n.id.client || cur.state !== 'added') { - packStr() - currentAttributes.set('ychange', { user: n.id.client, state: 'added' }) - } - } else if (cur !== undefined) { - packStr() - currentAttributes.delete('ychange') - } - str += /** @type {ContentString} */ (n.content).str - break - case ContentEmbed: - packStr() - ops.push({ - insert: /** @type {ContentEmbed} */ (n.content).embed - }) - break - case ContentFormat: - packStr() - updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (n.content)) - break - } + // snapshots are merged again after the transaction, so we need to keep the + // transalive until we are done + transact(doc, transaction => { + if (snapshot) { + splitSnapshotAffectedStructs(transaction, snapshot) } - n = n.right - } - packStr() + if (prevSnapshot) { + splitSnapshotAffectedStructs(transaction, prevSnapshot) + } + while (n !== null) { + if (isVisible(n, snapshot) || (prevSnapshot !== undefined && isVisible(n, prevSnapshot))) { + switch (n.content.constructor) { + case ContentString: + const cur = currentAttributes.get('ychange') + if (snapshot !== undefined && !isVisible(n, snapshot)) { + if (cur === undefined || cur.user !== n.id.client || cur.state !== 'removed') { + packStr() + currentAttributes.set('ychange', computeYChange ? computeYChange('removed', n.id) : { type: 'removed' }) + } + } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { + if (cur === undefined || cur.user !== n.id.client || cur.state !== 'added') { + packStr() + currentAttributes.set('ychange', computeYChange ? computeYChange('added', n.id) : { type: 'added' }) + } + } else if (cur !== undefined) { + packStr() + currentAttributes.delete('ychange') + } + str += /** @type {ContentString} */ (n.content).str + break + case ContentEmbed: + packStr() + ops.push({ + insert: /** @type {ContentEmbed} */ (n.content).embed + }) + break + case ContentFormat: + if (isVisible(n, snapshot)) { + packStr() + updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (n.content)) + } + break + } + } + n = n.right + } + packStr() + }, splitSnapshotAffectedStructs) return ops } @@ -784,19 +805,23 @@ export class YText extends AbstractType { * * @param {number} index The index at which to start inserting. * @param {String} text The text to insert at the specified position. - * @param {TextAttributes} attributes Optionally define some formatting + * @param {TextAttributes} [attributes] Optionally define some formatting * information to apply on the inserted * Text. * @public */ - insert (index, text, attributes = {}) { + insert (index, text, attributes) { if (text.length <= 0) { return } const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + const { left, right, currentAttributes } = findPosition(transaction, this, index) + if (!attributes) { + attributes = {} + currentAttributes.forEach((v, k) => { attributes[k] = v }) + } insertText(transaction, this, left, right, currentAttributes, text, attributes) }) } else { @@ -821,7 +846,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + const { left, right, currentAttributes } = findPosition(transaction, this, index) insertText(transaction, this, left, right, currentAttributes, embed, attributes) }) } else { @@ -844,7 +869,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - const { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + const { left, right, currentAttributes } = findPosition(transaction, this, index) deleteText(transaction, left, right, currentAttributes, length) }) } else { @@ -866,7 +891,7 @@ export class YText extends AbstractType { const y = this.doc if (y !== null) { transact(y, transaction => { - let { left, right, currentAttributes } = findPosition(transaction, y.store, this, index) + let { left, right, currentAttributes } = findPosition(transaction, this, index) if (right === null) { return } diff --git a/src/types/YXmlFragment.js b/src/types/YXmlFragment.js index 62fa543e..247f4b1b 100644 --- a/src/types/YXmlFragment.js +++ b/src/types/YXmlFragment.js @@ -151,6 +151,10 @@ export class YXmlFragment extends AbstractType { return new YXmlFragment() } + get length () { + return this._prelimContent === null ? this._length : this._prelimContent.length + } + /** * Create a subtree of childNodes. * diff --git a/src/types/YXmlText.js b/src/types/YXmlText.js index 6c4088db..e2a51e1e 100644 --- a/src/types/YXmlText.js +++ b/src/types/YXmlText.js @@ -56,7 +56,7 @@ export class YXmlText extends YText { const node = nestedNodes[i] str += `<${node.nodeName}` for (let j = 0; j < node.attrs.length; j++) { - const attr = node.attrs[i] + const attr = node.attrs[j] str += ` ${attr.key}="${attr.value}"` } str += '>' diff --git a/src/utils/DeleteSet.js b/src/utils/DeleteSet.js index 747d8162..16c4a2ba 100644 --- a/src/utils/DeleteSet.js +++ b/src/utils/DeleteSet.js @@ -8,6 +8,7 @@ import { Item, GC, StructStore, Transaction, ID // eslint-disable-line } from '../internals.js' +import * as array from 'lib0/array.js' import * as math from 'lib0/math.js' import * as map from 'lib0/map.js' import * as encoding from 'lib0/encoding.js' @@ -52,14 +53,13 @@ export class DeleteSet { * * @param {Transaction} transaction * @param {DeleteSet} ds - * @param {StructStore} store * @param {function(GC|Item):void} f * * @function */ -export const iterateDeletedStructs = (transaction, ds, store, f) => +export const iterateDeletedStructs = (transaction, ds, f) => ds.clients.forEach((deletes, clientid) => { - const structs = /** @type {Array<GC|Item>} */ (store.clients.get(clientid)) + const structs = /** @type {Array<GC|Item>} */ (transaction.doc.store.clients.get(clientid)) for (let i = 0; i < deletes.length; i++) { const del = deletes[i] iterateStructs(transaction, structs, del.clock, del.len, f) @@ -137,22 +137,27 @@ export const sortAndMergeDeleteSet = ds => { } /** - * @param {DeleteSet} ds1 - * @param {DeleteSet} ds2 + * @param {Array<DeleteSet>} dss * @return {DeleteSet} A fresh DeleteSet */ -export const mergeDeleteSets = (ds1, ds2) => { +export const mergeDeleteSets = dss => { const merged = new DeleteSet() - // Write all keys from ds1 to merged. If ds2 has the same key, combine the sets. - ds1.clients.forEach((dels1, client) => - merged.clients.set(client, dels1.concat(ds2.clients.get(client) || [])) - ) - // Write all missing keys from ds2 to merged. - ds2.clients.forEach((dels2, client) => { - if (!merged.clients.has(client)) { - merged.clients.set(client, dels2) - } - }) + for (let dssI = 0; dssI < dss.length; dssI++) { + dss[dssI].clients.forEach((delsLeft, client) => { + if (!merged.clients.has(client)) { + // Write all missing keys from current ds and all following. + // If merged already contains `client` current ds has already been added. + /** + * @type {Array<DeleteItem>} + */ + const dels = delsLeft.slice() + for (let i = dssI + 1; i < dss.length; i++) { + array.appendTo(dels, dss[i].clients.get(client) || []) + } + merged.clients.set(client, dels) + } + }) + } sortAndMergeDeleteSet(merged) return merged } @@ -169,6 +174,8 @@ export const addToDeleteSet = (ds, id, length) => { map.setIfUndefined(ds.clients, id.client, () => []).push(new DeleteItem(id.clock, length)) } +export const createDeleteSet = () => new DeleteSet() + /** * @param {StructStore} ss * @return {DeleteSet} Merged and sorted DeleteSet @@ -177,7 +184,7 @@ export const addToDeleteSet = (ds, id, length) => { * @function */ export const createDeleteSetFromStructStore = ss => { - const ds = new DeleteSet() + const ds = createDeleteSet() ss.clients.forEach((structs, client) => { /** * @type {Array<DeleteItem>} @@ -224,6 +231,26 @@ export const writeDeleteSet = (encoder, ds) => { }) } +/** + * @param {decoding.Decoder} decoder + * @return {DeleteSet} + * + * @private + * @function + */ +export const readDeleteSet = decoder => { + const ds = new DeleteSet() + const numClients = decoding.readVarUint(decoder) + for (let i = 0; i < numClients; i++) { + const client = decoding.readVarUint(decoder) + const numberOfDeletes = decoding.readVarUint(decoder) + for (let i = 0; i < numberOfDeletes; i++) { + addToDeleteSet(ds, createID(client, decoding.readVarUint(decoder)), decoding.readVarUint(decoder)) + } + } + return ds +} + /** * @param {decoding.Decoder} decoder * @param {Transaction} transaction @@ -232,7 +259,7 @@ export const writeDeleteSet = (encoder, ds) => { * @private * @function */ -export const readDeleteSet = (decoder, transaction, store) => { +export const readAndApplyDeleteSet = (decoder, transaction, store) => { const unappliedDS = new DeleteSet() const numClients = decoding.readVarUint(decoder) for (let i = 0; i < numClients; i++) { @@ -279,6 +306,7 @@ export const readDeleteSet = (decoder, transaction, store) => { } } if (unappliedDS.clients.size > 0) { + // TODO: no need for encoding+decoding ds anymore const unappliedDSEncoder = encoding.createEncoder() writeDeleteSet(unappliedDSEncoder, unappliedDS) store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toUint8Array(unappliedDSEncoder))) diff --git a/src/utils/PermanentUserData.js b/src/utils/PermanentUserData.js new file mode 100644 index 00000000..289a3755 --- /dev/null +++ b/src/utils/PermanentUserData.js @@ -0,0 +1,134 @@ + +import { + YArray, + YMap, + readDeleteSet, + writeDeleteSet, + createDeleteSet, + ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line +} from '../internals.js' + +import * as decoding from 'lib0/decoding.js' +import * as encoding from 'lib0/encoding.js' +import { mergeDeleteSets, isDeleted } from './DeleteSet.js' + +export class PermanentUserData { + /** + * @param {Doc} doc + * @param {string} key + */ + constructor (doc, key = 'users') { + const users = doc.getMap(key) + /** + * @type {Map<string,DeleteSet>} + */ + const dss = new Map() + this.yusers = users + this.doc = doc + /** + * Maps from clientid to userDescription + * + * @type {Map<number,string>} + */ + this.clients = new Map() + this.dss = dss + /** + * @param {YMap<any>} user + * @param {string} userDescription + */ + const initUser = (user, userDescription) => { + /** + * @type {YArray<Uint8Array>} + */ + const ds = user.get('ds') + const ids = user.get('ids') + const addClientId = /** @param {number} clientid */ clientid => this.clients.set(clientid, userDescription) + ds.observe(/** @param {YArrayEvent<any>} event */ event => { + event.changes.added.forEach(item => { + item.content.getContent().forEach(encodedDs => { + if (encodedDs instanceof Uint8Array) { + this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(decoding.createDecoder(encodedDs))])) + } + }) + }) + }) + this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(decoding.createDecoder(encodedDs))))) + ids.observe(/** @param {YArrayEvent<any>} event */ event => + event.changes.added.forEach(item => item.content.getContent().forEach(addClientId)) + ) + ids.forEach(addClientId) + } + // observe users + users.observe(event => { + event.keysChanged.forEach(userDescription => + initUser(users.get(userDescription), userDescription) + ) + }) + // add intial data + users.forEach(initUser) + } + /** + * @param {Doc} doc + * @param {number} clientid + * @param {string} userDescription + */ + setUserMapping (doc, clientid, userDescription) { + const users = this.yusers + let user = users.get(userDescription) + if (!user) { + user = new YMap() + user.set('ids', new YArray()) + user.set('ds', new YArray()) + users.set(userDescription, user) + } + user.get('ids').push([clientid]) + users.observe(event => { + const userOverwrite = users.get(userDescription) + if (userOverwrite !== user) { + // user was overwritten, port all data over to the next user object + // @todo Experiment with Y.Sets here + user = userOverwrite + // @todo iterate over old type + this.clients.forEach((_userDescription, clientid) => { + if (userDescription === _userDescription) { + user.get('ids').push([clientid]) + } + }) + const encoder = encoding.createEncoder() + const ds = this.dss.get(userDescription) + if (ds) { + writeDeleteSet(encoder, ds) + user.get('ds').push([encoding.toUint8Array(encoder)]) + } + } + }) + doc.on('afterTransaction', /** @param {Transaction} transaction */ transaction => { + const yds = user.get('ds') + const ds = transaction.deleteSet + if (transaction.local && ds.clients.size > 0) { + const encoder = encoding.createEncoder() + writeDeleteSet(encoder, ds) + yds.push([encoding.toUint8Array(encoder)]) + } + }) + } + /** + * @param {number} clientid + * @return {any} + */ + getUserByClientId (clientid) { + return this.clients.get(clientid) || null + } + /** + * @param {ID} id + * @return {string | null} + */ + getUserByDeletedId (id) { + for (const [userDescription, ds] of this.dss) { + if (isDeleted(ds, id)) { + return userDescription + } + } + return null + } +} diff --git a/src/utils/RelativePosition.js b/src/utils/RelativePosition.js index d2f7eb10..333111ed 100644 --- a/src/utils/RelativePosition.js +++ b/src/utils/RelativePosition.js @@ -228,7 +228,7 @@ export const createAbsolutePositionFromRelativePosition = (rpos, doc) => { return null } type = right.parent - if (type._item !== null && !type._item.deleted) { + if (type._item === null || !type._item.deleted) { index = right.deleted || !right.countable ? 0 : res.diff let n = right.left while (n !== null) { diff --git a/src/utils/Snapshot.js b/src/utils/Snapshot.js index 2d33941f..716ddeed 100644 --- a/src/utils/Snapshot.js +++ b/src/utils/Snapshot.js @@ -1,15 +1,31 @@ import { isDeleted, - DeleteSet, Item // eslint-disable-line + createDeleteSetFromStructStore, + getStateVector, + getItemCleanStart, + createID, + iterateDeletedStructs, + writeDeleteSet, + writeStateVector, + readDeleteSet, + readStateVector, + createDeleteSet, + getState, + Transaction, Doc, DeleteSet, Item // eslint-disable-line } from '../internals.js' +import * as map from 'lib0/map.js' +import * as set from 'lib0/set.js' +import * as encoding from 'lib0/encoding.js' +import * as decoding from 'lib0/decoding.js' + export class Snapshot { /** * @param {DeleteSet} ds - * @param {Map<number,number>} sm state map + * @param {Map<number,number>} sv state map */ - constructor (ds, sm) { + constructor (ds, sv) { /** * @type {DeleteSet} * @private @@ -20,16 +36,79 @@ export class Snapshot { * @type {Map<number,number>} * @private */ - this.sm = sm + this.sv = sv } } +/** + * @param {Snapshot} snap1 + * @param {Snapshot} snap2 + * @return {boolean} + */ +export const equalSnapshots = (snap1, snap2) => { + const ds1 = snap1.ds.clients + const ds2 = snap2.ds.clients + const sv1 = snap1.sv + const sv2 = snap2.sv + if (sv1.size !== sv2.size || ds1.size !== ds2.size) { + return false + } + for (const [key, value] of sv1) { + if (sv2.get(key) !== value) { + return false + } + } + for (const [client, dsitems1] of ds1) { + const dsitems2 = ds2.get(client) || [] + if (dsitems1.length !== dsitems2.length) { + return false + } + for (let i = 0; i < dsitems1.length; i++) { + const dsitem1 = dsitems1[i] + const dsitem2 = dsitems2[i] + if (dsitem1.clock !== dsitem2.clock || dsitem1.len !== dsitem2.len) { + return false + } + } + } + return true +} + +/** + * @param {Snapshot} snapshot + * @return {Uint8Array} + */ +export const encodeSnapshot = snapshot => { + const encoder = encoding.createEncoder() + writeDeleteSet(encoder, snapshot.ds) + writeStateVector(encoder, snapshot.sv) + return encoding.toUint8Array(encoder) +} + +/** + * @param {Uint8Array} buf + * @return {Snapshot} + */ +export const decodeSnapshot = buf => { + const decoder = decoding.createDecoder(buf) + return new Snapshot(readDeleteSet(decoder), readStateVector(decoder)) +} + /** * @param {DeleteSet} ds * @param {Map<number,number>} sm + * @return {Snapshot} */ export const createSnapshot = (ds, sm) => new Snapshot(ds, sm) +export const emptySnapshot = createSnapshot(createDeleteSet(), new Map()) + +/** + * @param {Doc} doc + * @return {Snapshot} + */ +export const snapshot = doc => createSnapshot(createDeleteSetFromStructStore(doc.store), getStateVector(doc.store)) + /** * @param {Item} item * @param {Snapshot|undefined} snapshot @@ -38,5 +117,24 @@ export const createSnapshot = (ds, sm) => new Snapshot(ds, sm) * @function */ export const isVisible = (item, snapshot) => snapshot === undefined ? !item.deleted : ( - snapshot.sm.has(item.id.client) && (snapshot.sm.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item.id) + snapshot.sv.has(item.id.client) && (snapshot.sv.get(item.id.client) || 0) > item.id.clock && !isDeleted(snapshot.ds, item.id) ) + +/** + * @param {Transaction} transaction + * @param {Snapshot} snapshot + */ +export const splitSnapshotAffectedStructs = (transaction, snapshot) => { + const meta = map.setIfUndefined(transaction.meta, splitSnapshotAffectedStructs, set.create) + const store = transaction.doc.store + // check if we already split for this snapshot + if (!meta.has(snapshot)) { + snapshot.sv.forEach((clock, client) => { + if (clock < getState(store, client)) { + getItemCleanStart(transaction, createID(client, clock)) + } + }) + iterateDeletedStructs(transaction, snapshot.ds, item => {}) + meta.add(snapshot) + } +} diff --git a/src/utils/StructStore.js b/src/utils/StructStore.js index 84dd4b93..969acfa4 100644 --- a/src/utils/StructStore.js +++ b/src/utils/StructStore.js @@ -197,16 +197,15 @@ export const findIndexCleanStart = (transaction, structs, clock) => { * Expects that id is actually in store. This function throws or is an infinite loop otherwise. * * @param {Transaction} transaction - * @param {StructStore} store * @param {ID} id * @return {Item} * * @private * @function */ -export const getItemCleanStart = (transaction, store, id) => { - const structs = /** @type {Array<GC|Item>} */ (store.clients.get(id.client)) - return /** @type {Item} */ (structs[findIndexCleanStart(transaction, structs, id.clock)]) +export const getItemCleanStart = (transaction, id) => { + const structs = /** @type {Array<Item>} */ (transaction.doc.store.clients.get(id.client)) + return structs[findIndexCleanStart(transaction, structs, id.clock)] } /** diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index a844a32c..55ce90cc 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -17,6 +17,7 @@ import * as encoding from 'lib0/encoding.js' import * as map from 'lib0/map.js' import * as math from 'lib0/math.js' import * as set from 'lib0/set.js' +import { callAll } from 'lib0/function.js' /** * A transaction is created for every change on the Yjs model. It is possible @@ -46,8 +47,9 @@ export class Transaction { /** * @param {Doc} doc * @param {any} origin + * @param {boolean} local */ - constructor (doc, origin) { + constructor (doc, origin, local) { /** * The Yjs instance. * @type {Doc} @@ -90,6 +92,16 @@ export class Transaction { * @type {any} */ this.origin = origin + /** + * Stores meta information on the transaction + * @type {Map<any,any>} + */ + this.meta = new Map() + /** + * Whether this change originates from this doc. + * @type {boolean} + */ + this.local = local } } @@ -133,22 +145,180 @@ export const addChangedTypeToTransaction = (transaction, type, parentSub) => { } } +/** + * @param {Array<Transaction>} transactionCleanups + * @param {number} i + */ +const cleanupTransactions = (transactionCleanups, i) => { + if (i < transactionCleanups.length) { + const transaction = transactionCleanups[i] + const doc = transaction.doc + const store = doc.store + const ds = transaction.deleteSet + try { + sortAndMergeDeleteSet(ds) + transaction.afterState = getStateVector(transaction.doc.store) + doc._transaction = null + doc.emit('beforeObserverCalls', [transaction, doc]) + /** + * An array of event callbacks. + * + * Each callback is called even if the other ones throw errors. + * + * @type {Array<function():void>} + */ + const fs = [] + // observe events on changed types + transaction.changed.forEach((subs, itemtype) => + fs.push(() => { + if (itemtype._item === null || !itemtype._item.deleted) { + itemtype._callObserver(transaction, subs) + } + }) + ) + fs.push(() => { + // deep observe events + transaction.changedParentTypes.forEach((events, type) => + fs.push(() => { + // We need to think about the possibility that the user transforms the + // Y.Doc in the event. + if (type._item === null || !type._item.deleted) { + events = events + .filter(event => + event.target._item === null || !event.target._item.deleted + ) + events + .forEach(event => { + event.currentTarget = type + }) + // We don't need to check for events.length + // because we know it has at least one element + callEventHandlerListeners(type._dEH, events, transaction) + } + }) + ) + fs.push(() => doc.emit('afterTransaction', [transaction, doc])) + }) + callAll(fs, []) + } finally { + /** + * @param {Array<AbstractStruct>} structs + * @param {number} pos + */ + const tryToMergeWithLeft = (structs, pos) => { + const left = structs[pos - 1] + const right = structs[pos] + if (left.deleted === right.deleted && left.constructor === right.constructor) { + if (left.mergeWith(right)) { + structs.splice(pos, 1) + if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) { + right.parent._map.set(right.parentSub, /** @type {Item} */ (left)) + } + } + } + } + // Replace deleted items with ItemDeleted / GC. + // This is where content is actually remove from the Yjs Doc. + if (doc.gc) { + for (const [client, deleteItems] of ds.clients) { + const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) + for (let di = deleteItems.length - 1; di >= 0; di--) { + const deleteItem = deleteItems[di] + const endDeleteItemClock = deleteItem.clock + deleteItem.len + for ( + let si = findIndexSS(structs, deleteItem.clock), struct = structs[si]; + si < structs.length && struct.id.clock < endDeleteItemClock; + struct = structs[++si] + ) { + const struct = structs[si] + if (deleteItem.clock + deleteItem.len <= struct.id.clock) { + break + } + if (struct instanceof Item && struct.deleted && !struct.keep) { + struct.gc(store, false) + } + } + } + } + } + // try to merge deleted / gc'd items + // merge from right to left for better efficiecy and so we don't miss any merge targets + for (const [client, deleteItems] of ds.clients) { + const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) + for (let di = deleteItems.length - 1; di >= 0; di--) { + const deleteItem = deleteItems[di] + // start with merging the item next to the last deleted item + const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1)) + for ( + let si = mostRightIndexToCheck, struct = structs[si]; + si > 0 && struct.id.clock >= deleteItem.clock; + struct = structs[--si] + ) { + tryToMergeWithLeft(structs, si) + } + } + } + + // on all affected store.clients props, try to merge + for (const [client, clock] of transaction.afterState) { + const beforeClock = transaction.beforeState.get(client) || 0 + if (beforeClock !== clock) { + const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) + // we iterate from right to left so we can safely remove entries + const firstChangePos = math.max(findIndexSS(structs, beforeClock), 1) + for (let i = structs.length - 1; i >= firstChangePos; i--) { + tryToMergeWithLeft(structs, i) + } + } + } + // try to merge mergeStructs + // @todo: it makes more sense to transform mergeStructs to a DS, sort it, and merge from right to left + // but at the moment DS does not handle duplicates + for (const mid of transaction._mergeStructs) { + const client = mid.client + const clock = mid.clock + const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) + const replacedStructPos = findIndexSS(structs, clock) + if (replacedStructPos + 1 < structs.length) { + tryToMergeWithLeft(structs, replacedStructPos + 1) + } + if (replacedStructPos > 0) { + tryToMergeWithLeft(structs, replacedStructPos) + } + } + // @todo Merge all the transactions into one and provide send the data as a single update message + doc.emit('afterTransactionCleanup', [transaction, doc]) + if (doc._observers.has('update')) { + const updateMessage = computeUpdateMessageFromTransaction(transaction) + if (updateMessage !== null) { + doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc]) + } + } + if (transactionCleanups.length <= i + 1) { + doc._transactionCleanups = [] + } else { + cleanupTransactions(transactionCleanups, i + 1) + } + } + } +} + /** * Implements the functionality of `y.transact(()=>{..})` * * @param {Doc} doc * @param {function(Transaction):void} f - * @param {any} [origin] + * @param {any} [origin=true] * * @private * @function */ -export const transact = (doc, f, origin = null) => { +export const transact = (doc, f, origin = null, local = true) => { const transactionCleanups = doc._transactionCleanups let initialCall = false if (doc._transaction === null) { initialCall = true - doc._transaction = new Transaction(doc, origin) + doc._transaction = new Transaction(doc, origin, local) transactionCleanups.push(doc._transaction) doc.emit('beforeTransaction', [doc._transaction, doc]) } @@ -158,134 +328,13 @@ export const transact = (doc, f, origin = null) => { if (initialCall && transactionCleanups[0] === doc._transaction) { // The first transaction ended, now process observer calls. // Observer call may create new transactions for which we need to call the observers and do cleanup. - // We don't want to nest these calls, so we execute these calls one after another - for (let i = 0; i < transactionCleanups.length; i++) { - const transaction = transactionCleanups[i] - const store = transaction.doc.store - const ds = transaction.deleteSet - sortAndMergeDeleteSet(ds) - transaction.afterState = getStateVector(transaction.doc.store) - doc._transaction = null - doc.emit('beforeObserverCalls', [transaction, doc]) - // emit change events on changed types - transaction.changed.forEach((subs, itemtype) => { - if (itemtype._item === null || !itemtype._item.deleted) { - itemtype._callObserver(transaction, subs) - } - }) - transaction.changedParentTypes.forEach((events, type) => { - // We need to think about the possibility that the user transforms the - // Y.Doc in the event. - if (type._item === null || !type._item.deleted) { - events = events - .filter(event => - event.target._item === null || !event.target._item.deleted - ) - events - .forEach(event => { - event.currentTarget = type - }) - // We don't need to check for events.length - // because we know it has at least one element - callEventHandlerListeners(type._dEH, events, transaction) - } - }) - doc.emit('afterTransaction', [transaction, doc]) - /** - * @param {Array<AbstractStruct>} structs - * @param {number} pos - */ - const tryToMergeWithLeft = (structs, pos) => { - const left = structs[pos - 1] - const right = structs[pos] - if (left.deleted === right.deleted && left.constructor === right.constructor) { - if (left.mergeWith(right)) { - structs.splice(pos, 1) - if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) { - right.parent._map.set(right.parentSub, /** @type {Item} */ (left)) - } - } - } - } - // Replace deleted items with ItemDeleted / GC. - // This is where content is actually remove from the Yjs Doc. - if (doc.gc) { - for (const [client, deleteItems] of ds.clients) { - const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) - for (let di = deleteItems.length - 1; di >= 0; di--) { - const deleteItem = deleteItems[di] - const endDeleteItemClock = deleteItem.clock + deleteItem.len - for ( - let si = findIndexSS(structs, deleteItem.clock), struct = structs[si]; - si < structs.length && struct.id.clock < endDeleteItemClock; - struct = structs[++si] - ) { - const struct = structs[si] - if (deleteItem.clock + deleteItem.len <= struct.id.clock) { - break - } - if (struct instanceof Item && struct.deleted && !struct.keep) { - struct.gc(store, false) - } - } - } - } - } - // try to merge deleted / gc'd items - // merge from right to left for better efficiecy and so we don't miss any merge targets - for (const [client, deleteItems] of ds.clients) { - const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) - for (let di = deleteItems.length - 1; di >= 0; di--) { - const deleteItem = deleteItems[di] - // start with merging the item next to the last deleted item - const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1)) - for ( - let si = mostRightIndexToCheck, struct = structs[si]; - si > 0 && struct.id.clock >= deleteItem.clock; - struct = structs[--si] - ) { - tryToMergeWithLeft(structs, si) - } - } - } - - // on all affected store.clients props, try to merge - for (const [client, clock] of transaction.afterState) { - const beforeClock = transaction.beforeState.get(client) || 0 - if (beforeClock !== clock) { - const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) - // we iterate from right to left so we can safely remove entries - const firstChangePos = math.max(findIndexSS(structs, beforeClock), 1) - for (let i = structs.length - 1; i >= firstChangePos; i--) { - tryToMergeWithLeft(structs, i) - } - } - } - // try to merge mergeStructs - // @todo: it makes more sense to transform mergeStructs to a DS, sort it, and merge from right to left - // but at the moment DS does not handle duplicates - for (const mid of transaction._mergeStructs) { - const client = mid.client - const clock = mid.clock - const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client)) - const replacedStructPos = findIndexSS(structs, clock) - if (replacedStructPos + 1 < structs.length) { - tryToMergeWithLeft(structs, replacedStructPos + 1) - } - if (replacedStructPos > 0) { - tryToMergeWithLeft(structs, replacedStructPos) - } - } - // @todo Merge all the transactions into one and provide send the data as a single update message - doc.emit('afterTransactionCleanup', [transaction, doc]) - if (doc._observers.has('update')) { - const updateMessage = computeUpdateMessageFromTransaction(transaction) - if (updateMessage !== null) { - doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc]) - } - } - } - doc._transactionCleanups = [] + // We don't want to nest these calls, so we execute these calls one after + // another. + // Also we need to ensure that all cleanups are called, even if the + // observes throw errors. + // This file is full of hacky try {} finally {} blocks to ensure that an + // event can throw errors and also that the cleanup is called. + cleanupTransactions(transactionCleanups, 0) } } } diff --git a/src/utils/UndoManager.js b/src/utils/UndoManager.js index d62f06dc..667cc9ac 100644 --- a/src/utils/UndoManager.js +++ b/src/utils/UndoManager.js @@ -9,6 +9,7 @@ import { createID, followRedone, getItemCleanStart, + getState, Transaction, Doc, Item, GC, DeleteSet, AbstractType // eslint-disable-line } from '../internals.js' @@ -49,35 +50,64 @@ const popStackItem = (undoManager, stack, eventType) => { transact(doc, transaction => { while (stack.length > 0 && result === null) { const store = doc.store + const clientID = doc.clientID const stackItem = /** @type {StackItem} */ (stack.pop()) + const stackStartClock = stackItem.start + const stackEndClock = stackItem.start + stackItem.len const itemsToRedo = new Set() + // @todo iterateStructs should not need the structs parameter + const structs = /** @type {Array<GC|Item>} */ (store.clients.get(clientID)) let performedChange = false - iterateDeletedStructs(transaction, stackItem.ds, store, struct => { - if (struct instanceof Item && scope.some(type => isParentOf(type, struct))) { + if (stackStartClock !== stackEndClock) { + // make sure structs don't overlap with the range of created operations [stackItem.start, stackItem.start + stackItem.end) + getItemCleanStart(transaction, createID(clientID, stackStartClock)) + if (stackEndClock < getState(doc.store, clientID)) { + getItemCleanStart(transaction, createID(clientID, stackEndClock)) + } + } + iterateDeletedStructs(transaction, stackItem.ds, struct => { + if ( + struct instanceof Item && + scope.some(type => isParentOf(type, struct)) && + // Never redo structs in [stackItem.start, stackItem.start + stackItem.end) because they were created and deleted in the same capture interval. + !(struct.id.client === clientID && struct.id.clock >= stackStartClock && struct.id.clock < stackEndClock) + ) { itemsToRedo.add(struct) } }) - itemsToRedo.forEach(item => { - performedChange = redoItem(transaction, item, itemsToRedo) !== null || performedChange + itemsToRedo.forEach(struct => { + performedChange = redoItem(transaction, struct, itemsToRedo) !== null || performedChange }) - const structs = /** @type {Array<GC|Item>} */ (store.clients.get(doc.clientID)) - iterateStructs(transaction, structs, stackItem.start, stackItem.len, struct => { - if (struct instanceof Item && !struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) { + /** + * @type {Array<Item>} + */ + const itemsToDelete = [] + iterateStructs(transaction, structs, stackStartClock, stackItem.len, struct => { + if (struct instanceof Item) { if (struct.redone !== null) { let { item, diff } = followRedone(store, struct.id) if (diff > 0) { - item = getItemCleanStart(transaction, store, createID(item.id.client, item.id.clock + diff)) + item = getItemCleanStart(transaction, createID(item.id.client, item.id.clock + diff)) } if (item.length > stackItem.len) { - getItemCleanStart(transaction, store, createID(item.id.client, item.id.clock + stackItem.len)) + getItemCleanStart(transaction, createID(item.id.client, stackEndClock)) } struct = item } - keepItem(struct) - struct.delete(transaction) - performedChange = true + if (!struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) { + itemsToDelete.push(struct) + } } }) + // We want to delete in reverse order so that children are deleted before + // parents, so we have more information available when items are filtered. + for (let i = itemsToDelete.length - 1; i >= 0; i--) { + const item = itemsToDelete[i] + if (undoManager.deleteFilter(item)) { + item.delete(transaction) + performedChange = true + } + } result = stackItem if (result != null) { undoManager.emit('stack-item-popped', [{ stackItem: result, type: eventType }, undoManager]) @@ -87,26 +117,39 @@ const popStackItem = (undoManager, stack, eventType) => { return result } +/** + * @typedef {Object} UndoManagerOptions + * @property {number} [UndoManagerOptions.captureTimeout=500] + * @property {function(Item):boolean} [UndoManagerOptions.deleteFilter=()=>true] Sometimes + * it is necessary to filter whan an Undo/Redo operation can delete. If this + * filter returns false, the type/item won't be deleted even it is in the + * undo/redo scope. + * @property {Set<any>} [UndoManagerOptions.trackedOrigins=new Set([null])] + */ + /** * Fires 'stack-item-added' event when a stack item was added to either the undo- or * the redo-stack. You may store additional stack information via the - * metadata property on `event.stackItem.metadata` (it is a `Map` of metadata properties). + * metadata property on `event.stackItem.meta` (it is a `Map` of metadata properties). * Fires 'stack-item-popped' event when a stack item was popped from either the - * undo- or the redo-stack. You may restore the saved stack information from `event.stackItem.metadata`. + * undo- or the redo-stack. You may restore the saved stack information from `event.stackItem.meta`. * * @extends {Observable<'stack-item-added'|'stack-item-popped'>} */ export class UndoManager extends Observable { /** * @param {AbstractType<any>|Array<AbstractType<any>>} typeScope Accepts either a single type, or an array of types - * @param {Set<any>} [trackedTransactionOrigins=new Set([null])] - * @param {object} [options={captureTimeout=500}] + * @param {UndoManagerOptions} options */ - constructor (typeScope, trackedTransactionOrigins = new Set([null]), { captureTimeout = 500 } = {}) { + constructor (typeScope, { captureTimeout, deleteFilter = () => true, trackedOrigins = new Set([null]) } = {}) { + if (captureTimeout == null) { + captureTimeout = 500 + } super() this.scope = typeScope instanceof Array ? typeScope : [typeScope] - trackedTransactionOrigins.add(this) - this.trackedTransactionOrigins = trackedTransactionOrigins + this.deleteFilter = deleteFilter + trackedOrigins.add(this) + this.trackedOrigins = trackedOrigins /** * @type {Array<StackItem>} */ @@ -126,7 +169,7 @@ export class UndoManager extends Observable { this.lastChange = 0 this.doc.on('afterTransaction', /** @param {Transaction} transaction */ transaction => { // Only track certain transactions - if (!this.scope.some(type => transaction.changedParentTypes.has(type)) || (!this.trackedTransactionOrigins.has(transaction.origin) && (!transaction.origin || !this.trackedTransactionOrigins.has(transaction.origin.constructor)))) { + if (!this.scope.some(type => transaction.changedParentTypes.has(type)) || (!this.trackedOrigins.has(transaction.origin) && (!transaction.origin || !this.trackedOrigins.has(transaction.origin.constructor)))) { return } const undoing = this.undoing @@ -144,7 +187,7 @@ export class UndoManager extends Observable { if (now - this.lastChange < captureTimeout && stack.length > 0 && !undoing && !redoing) { // append change to last stack op const lastOp = stack[stack.length - 1] - lastOp.ds = mergeDeleteSets(lastOp.ds, transaction.deleteSet) + lastOp.ds = mergeDeleteSets([lastOp.ds, transaction.deleteSet]) lastOp.len = afterState - lastOp.start } else { // create a new stack op @@ -154,7 +197,7 @@ export class UndoManager extends Observable { this.lastChange = now } // make sure that deleted structs are not gc'd - iterateDeletedStructs(transaction, transaction.deleteSet, transaction.doc.store, /** @param {Item|GC} item */ item => { + iterateDeletedStructs(transaction, transaction.deleteSet, /** @param {Item|GC} item */ item => { if (item instanceof Item && this.scope.some(type => isParentOf(type, item))) { keepItem(item) } diff --git a/src/utils/YEvent.js b/src/utils/YEvent.js index 63435563..3a9d1dc7 100644 --- a/src/utils/YEvent.js +++ b/src/utils/YEvent.js @@ -1,9 +1,12 @@ import { isDeleted, - AbstractType, Transaction, AbstractStruct // eslint-disable-line + Item, AbstractType, Transaction, AbstractStruct // eslint-disable-line } from '../internals.js' +import * as set from 'lib0/set.js' +import * as array from 'lib0/array.js' + /** * YEvent describes the changes on a YType. */ @@ -28,6 +31,10 @@ export class YEvent { * @type {Transaction} */ this.transaction = transaction + /** + * @type {Object|null} + */ + this._changes = null } /** @@ -49,6 +56,8 @@ export class YEvent { /** * Check if a struct is deleted by this event. * + * In contrast to change.deleted, this method also returns true if the struct was added and then deleted. + * * @param {AbstractStruct} struct * @return {boolean} */ @@ -59,12 +68,121 @@ export class YEvent { /** * Check if a struct is added by this event. * + * In contrast to change.deleted, this method also returns true if the struct was added and then deleted. + * * @param {AbstractStruct} struct * @return {boolean} */ adds (struct) { return struct.id.clock >= (this.transaction.beforeState.get(struct.id.client) || 0) } + + /** + * @return {{added:Set<Item>,deleted:Set<Item>,delta:Array<{insert:Array<any>}|{delete:number}|{retain:number}>}} + */ + get changes () { + let changes = this._changes + if (changes === null) { + const target = this.target + const added = set.create() + const deleted = set.create() + /** + * @type {Array<{insert:Array<any>}|{delete:number}|{retain:number}>} + */ + const delta = [] + /** + * @type {Map<string,{ action: 'add' | 'update' | 'delete', oldValue: any}>} + */ + const keys = new Map() + changes = { + added, deleted, delta, keys + } + const changed = /** @type Set<string|null> */ (this.transaction.changed.get(target)) + if (changed.has(null)) { + /** + * @type {any} + */ + let lastOp = null + const packOp = () => { + if (lastOp) { + delta.push(lastOp) + } + } + for (let item = target._start; item !== null; item = item.right) { + if (item.deleted) { + if (this.deletes(item) && !this.adds(item)) { + if (lastOp === null || lastOp.delete === undefined) { + packOp() + lastOp = { delete: 0 } + } + lastOp.delete += item.length + deleted.add(item) + } // else nop + } else { + if (this.adds(item)) { + if (lastOp === null || lastOp.insert === undefined) { + packOp() + lastOp = { insert: [] } + } + lastOp.insert = lastOp.insert.concat(item.content.getContent()) + added.add(item) + } else { + if (lastOp === null || lastOp.retain === undefined) { + packOp() + lastOp = { retain: 0 } + } + lastOp.retain += item.length + } + } + } + if (lastOp !== null && lastOp.retain === undefined) { + packOp() + } + } + changed.forEach(key => { + if (key !== null) { + const item = /** @type {Item} */ (target._map.get(key)) + /** + * @type {'delete' | 'add' | 'update'} + */ + let action + let oldValue + if (this.adds(item)) { + let prev = item.left + while (prev !== null && this.adds(prev)) { + prev = prev.left + } + if (this.deletes(item)) { + if (prev !== null && this.deletes(prev)) { + action = 'delete' + oldValue = array.last(prev.content.getContent()) + } else { + return + } + } else { + if (prev !== null && this.deletes(prev)) { + action = 'update' + oldValue = array.last(prev.content.getContent()) + } else { + action = 'add' + oldValue = undefined + } + } + } else { + if (this.deletes(item)) { + action = 'delete' + oldValue = array.last(/** @type {Item} */ item.content.getContent()) + } else { + return // nop + } + } + keys.set(key, { action, oldValue }) + } + }) + this._changes = changes + } + return changes + } } /** diff --git a/src/utils/encoding.js b/src/utils/encoding.js index c4ee0d2a..00aee8ac 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -23,9 +23,10 @@ import { readID, getState, getStateVector, - readDeleteSet, + readAndApplyDeleteSet, writeDeleteSet, createDeleteSetFromStructStore, + transact, Doc, Transaction, AbstractStruct, StructStore, ID // eslint-disable-line } from '../internals.js' @@ -230,7 +231,7 @@ export const tryResumePendingDeleteReaders = (transaction, store) => { const pendingReaders = store.pendingDeleteReaders store.pendingDeleteReaders = [] for (let i = 0; i < pendingReaders.length; i++) { - readDeleteSet(pendingReaders[i], transaction, store) + readAndApplyDeleteSet(pendingReaders[i], transaction, store) } } @@ -299,10 +300,10 @@ export const readStructs = (decoder, transaction, store) => { * @function */ export const readUpdate = (decoder, ydoc, transactionOrigin) => - ydoc.transact(transaction => { + transact(ydoc, transaction => { readStructs(decoder, transaction, ydoc.store) - readDeleteSet(decoder, transaction, ydoc.store) - }, transactionOrigin) + readAndApplyDeleteSet(decoder, transaction, ydoc.store) + }, transactionOrigin, false) /** * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`. @@ -381,6 +382,22 @@ export const readStateVector = decoder => { */ export const decodeStateVector = decodedState => readStateVector(decoding.createDecoder(decodedState)) +/** + * Write State Vector to `lib0/encoding.js#Encoder`. + * + * @param {encoding.Encoder} encoder + * @param {Map<number,number>} sv + * @function + */ +export const writeStateVector = (encoder, sv) => { + encoding.writeVarUint(encoder, sv.size) + sv.forEach((clock, client) => { + encoding.writeVarUint(encoder, client) + encoding.writeVarUint(encoder, clock) + }) + return encoder +} + /** * Write State Vector to `lib0/encoding.js#Encoder`. * @@ -389,16 +406,7 @@ export const decodeStateVector = decodedState => readStateVector(decoding.create * * @function */ -export const writeDocumentStateVector = (encoder, doc) => { - encoding.writeVarUint(encoder, doc.store.clients.size) - doc.store.clients.forEach((structs, client) => { - const struct = structs[structs.length - 1] - const id = struct.id - encoding.writeVarUint(encoder, id.client) - encoding.writeVarUint(encoder, id.clock + struct.length) - }) - return encoder -} +export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encoder, getStateVector(doc.store)) /** * Encode State as Uint8Array. diff --git a/tests/encoding.tests.js b/tests/encoding.tests.js index 387c385d..a9160ef8 100644 --- a/tests/encoding.tests.js +++ b/tests/encoding.tests.js @@ -8,19 +8,21 @@ import { readContentJSON, readContentEmbed, readContentType, - readContentFormat + readContentFormat, + readContentAny } from '../src/internals.js' /** * @param {t.TestCase} tc */ export const testStructReferences = tc => { - t.assert(contentRefs.length === 8) + t.assert(contentRefs.length === 9) t.assert(contentRefs[1] === readContentDeleted) - t.assert(contentRefs[2] === readContentJSON) + t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json? t.assert(contentRefs[3] === readContentBinary) t.assert(contentRefs[4] === readContentString) t.assert(contentRefs[5] === readContentEmbed) t.assert(contentRefs[6] === readContentFormat) t.assert(contentRefs[7] === readContentType) + t.assert(contentRefs[8] === readContentAny) } diff --git a/tests/testHelper.js b/tests/testHelper.js index 913e9878..6296b7a6 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -378,7 +378,7 @@ export const compareDS = (ds1, ds2) => { */ export const applyRandomTests = (tc, mods, iterations, initTestObject) => { const gen = tc.prng - const result = init(tc, { users: 5 }, initTestObject || (() => null)) + const result = init(tc, { users: 5 }, initTestObject) const { testConnector, users } = result for (let i = 0; i < iterations; i++) { if (prng.int31(gen, 0, 100) <= 2) { diff --git a/tests/undo-redo.tests.js b/tests/undo-redo.tests.js index ca4051f8..349dcf7f 100644 --- a/tests/undo-redo.tests.js +++ b/tests/undo-redo.tests.js @@ -13,6 +13,23 @@ import * as t from 'lib0/testing.js' export const testUndoText = tc => { const { testConnector, text0, text1 } = init(tc, { users: 3 }) const undoManager = new UndoManager(text0) + + // items that are added & deleted in the same transaction won't be undo + text0.insert(0, 'test') + text0.delete(0, 4) + undoManager.undo() + t.assert(text0.toString() === '') + + // follow redone items + text0.insert(0, 'a') + undoManager.stopCapturing() + text0.delete(0, 1) + undoManager.stopCapturing() + undoManager.undo() + t.assert(text0.toString() === 'a') + undoManager.undo() + t.assert(text0.toString() === '') + text0.insert(0, 'abc') text1.insert(0, 'xyz') testConnector.syncAll() @@ -65,6 +82,15 @@ export const testUndoMap = tc => { t.assert(map0.get('a') === 44) undoManager.redo() t.assert(map0.get('a') === 44) + + // test setting value multiple times + map0.set('b', 'initial') + undoManager.stopCapturing() + map0.set('b', 'val1') + map0.set('b', 'val2') + undoManager.stopCapturing() + undoManager.undo() + t.assert(map0.get('b') === 'initial') } /** @@ -172,7 +198,7 @@ export const testUndoEvents = tc => { export const testTrackClass = tc => { const { users, text0 } = init(tc, { users: 3 }) // only track origins that are numbers - const undoManager = new UndoManager(text0, new Set([Number])) + const undoManager = new UndoManager(text0, { trackedOrigins: new Set([Number]) }) users[0].transact(() => { text0.insert(0, 'abc') }, 42) @@ -201,3 +227,22 @@ export const testTypeScope = tc => { undoManagerBoth.undo() t.assert(text1.toString() === '') } + +/** + * @param {t.TestCase} tc + */ +export const testUndoDeleteFilter = tc => { + /** + * @type {Array<Y.Map<any>>} + */ + const array0 = /** @type {any} */ (init(tc, { users: 3 }).array0) + const undoManager = new UndoManager(array0, { deleteFilter: item => !(item instanceof Y.Item) || (item.content instanceof Y.ContentType && item.content.type._map.size === 0) }) + const map0 = new Y.Map() + map0.set('hi', 1) + const map1 = new Y.Map() + array0.insert(0, [map0, map1]) + undoManager.undo() + t.assert(array0.length === 1) + array0.get(0) + t.assert(Array.from(array0.get(0).keys()).length === 1) +} diff --git a/tests/y-array.tests.js b/tests/y-array.tests.js index b3bb8454..085ac310 100644 --- a/tests/y-array.tests.js +++ b/tests/y-array.tests.js @@ -3,6 +3,7 @@ import { init, compare, applyRandomTests, Doc } from './testHelper.js' // eslint import * as Y from '../src/index.js' import * as t from 'lib0/testing.js' import * as prng from 'lib0/prng.js' +import * as math from 'lib0/math.js' /** * @param {t.TestCase} tc @@ -191,6 +192,33 @@ export const testInsertAndDeleteEventsForTypes = tc => { compare(users) } +/** + * @param {t.TestCase} tc + */ +export const testChangeEvent = tc => { + const { array0, users } = init(tc, { users: 2 }) + /** + * @type {any} + */ + let changes = null + array0.observe(e => { + changes = e.changes + }) + const newArr = new Y.Array() + array0.insert(0, [newArr, 4, 'dtrn']) + t.assert(changes !== null && changes.added.size === 2 && changes.deleted.size === 0) + t.compare(changes.delta, [{ insert: [newArr, 4, 'dtrn'] }]) + changes = null + array0.delete(0, 2) + t.assert(changes !== null && changes.added.size === 0 && changes.deleted.size === 2) + t.compare(changes.delta, [{ delete: 2 }]) + changes = null + array0.insert(1, [0.1]) + t.assert(changes !== null && changes.added.size === 1 && changes.deleted.size === 0) + t.compare(changes.delta, [{ retain: 1 }, { insert: [0.1] }]) + compare(users) +} + /** * @param {t.TestCase} tc */ @@ -211,7 +239,7 @@ export const testInsertAndDeleteEventsForTypes2 = tc => { } /** - * This issue has been reported here https://github.com/y-js/yjs/issues/155 + * This issue has been reported here https://github.com/yjs/yjs/issues/155 * @param {t.TestCase} tc */ export const testNewChildDoesNotEmitEventInTransaction = tc => { @@ -335,12 +363,12 @@ const arrayTransactions = [ var length = yarray.length if (length > 0) { var somePos = prng.int31(gen, 0, length - 1) - var delLength = prng.int31(gen, 1, Math.min(2, length - somePos)) + var delLength = prng.int31(gen, 1, math.min(2, length - somePos)) if (prng.bool(gen)) { var type = yarray.get(somePos) if (type.length > 0) { somePos = prng.int31(gen, 0, type.length - 1) - delLength = prng.int31(gen, 0, Math.min(2, type.length - somePos)) + delLength = prng.int31(gen, 0, math.min(2, type.length - somePos)) type.delete(somePos, delLength) } } else { diff --git a/tests/y-map.tests.js b/tests/y-map.tests.js index c47c9e46..5e01757e 100644 --- a/tests/y-map.tests.js +++ b/tests/y-map.tests.js @@ -292,6 +292,104 @@ export const testThrowsAddAndUpdateAndDeleteEvents = tc => { compare(users) } +/** + * @param {t.TestCase} tc + */ +export const testChangeEvent = tc => { + const { map0, users } = init(tc, { users: 2 }) + /** + * @type {any} + */ + let changes = null + /** + * @type {any} + */ + let keyChange = null + map0.observe(e => { + changes = e.changes + }) + map0.set('a', 1) + keyChange = changes.keys.get('a') + t.assert(changes !== null && keyChange.action === 'add' && keyChange.oldValue === undefined) + map0.set('a', 2) + keyChange = changes.keys.get('a') + t.assert(changes !== null && keyChange.action === 'update' && keyChange.oldValue === 1) + users[0].transact(() => { + map0.set('a', 3) + map0.set('a', 4) + }) + keyChange = changes.keys.get('a') + t.assert(changes !== null && keyChange.action === 'update' && keyChange.oldValue === 2) + users[0].transact(() => { + map0.set('b', 1) + map0.set('b', 2) + }) + keyChange = changes.keys.get('b') + t.assert(changes !== null && keyChange.action === 'add' && keyChange.oldValue === undefined) + users[0].transact(() => { + map0.set('c', 1) + map0.delete('c') + }) + t.assert(changes !== null && changes.keys.size === 0) + users[0].transact(() => { + map0.set('d', 1) + map0.set('d', 2) + }) + keyChange = changes.keys.get('d') + t.assert(changes !== null && keyChange.action === 'add' && keyChange.oldValue === undefined) + compare(users) +} + +/** + * @param {t.TestCase} tc + */ +export const testYmapEventExceptionsShouldCompleteTransaction = tc => { + const doc = new Y.Doc() + const map = doc.getMap('map') + + let updateCalled = false + let throwingObserverCalled = false + let throwingDeepObserverCalled = false + doc.on('update', () => { + updateCalled = true + }) + + const throwingObserver = () => { + throwingObserverCalled = true + throw new Error('Failure') + } + + const throwingDeepObserver = () => { + throwingDeepObserverCalled = true + throw new Error('Failure') + } + + map.observe(throwingObserver) + map.observeDeep(throwingDeepObserver) + + t.fails(() => { + map.set('y', '2') + }) + + t.assert(updateCalled) + t.assert(throwingObserverCalled) + t.assert(throwingDeepObserverCalled) + + // check if it works again + updateCalled = false + throwingObserverCalled = false + throwingDeepObserverCalled = false + t.fails(() => { + map.set('z', '3') + }) + + t.assert(updateCalled) + t.assert(throwingObserverCalled) + t.assert(throwingDeepObserverCalled) + + t.assert(map.get('z') === '3') +} + /** * @param {t.TestCase} tc */ diff --git a/tests/y-text.tests.js b/tests/y-text.tests.js index 2e860a79..86175479 100644 --- a/tests/y-text.tests.js +++ b/tests/y-text.tests.js @@ -1,6 +1,6 @@ -import { init, compare } from './testHelper.js' - +import * as Y from './testHelper.js' import * as t from 'lib0/testing.js' +const { init, compare } = Y /** * @param {t.TestCase} tc @@ -81,9 +81,71 @@ export const testBasicFormat = tc => { export const testGetDeltaWithEmbeds = tc => { const { text0 } = init(tc, { users: 1 }) text0.applyDelta([{ - insert: {linebreak: 's'} + insert: { linebreak: 's' } }]) t.compare(text0.toDelta(), [{ - insert: {linebreak: 's'} + insert: { linebreak: 's' } }]) } + +/** + * @param {t.TestCase} tc + */ +export const testSnapshot = tc => { + const { text0 } = init(tc, { users: 1 }) + const doc0 = /** @type {Y.Doc} */ (text0.doc) + doc0.gc = false + text0.applyDelta([{ + insert: 'abcd' + }]) + const snapshot1 = Y.snapshot(doc0) + text0.applyDelta([{ + retain: 1 + }, { + insert: 'x' + }, { + delete: 1 + }]) + const snapshot2 = Y.snapshot(doc0) + text0.applyDelta([{ + retain: 2 + }, { + delete: 3 + }, { + insert: 'x' + }, { + delete: 1 + }]) + const state1 = text0.toDelta(snapshot1) + t.compare(state1, [{ insert: 'abcd' }]) + const state2 = text0.toDelta(snapshot2) + t.compare(state2, [{ insert: 'axcd' }]) + const state2Diff = text0.toDelta(snapshot2, snapshot1) + // @ts-ignore Remove userid info + state2Diff.forEach(v => { + if (v.attributes && v.attributes.ychange) { + delete v.attributes.ychange.user + } + }) + t.compare(state2Diff, [{ insert: 'a' }, { insert: 'x', attributes: { ychange: { type: 'added' } } }, { insert: 'b', attributes: { ychange: { type: 'removed' } } }, { insert: 'cd' }]) +} + +/** + * @param {t.TestCase} tc + */ +export const testSnapshotDeleteAfter = tc => { + const { text0 } = init(tc, { users: 1 }) + const doc0 = /** @type {Y.Doc} */ (text0.doc) + doc0.gc = false + text0.applyDelta([{ + insert: 'abcd' + }]) + const snapshot1 = Y.snapshot(doc0) + text0.applyDelta([{ + retain: 4 + }, { + insert: 'e' + }]) + const state1 = text0.toDelta(snapshot1) + t.compare(state1, [{ insert: 'abcd' }]) +} diff --git a/tsconfig.json b/tsconfig.json index 45859200..54eacaa0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -38,7 +38,10 @@ "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ "paths": { - "yjs": ["./src/index.js"] + "yjs": ["./src/index.js"], + "lib0/*": ["node_modules/lib0/*"], + "lib0/set.js": ["node_modules/lib0/set.js"], + "lib0/function.js": ["node_modules/lib0/function.js"] }, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ // "typeRoots": [], /* List of folders to include type definitions from. */