13.0.0-77

updating YArray's iterator to iterate Types correctly

.jsdoc.json

@@ -5,7 +5,7 @@
     "dictionaries": ["jsdoc"]
   },
   "source": {
-    "include": ["./types", "./utils/UndoManager.mjs", "./utils/Y.mjs", "./provider", "./bindings"],
+    "include": ["./structs/Type.js", "./types", "./utils/UndoManager.js", "./utils/YEvent.js", "./utils/Y.js", "./provider", "./bindings"],
     "includePattern": ".js$"
   },
   "plugins": [

README.v13.md

@@ -1,7 +1,343 @@
 # ![Yjs](https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png)
 > A CRDT library with a powerful abstraction of shared data
 
-Yjs v13 is a work in progress.
+Yjs is a CRDT implementatation that exposes its internal structure as actual data types that can be manipulated and fire changes when remote or local changes happen. While Yjs can be used for all kinds of state management, we lay a special focus on shared editing.
+
+* Chat: [https://gitter.im/y-js/yjs](https://gitter.im/y-js/yjs)
+* Demos: [https://yjs.website/tutorial-prosemirror.html](https://yjs.website/tutorial-prosemirror.html)
+* API Docs: [https://yjs.website/](https://yjs.website/)
+
+### Supported Editors:
+
+| Name                                                   | Cursors |  Demo |
+|---|:-:|---|
+| [ProseMirror](https://prosemirror.net/) | ✔ | [link](https://yjs.website/tutorial-prosemirror.html) |
+| [Quill](https://quilljs.com/) |  | [link](https://yjs.website/tutorial-quill.html) |
+| [CodeMirror](https://codemirror.net/) | ✔ | [link](https://yjs.website/tutorial-codemirror.html) |
+| [Ace](https://ace.c9.io/) | | [link]() |
+| [Monaco](https://microsoft.github.io/monaco-editor/) | | [link]() |
+
+### Distinguishing Features
+
+* **Binary Encoding:**
+* **Undo/Redo:**
+* **Types:**
+* **Offline:** Yjs is designed to support offline editing. Read [this section](#Offline) about the limitations of offline editing in Yjs. The only provider supporting full offline editing is Ydb.
+* **Network-agnostic:** Yjs ships with many providers that handle connection and distribution of updates to other peers. Yjs itself is network-agnostic and does not depend on a central source of truth that distributes updates to other peers. Check [this section](#Create-a-Custom-Provider) to find out how the sync mechanism works and how to implement your custom provider.
+
+# Table of Contents
+
+* [Getting Started](#Getting-Started)
+  * [Tutorial](#Short-Tutorial)
+* [Providers](#Providers)
+  * [Websocket](#Websocket)
+  * [Ydb](#Ydb)
+  * [Create a Custom Provider](#Create-a-Custom-Provider)
+* [Shared Types](#Shared-Types)
+  * [YArray](#Yarray)
+  * [YMap](#YMap)
+  * [YText](#YText)
+  * [YXmlFragment and YXmlElement](#YXmlFragment-and-YXmlElement)
+  * [Create a Custom Type](#Create-a-Custom-Type)
+* [Bindings](#Bindings)
+  * [PromeMirror](#ProseMirror)
+  * [Quill](#Quill)
+  * [CodeMirror](#CodeMirror)
+  * [Ace](#Ace)
+  * [Monaco](#Monace)
+  * [DOM](#DOM)
+  * [Textarea](#Textarea)
+  * [Create a Custom Binding](#Create-a-Custom-Binding)
+* [Transaction](#Transaction)
+* [Offline Editing](#Offline-Editing)
+* [Awareness](#Awareness)
+* [Working with Yjs](#Working-with-Yjs)
+  * [Typescript Declarations](#Typescript-Declarations)
+* [Binary Protocols](#Binary-Protocols)
+  * [Sync Protocol](#Sync-Protocols)
+  * [Awareness Protocol](#Awareness-Protocols)
+  * [Auth Protocol](#Auth-Protocol)
+* [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm)
+* [Evaluation](#Evaluation)
+  * [Existing shared editing libraries](#Exisisting-Javascript-Products)
+  * [CRDT Algorithms](#CRDT-Algorithms)
+  * [Comparison of CRDT with OT](#Comparing-CRDT-with-OT)
+  * [Comparison of CRDT Algorithms](#Comparing-CRDT-Algorithms)
+  * [Comparison of Yjs with other Implementations](#Comparing-Yjs-with-other-Implementations)
+* [License and Author](#License-and-Author)
+
+## Getting Started
+
+Yjs does not hava any dependencies. Install this package with your favorite package manager, or just copy the files into your project.
+
+```sh
+npm i yjs
+```
+
+##### Tutorial
+
+In this *short* tutorial I will give an overview of the basic concepts in Yjs.
+
+Yjs itself only knows how to do conflict resolution. You need to choose a provider, that handles how document updates are distributed over the network.
+
+We will start by running a websocket server (part of the [websocket provider](#Websocket-Provider)):
+
+```sh
+PORT=1234 node ./node_modules/yjs/provider/websocket/server.js
+```
+
+The following client-side code connects to the websocket server and opens a shared document.
+
+```js
+import * as Y from 'yjs'
+import { WebsocketProvider } from 'yjs/provider/websocket.js'
+
+const provider = new WebsocketProvider('http://localhost:1234')
+const sharedDocument = provider.get('my-favourites')
+```
+
+All content created in a shared document is shared among all peers that request the same document. Now we define types on the shared document:
+
+```js
+sharedDocument.define('movie-ratings', Y.Map)
+sharedDocument.define('favourite-food', Y.Array)
+```
+
+All clients that define `'movie-ratings'` as `Y.Map` on the shared document named `'my-favourites'` have access to the same shared type. Example:
+
+**Client 1:**
+
+```js
+sharedDocument.define('movie-ratings', Y.Map)
+sharedDocument.define('favourite-food', Y.Array)
+
+const movies = sharedDocument.get('movie-ratings')
+const food = sharedDocument.get('fovourite-food')
+
+movies.set('deadpool', 10)
+food.insert(0, ['burger'])
+```
+
+**Client 2:**
+
+```js
+sharedDocument.define('movie-ratings', Y.Map)
+sharedDocument.define('favourite-food', Y.Map) // <- note that this definition differs from client1
+
+const movies = sharedDocument.get('movie-ratings')
+const food = sharedDocument.get('fovourite-food')
+
+movies.set('yt rewind', -10)
+food.set('pancake', 10)
+
+// after some time, when client1 and client2 synced, the movie list will be merged:
+movies.toJSON() // => { 'deadpool': 10, 'yt rewind': -10 }
+
+// But since client1 and client2 defined the types differently,
+// they do not have access to each others food list.
+food.toJSON() // => { pancake: 10 }
+```
+
+Now you understand how types are defined on a shared document. Next you can jump to one of the [tutorials on our website](https://yjs.website/tutorial-prosemirror.html) or continue reading about [Providers](#Providers), [Shared Types](#Shared-Types), and [Bindings](#Bindings).
+
+## Providers
+
+In Yjs, a provider handles the communication channel to *authenticate*, *authorize*, and *exchange document updates*. Yjs ships with some existing providers. 
+
+### Websocket Provider
+
+The websocket provider implements a classical client server model. Clients connect to a single endpoint over websocket. The server distributes awareness information and document updates among clients.
+
+The Websocket Provider is a solid choice if you want a central source that handles authentication and authorization. Websockets also send header information and cookies, so you can use existing authentication mechanisms with this server. I recommend that you slightly adapt the server in `./provider/websocket/server.js` to your needs.
+
+* Supports cross-tab communication. When you open the same document in the same browser, changes on the document are exchanged via cross-tab communication ([Broadcast Channel](https://developer.mozilla.org/en-US/docs/Web/API/Broadcast_Channel_API) and [localStorage](https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage) as fallback).
+* Supports exange of awareness information (e.g. cursors)
+
+##### Start a Websocket Server:
+
+```sh
+PORT=1234 node ./node_modules/yjs/provider/websocket/server.js
+```
+
+**Websocket Server with Persistence**
+
+Persist document updates in a LevelDB database.
+
+See [LevelDB Persistence](#LevelDB Persistence) for more info.
+
+```sh
+PORT=1234 YPERSISTENCE=./dbDir node ./node_modules/yjs/provider/websocket/server.js
+```
+
+##### Client Code:
+
+```js
+import * as Y from 'yjs'
+import { WebsocketProvider } from 'yjs/provider/websocket.js'
+
+const provider = new WebsocketProvider('http://localhost:1234')
+
+// open a websocket connection to http://localhost:1234/my-document-name
+const sharedDocument = provider.get('my-document-name')
+
+sharedDocument.on('status', event => {
+  console.log(event.status) // logs "connected" or "disconnected"
+})
+```
+
+#### Scaling
+
+These are mere suggestions how you could scale your server environment.
+
+**Option 1:** Websocket servers communicate with each other via a PubSub server. A room is represented by a PubSub channel. The downside of this approach is that the same shared document may be handled by many servers. But the upside is that this approach is fault tolerant, does not have a single point of failure, and is perfectly fit for route balancing.
+
+**Option 2:** Sharding with *consistent hashing*. Each document is handled by a unique server. This patterns requires an entity, like etcd, that performs regular health checks and manages servers. Based on the list of available servers (which is managed by etcd) a proxy calculates which server is responsible for each requested document. The disadvantage of this approach is that it is that load distribution may not be fair. Still, this approach may be the preferred solution if you want to store the shared document in a database - e.g. for indexing.
+
+### Ydb Provider
+
+TODO
+
+### Create Custom Provider
+
+A provider is only a concept. I encourage you to implement the same provider interface found above. This makes it easy to exchange communication protocols.
+
+Since providers handle the communication channel, they will necessarily interact with the [binary protocols](#Binary-Protocols). I suggest that you build upon the existing protocols. But you may also implement a custom communication protocol.
+
+Read section [Sync Protocol](#Sync-Protocol) to learn how syncing works.
+
+## Shared Types
+
+A shared type is just a normal data type like [Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map) or [Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array). But a shared type may also be modified by a remote client. Conflicts are automatically resolved by the rules described in this section - but please note that this is only a rough overview of how conflict resolution works. Please read the [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm) section for an in-depth description of the conflict resolution approach.
+
+As explained in [Tutorial](#Tutorial), a shared type is shared among all peers when they are defined with the same name on the same shared document. I.e.
+
+```js
+sharedDocument.define('my-array', Y.Array)
+
+const myArray = sharedDocument.get('my-array')
+```
+
+You may define a shared types several times, as long as you don't change the type definition.
+
+```js
+sharedDocument.define('my-array', Y.Array)
+
+const myArray = sharedDocument.get('my-array')
+
+const alsoMyArray = sharedDocument.define('my-array', Y.Array)
+
+console.log(myArray === alsoMyArray) // => true
+```
+
+All shared types have an `type.observe(event => ..)` method that allows you to observe any changes. You may also observe all changes on a type and any of its children with the `type.observeDeep(events => ..)` method. Here, `events` is the [Set](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set) of events that were fired on type, or any of its children.
+
+All Events inherit from [YEvent](https://yjs.website/module-utils.YEvent.html).
+
+### YMap
+> Complete API docs: [https://yjs.website/module-types.ymap](https://yjs.website/module-types.ymap)
+
+The YMap type is very similar to the JavaScript [Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map). 
+
+YMap fires [YMapEvents](https://yjs.website/module-types.YMapEvent.html).
+
+```js
+import * as Y from 'yjs'
+
+const ymap = new Y.Map()
+
+ymap.observe(event => {
+  console.log('ymap keys changed:', event.keysChanged, event.remote)
+})
+
+ymap.set('key', 'value') // => ymap keys changed: Set{ 'key' } false
+ymap.delete('key') // => ymap keys changed: Set{ 'key' }
+
+const ymap2 = new YMap()
+ymap2.set(1, 'two')
+
+ymap.set('type', ymap2) // => ymap keys changed: Set{ 'type' }
+```
+
+##### Concurrent YMap changes
+
+* Concurrent edits on different keys do not affect each other. E.g. if client1 does `ymap.set('a', 1)` and client2 does `ymap.set('b', 2)`, both clients will end up with `YMap{ a: 1, b: 2 }`
+* If client1 and client2 `set` the same property at the same time, the edit from the client with the smaller userID will prevail (`sharedDocument.userID`)
+* If client1 sets a property `ymap.set('a', 1)` and client2 deletes a property `ymap.delete('a')`, the set operation always prevails.
+
+### YArray
+> Complete API docs: [https://yjs.website/module-types.yarray](https://yjs.website/module-types.yarray)
+
+YArray fires [YArrayEvents](https://yjs.website/module-types.YMapEvent.html).
+
+
+```js
+import * as Y from 'yjs'
+
+const yarray = new Y.Array()
+
+yarray.observe(event => {
+  console.log('yarray changed:', event.addedElements, event.removedElements, event.remote)
+})
+
+// insert two elements at position 0
+yarray.insert(0, ['a', 1]) // => yarray changed: Set{Item{'a'}, Item{1}}, Set{}, false 
+console.log(yarray.toArray()) // => ['a', 1]
+yarray.delete(1, 1) // yarray changed: Set{}, Set{Item{1}}, false
+
+yarray.insert(1, new Y.Map()) // => yarray changed: Set{YMap{}}, Set{}, false
+
+// The difference between .toArray and .toJSON:
+console.log(yarray.toArray()) // => ['a', YMap{}]
+console.log(yarray.toJSON()) // => ['a', {}]
+```
+
+As you can see from the above example, primitive data is wrapped into an Item. This makes it possible to find the exact location of the change.
+
+##### Concurrent YArray changes
+
+* YArray internally represents the data as a doubly linked list. The Array `['a', YMap{}, 1]` is internally represented as `Item{'a'} <-> YMap{} <-> Item{1}`. Accordingly, the insert operation `yarray.insert(1, ['b'])` is internally transformed to `insert Item{'b'} between Item{'a'} and YMap{}`.
+* When an Item is deleted, it is only marked as deleted. Only its content is garbage collected and freed from memory.
+* Therefore, the remote operation `insert x between a and b` can still be fulfilled when item `a` or item `b` are deleted.
+* In case that two clients insert content between the same items (a concurrent insertion), the order of the insertions is decided based on the `sharedDocument.userID`.
+
+### YText
+> Complete API docs: [https://yjs.website/module-types.ytext](https://yjs.website/module-types.ytext)
+
+A YText is basically a [YArray](#YArray) that is optimized for text content. 
+
+### YXmlFragment and YXmlElement
+> Complete API docs: [https://yjs.website/module-types.yxmlfragment](https://yjs.website/module-types.yxmlfragment) and [https://yjs.website/module-types.yxmlelement](https://yjs.website/module-types.yxmlelement)
+
+### Custom Types
+
+## Bindings
+
+## Transaction
+
+
+
+## Binary Protocols
+
+### Sync Protocol
+
+Sync steps
+
+### Awareness Protocol
+
+### Auth Protocol
+
+## Offline Editing
+
+It is trivial with Yjs to persist the local state to indexeddb, so it is always available when working offline. But there are two non-trivial questions that need to answered when implementing a professional offline editing app:
+
+1. How does a client sync down all rooms that were modified while offline?
+2. How does a client sync up all rooms that were modified while offline?
+
+Assuming 5000 documents are stored on each client for offline usage. How do we sync up/down each of those documents after a client comes online? It would be inefficient to sync each of those rooms separately. The only provider that currently supports syncing many rooms efficiently is Ydb, because its database layer is optimized to sync many rooms with each client.
+
+If you do not care about 1. and 2. you can use `/persistences/indexeddb.js` to mirror the local state to indexeddb.
+
+## Working with Yjs
 
 ### Typescript Declarations
 
@@ -9,10 +345,23 @@ Until [this](https://github.com/Microsoft/TypeScript/issues/7546) is fixed, the
 
 ```json
 {
-  "checkJs": true,
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    ..
+  },
   "include": [
     "./node_modules/yjs/"
   ]
-  ..
 }
 ```
+
+## CRDT Algorithm
+
+## License and Author
+
+Yjs and all related projects are [**MIT licensed**](./LICENSE). Some files also contain an additional copyright notice that allows you to copy and modify the code without shipping the copyright notice (e.g. `./provider/websocket/WebsocketProvider.js` and `./provider/websocket/server.js`)
+
+Yjs is based on the research I did as a student at the RWTH i5. I am working on Yjs in my spare time. Please help me by donating or hiring me for consulting, so I can continue to work on this project.
+
+kevin.jahns@protonmail.com

bindings/codemirror.js

@@ -0,0 +1,180 @@
+/**
+ * @module bindings/textarea
+ */
+
+import { createMutex } from '../lib/mutex.js'
+import * as math from '../lib/math.js'
+import * as ypos from '../utils/relativePosition.js'
+
+const typeObserver = (binding, event) => {
+  binding._mux(() => {
+    const cm = binding.target
+    cm.operation(() => {
+      const delta = event.delta
+      let index = 0
+      for (let i = 0; i < event.delta.length; i++) {
+        const d = delta[i]
+        if (d.retain) {
+          index += d.retain
+        } else if (d.insert) {
+          const pos = cm.posFromIndex(index)
+          cm.replaceRange(d.insert, pos, pos, 'prosemirror-binding')
+          index += d.insert.length
+        } else if (d.delete) {
+          const start = cm.posFromIndex(index)
+          const end = cm.posFromIndex(index + d.delete)
+          cm.replaceRange('', start, end, 'prosemirror-binding')
+        }
+      }
+    })
+  })
+}
+
+const targetObserver = (binding, change) => {
+  binding._mux(() => {
+    const start = binding.target.indexFromPos(change.from)
+    const delLen = change.removed.map(s => s.length).reduce(math.add) + change.removed.length - 1
+    if (delLen > 0) {
+      binding.type.delete(start, delLen)
+    }
+    if (change.text.length > 0) {
+      binding.type.insert(start, change.text.join('\n'))
+    }
+  })
+}
+
+const createRemoteCaret = (username, color) => {
+  const caret = document.createElement('span')
+  caret.classList.add('remote-caret')
+  caret.setAttribute('style', `border-color: ${color}`)
+  const userDiv = document.createElement('div')
+  userDiv.setAttribute('style', `background-color: ${color}`)
+  userDiv.insertBefore(document.createTextNode(username), null)
+  caret.insertBefore(userDiv, null)
+  return caret
+}
+
+const updateRemoteSelection = (y, cm, type, cursors, clientId) => {
+  // destroy current text mark
+  const m = cursors.get(clientId)
+  if (m !== undefined) {
+    m.caret.clear()
+    if (m.sel !== null) {
+      m.sel.clear()
+    }
+    cursors.delete(clientId)
+  }
+  // redraw caret and selection for clientId
+  const aw = y.awareness.get(clientId)
+  if (aw === undefined) {
+    return
+  }
+  const user = aw.user || {}
+  if (user.color == null) {
+    user.color = '#ffa500'
+  }
+  if (user.name == null) {
+    user.name = `User: ${clientId}`
+  }
+  const cursor = aw.cursor
+  if (cursor == null || cursor.anchor == null || cursor.head == null) {
+    return
+  }
+  const anchor = ypos.fromRelativePosition(y, cursor.anchor || null)
+  const head = ypos.fromRelativePosition(y, cursor.head || null)
+  if (anchor !== null && head !== null && anchor.type === type && head.type === type) {
+    const headpos = cm.posFromIndex(head.offset)
+    const anchorpos = cm.posFromIndex(anchor.offset)
+    let from, to
+    if (head.offset < anchor.offset) {
+      from = headpos
+      to = anchorpos
+    } else {
+      from = anchorpos
+      to = headpos
+    }
+    const caretEl = createRemoteCaret(user.name, user.color)
+    const caret = cm.setBookmark(headpos, { widget: caretEl, insertLeft: true })
+    let sel = null
+    if (head.offset !== anchor.offset) {
+      sel = cm.markText(from, to, { css: `background-color: ${user.color}70`, inclusiveRight: true, inclusiveLeft: false })
+    }
+    cursors.set(clientId, { caret, sel })
+  }
+}
+
+const prosemirrorCursorActivity = (y, cm, type) => {
+  if (!cm.hasFocus()) {
+    return
+  }
+  const aw = y.getLocalAwarenessInfo()
+  const anchor = ypos.getRelativePosition(type, cm.indexFromPos(cm.getCursor('anchor')))
+  const head = ypos.getRelativePosition(type, cm.indexFromPos(cm.getCursor('head')))
+  if (aw.cursor == null || !ypos.equal(aw.cursor.anchor, anchor) || !ypos.equal(aw.cursor.head, head)) {
+    y.setAwarenessField('cursor', {
+      anchor, head
+    })
+  }
+}
+
+/**
+ * A binding that binds a YText to a CodeMirror editor.
+ *
+ * @example
+ *   const ytext = ydocument.define('codemirror', Y.Text)
+ *   const editor = new CodeMirror(document.querySelector('#container'), {
+ *     mode: 'javascript',
+ *     lineNumbers: true
+ *   })
+ *   const binding = new CodeMirrorBinding(editor)
+ *
+ */
+export class CodeMirrorBinding {
+  /**
+   * @param {YText} textType
+   * @param {CodeMirror} codeMirror
+   * @param {Object} [options={cursors: true}]
+   */
+  constructor (textType, codeMirror, { cursors = true } = {}) {
+    const y = textType._y
+    this.type = textType
+    this.target = codeMirror
+    /**
+     * @private
+     */
+    this._mux = createMutex()
+    // set initial value
+    codeMirror.setValue(textType.toString())
+    // observe type and target
+    this._typeObserver = event => typeObserver(this, event)
+    this._targetObserver = (_, change) => targetObserver(this, change)
+    this._cursors = new Map()
+    this._awarenessListener = event => {
+      const f = clientId => updateRemoteSelection(y, codeMirror, textType, this._cursors, clientId)
+      event.added.forEach(f)
+      event.removed.forEach(f)
+      event.updated.forEach(f)
+    }
+    this._cursorListener = () => prosemirrorCursorActivity(y, codeMirror, textType)
+    this._blurListeer = () =>
+      y.setAwarenessField('cursor', null)
+    textType.observe(this._typeObserver)
+    codeMirror.on('change', this._targetObserver)
+    if (cursors) {
+      y.on('awareness', this._awarenessListener)
+      codeMirror.on('cursorActivity', this._cursorListener)
+      codeMirror.on('blur', this._blurListeer)
+      codeMirror.on('focus', this._cursorListener)
+    }
+  }
+  destroy () {
+    this.type.unobserve(this._typeObserver)
+    this.target.off('change', this._targetObserver)
+    this.type.off('awareness', this._awarenessListener)
+    this.target.off('cursorActivity', this._cursorListener)
+    this.target.off('focus', this._cursorListener)
+    this.target.off('blur', this._blurListeer)
+    this.type = null
+    this.target = null
+  }
+}

bindings/dom.js

@@ -0,0 +1 @@
+export * from './dom/DomBinding.js'

bindings/dom.mjs

@@ -1 +0,0 @@
-export * from './dom/DomBinding.mjs'

bindings/dom/DomBinding.js

@@ -4,14 +4,14 @@
 
 /* global MutationObserver, getSelection */
 
-import { fromRelativePosition } from '../../utils/relativePosition.mjs'
-import { createMutex } from '../../lib/mutex.mjs'
-import { createAssociation, removeAssociation } from './util.mjs'
-import { beforeTransactionSelectionFixer, afterTransactionSelectionFixer, getCurrentRelativeSelection } from './selection.mjs'
-import { defaultFilter, applyFilterOnType } from './filter.mjs'
-import { typeObserver } from './typeObserver.mjs'
-import { domObserver } from './domObserver.mjs'
-import { YXmlFragment } from '../../types/YXmlElement.mjs' // eslint-disable-line
+import { fromRelativePosition } from '../../utils/relativePosition.js'
+import { createMutex } from '../../lib/mutex.js'
+import { createAssociation, removeAssociation } from './util.js'
+import { beforeTransactionSelectionFixer, afterTransactionSelectionFixer, getCurrentRelativeSelection } from './selection.js'
+import { defaultFilter, applyFilterOnType } from './filter.js'
+import { typeObserver } from './typeObserver.js'
+import { domObserver } from './domObserver.js'
+import { YXmlFragment } from '../../types/YXmlElement.js' // eslint-disable-line
 
 /**
  * @callback DomFilter

bindings/dom/domObserver.js

@@ -2,13 +2,13 @@
  * @module bindings/dom
  */
 
-import { YXmlHook } from '../../types/YXmlHook.mjs'
+import { YXmlHook } from '../../types/YXmlHook.js'
 import {
   iterateUntilUndeleted,
   removeAssociation,
-  insertNodeHelper } from './util.mjs'
-import { simpleDiff } from '../../lib/diff.mjs'
-import { YXmlFragment } from '../../types/YXmlElement.mjs'
+  insertNodeHelper } from './util.js'
+import { simpleDiff } from '../../lib/diff.js'
+import { YXmlFragment } from '../../types/YXmlElement.js'
 
 /**
  * 1. Check if any of the nodes was deleted
@@ -20,6 +20,8 @@ import { YXmlFragment } from '../../types/YXmlElement.mjs'
  *       recreate a new yxml element that is bound to that node.
  *       You can detect that a node was moved because expectedId
  *       !== actualId in the list
+ *
+ * @function
  * @private
  */
 const applyChangesFromDom = (binding, dom, yxml, _document) => {
@@ -85,6 +87,7 @@ const applyChangesFromDom = (binding, dom, yxml, _document) => {
 
 /**
  * @private
+ * @function
  */
 export function domObserver (mutations, _document) {
   this._mutualExclude(() => {

bindings/dom/domToType.js

@@ -3,12 +3,12 @@
  */
 
 /* eslint-env browser */
-import { YXmlText } from '../../types/YXmlText.mjs'
-import { YXmlHook } from '../../types/YXmlHook.mjs'
-import { YXmlElement } from '../../types/YXmlElement.mjs'
-import { createAssociation, domsToTypes } from './util.mjs'
-import { filterDomAttributes, defaultFilter } from './filter.mjs'
-import { DomBinding } from './DomBinding.mjs' // eslint-disable-line
+import { YXmlText } from '../../types/YXmlText.js'
+import { YXmlHook } from '../../types/YXmlHook.js'
+import { YXmlElement } from '../../types/YXmlElement.js'
+import { createAssociation, domsToTypes } from './util.js'
+import { filterDomAttributes, defaultFilter } from './filter.js'
+import { DomBinding } from './DomBinding.js' // eslint-disable-line
 
 /**
  * @callback DomFilter
@@ -20,6 +20,7 @@ import { DomBinding } from './DomBinding.mjs' // eslint-disable-line
 /**
  * Creates a Yjs type (YXml) based on the contents of a DOM Element.
  *
+ * @function
  * @param {Element|Text} element The DOM Element
  * @param {?Document} _document Optional. Provide the global document object
  * @param {Object<string, any>} [hooks = {}] Optional. Set of Yjs Hooks

bindings/dom/filter.js

@@ -2,14 +2,15 @@
  * @module bindings/dom
  */
 
-import { Y } from '../../utils/Y.mjs' // eslint-disable-line
-import { YXmlElement, YXmlFragment } from '../../types/YXmlElement.mjs' // eslint-disable-line
-import { isParentOf } from '../../utils/isParentOf.mjs'
-import { DomBinding } from './DomBinding.mjs' // eslint-disable-line
+import { Y } from '../../utils/Y.js' // eslint-disable-line
+import { YXmlElement, YXmlFragment } from '../../types/YXmlElement.js' // eslint-disable-line
+import { isParentOf } from '../../utils/isParentOf.js'
+import { DomBinding } from './DomBinding.js' // eslint-disable-line
 
 /**
  * Default filter method (does nothing).
  *
+ * @function
  * @param {String} nodeName The nodeName of the element
  * @param {Map} attrs Map of key-value pairs that are attributes of the node.
  * @return {Map | null} The allowed attributes or null, if the element should be
@@ -21,7 +22,10 @@ export const defaultFilter = (nodeName, attrs) => {
 }
 
 /**
- *
+ * @private
+ * @function
+ * @param {Element} dom
+ * @param {Function} filter
  */
 export const filterDomAttributes = (dom, filter) => {
   const attrs = new Map()
@@ -35,11 +39,11 @@ export const filterDomAttributes = (dom, filter) => {
 /**
  * Applies a filter on a type.
  *
+ * @private
+ * @function
  * @param {Y} y The Yjs instance.
  * @param {DomBinding} binding The DOM binding instance that has the dom filter.
  * @param {YXmlElement | YXmlFragment } type The type to apply the filter to.
- *
- * @private
  */
 export const applyFilterOnType = (y, binding, type) => {
   if (isParentOf(binding.type, type) && type instanceof YXmlElement) {

bindings/dom/selection.js

@@ -4,10 +4,13 @@
 
 /* globals getSelection */
 
-import { getRelativePosition } from '../../utils/relativePosition.mjs'
+import { getRelativePosition } from '../../utils/relativePosition.js'
 
 let relativeSelection = null
 
+/**
+ * @private
+ */
 const _getCurrentRelativeSelection = domBinding => {
   const { baseNode, baseOffset, extentNode, extentOffset } = getSelection()
   const baseNodeType = domBinding.domToType.get(baseNode)
@@ -21,8 +24,14 @@ const _getCurrentRelativeSelection = domBinding => {
   return null
 }
 
+/**
+ * @private
+ */
 export const getCurrentRelativeSelection = typeof getSelection !== 'undefined' ? _getCurrentRelativeSelection : domBinding => null
 
+/**
+ * @private
+ */
 export const beforeTransactionSelectionFixer = domBinding => {
   relativeSelection = getCurrentRelativeSelection(domBinding)
 }
@@ -30,6 +39,7 @@ export const beforeTransactionSelectionFixer = domBinding => {
 /**
  * Reset the browser range after every transaction.
  * This prevents any collapsing issues with the local selection.
+ *
  * @private
  */
 export const afterTransactionSelectionFixer = domBinding => {

bindings/dom/typeObserver.js

@@ -5,9 +5,9 @@
 /* eslint-env browser */
 /* global getSelection */
 
-import { YXmlText } from '../../types/YXmlText.mjs'
-import { YXmlHook } from '../../types/YXmlHook.mjs'
-import { removeDomChildrenUntilElementFound } from './util.mjs'
+import { YXmlText } from '../../types/YXmlText.js'
+import { YXmlHook } from '../../types/YXmlHook.js'
+import { removeDomChildrenUntilElementFound } from './util.js'
 
 const findScrollReference = scrollingElement => {
   if (scrollingElement !== null) {

bindings/dom/util.js

@@ -2,8 +2,8 @@
  * @module bindings/dom
  */
 
-import { domToType } from './domToType.mjs'
-import { DomBinding } from './DomBinding.mjs' // eslint-disable-line
+import { domToType } from './domToType.js'
+import { DomBinding } from './DomBinding.js' // eslint-disable-line
 
 /**
  * Iterates items until an undeleted item is found.
@@ -22,6 +22,7 @@ export const iterateUntilUndeleted = item => {
  * type).
  *
  * @private
+ * @function
  * @param {DomBinding} domBinding The binding object
  * @param {Element} dom The dom that is to be associated with type
  * @param {YXmlElement|YXmlHook} type The type that is to be associated with dom
@@ -37,6 +38,7 @@ export const removeAssociation = (domBinding, dom, type) => {
  * type).
  *
  * @private
+ * @function
  * @param {DomBinding} domBinding The binding object
  * @param {DocumentFragment|Element|Text} dom The dom that is to be associated with type
  * @param {YXmlFragment|YXmlElement|YXmlHook|YXmlText} type The type that is to be associated with dom
@@ -54,6 +56,7 @@ export const createAssociation = (domBinding, dom, type) => {
  * forget about oldDom. If oldDom is not associated with any type, nothing happens.
  *
  * @private
+ * @function
  * @param {DomBinding} domBinding The binding object
  * @param {Element} oldDom The existing dom
  * @param {Element} newDom The new dom object
@@ -74,6 +77,7 @@ export const switchAssociation = (domBinding, oldDom, newDom) => {
  * specified position.
  *
  * @private
+ * @function
  * @param {YXmlElement} type The type in which to insert DOM elements.
  * @param {YXmlElement|null} prev The reference node. New YxmlElements are
  *                           inserted after this node. Set null to insert at
@@ -101,6 +105,7 @@ export const domsToTypes = (doms, _document, hooks, filter, binding) => {
 
 /**
  * @private
+ * @function
  */
 export const insertNodeHelper = (yxml, prevExpectedNode, child, _document, binding) => {
   let insertedNodes = insertDomElementsAfter(yxml, prevExpectedNode, [child], _document, binding)
@@ -115,6 +120,7 @@ export const insertNodeHelper = (yxml, prevExpectedNode, child, _document, bindi
  * Remove children until `elem` is found.
  *
  * @private
+ * @function
  * @param {Element} parent The parent of `elem` and `currentChild`.
  * @param {Node} currentChild Start removing elements with `currentChild`. If
  *                               `currentChild` is `elem` it won't be removed.

bindings/prosemirror.js

@@ -0,0 +1,752 @@
+/**
+ * @module bindings/prosemirror
+ */
+
+import { YText } from '../types/YText.js' // eslint-disable-line
+import { YXmlElement, YXmlFragment } from '../types/YXmlElement.js' // eslint-disable-line
+import { createMutex } from '../lib/mutex.js'
+import * as PModel from 'prosemirror-model'
+import { EditorView,  Decoration, DecorationSet } from 'prosemirror-view' // eslint-disable-line
+import { Plugin, PluginKey, EditorState, TextSelection } from 'prosemirror-state' // eslint-disable-line
+import * as math from '../lib/math.js'
+import * as object from '../lib/object.js'
+import * as YPos from '../utils/relativePosition.js'
+import { isVisible } from '../utils/snapshot.js'
+import { simpleDiff } from '../lib/diff.js'
+
+/**
+ * @typedef {Map<YText | YXmlElement | YXmlFragment, PModel.Node>} ProsemirrorMapping
+ */
+
+/**
+ * The unique prosemirror plugin key for prosemirrorPlugin.
+ *
+ * @public
+ */
+export const prosemirrorPluginKey = new PluginKey('yjs')
+
+/**
+ * This plugin listens to changes in prosemirror view and keeps yXmlState and view in sync.
+ *
+ * This plugin also keeps references to the type and the shared document so other plugins can access it.
+ * @param {YXmlFragment} yXmlFragment
+ * @return {Plugin} Returns a prosemirror plugin that binds to this type
+ */
+export const prosemirrorPlugin = yXmlFragment => {
+  let changedInitialContent = false
+  const plugin = new Plugin({
+    props: {
+      editable: (state) => prosemirrorPluginKey.getState(state).snapshot == null
+    },
+    key: prosemirrorPluginKey,
+    state: {
+      init: (initargs, state) => {
+        return {
+          type: yXmlFragment,
+          y: yXmlFragment._y,
+          binding: null,
+          snapshot: null
+        }
+      },
+      apply: (tr, pluginState) => {
+        const change = tr.getMeta(prosemirrorPluginKey)
+        if (change !== undefined) {
+          pluginState = Object.assign({}, pluginState)
+          for (let key in change) {
+            pluginState[key] = change[key]
+          }
+        }
+        if (pluginState.binding !== null) {
+          if (change !== undefined && change.snapshot !== undefined) {
+            // snapshot changed, rerender next
+            setTimeout(() => {
+              if (change.restore == null) {
+                pluginState.binding._renderSnapshot(change.snapshot, change.prevSnapshot)
+              } else {
+                pluginState.binding._renderSnapshot(change.snapshot, change.snapshot)
+                // reset to current prosemirror state
+                delete pluginState.restore
+                delete pluginState.snapshot
+                delete pluginState.prevSnapshot
+                pluginState.binding._prosemirrorChanged(pluginState.binding.prosemirrorView.state.doc)
+              }
+            }, 0)
+          } else if (pluginState.snapshot == null) {
+            // only apply if no snapshot active
+            // update Yjs state when apply is called. We need to do this here to compute the correct cursor decorations with the cursor plugin
+            if (changedInitialContent || tr.doc.content.size > 4) {
+              changedInitialContent = true
+              pluginState.binding._prosemirrorChanged(tr.doc)
+            }
+          }
+        }
+        return pluginState
+      }
+    },
+    view: view => {
+      const binding = new ProsemirrorBinding(yXmlFragment, view)
+      view.dispatch(view.state.tr.setMeta(prosemirrorPluginKey, { binding }))
+      return {
+        update: () => {
+          const pluginState = plugin.getState(view.state)
+          if (pluginState.snapshot == null) {
+            if (changedInitialContent || view.state.doc.content.size > 4) {
+              changedInitialContent = true
+              binding._prosemirrorChanged(view.state.doc)
+            }
+          }
+        },
+        destroy: () => {
+          binding.destroy()
+        }
+      }
+    }
+  })
+  return plugin
+}
+
+/**
+ * The unique prosemirror plugin key for cursorPlugin.type
+ *
+ * @public
+ */
+export const cursorPluginKey = new PluginKey('yjs-cursor')
+
+/**
+ * A prosemirror plugin that listens to awareness information on Yjs.
+ * This requires that a `prosemirrorPlugin` is also bound to the prosemirror.
+ *
+ * @public
+ */
+export const cursorPlugin = new Plugin({
+  key: cursorPluginKey,
+  props: {
+    decorations: state => {
+      const ystate = prosemirrorPluginKey.getState(state)
+      const y = ystate.y
+      const awareness = y.getAwarenessInfo()
+      const decorations = []
+      if (ystate.snapshot != null) {
+        // do not render cursors while snapshot is active
+        return
+      }
+      awareness.forEach((aw, userID) => {
+        if (userID === y.userID) {
+          return
+        }
+        if (aw.cursor != null) {
+          let user = aw.user || {}
+          if (user.color == null) {
+            user.color = '#ffa500'
+          }
+          if (user.name == null) {
+            user.name = `User: ${userID}`
+          }
+          let anchor = relativePositionToAbsolutePosition(ystate.type, aw.cursor.anchor || null, ystate.binding.mapping)
+          let head = relativePositionToAbsolutePosition(ystate.type, aw.cursor.head || null, ystate.binding.mapping)
+          if (anchor !== null && head !== null) {
+            let maxsize = math.max(state.doc.content.size - 1, 0)
+            anchor = math.min(anchor, maxsize)
+            head = math.min(head, maxsize)
+            decorations.push(Decoration.widget(head, () => {
+              const cursor = document.createElement('span')
+              cursor.classList.add('ProseMirror-yjs-cursor')
+              cursor.setAttribute('style', `border-color: ${user.color}`)
+              const userDiv = document.createElement('div')
+              userDiv.setAttribute('style', `background-color: ${user.color}`)
+              userDiv.insertBefore(document.createTextNode(user.name), null)
+              cursor.insertBefore(userDiv, null)
+              return cursor
+            }, { key: userID + '' }))
+            const from = math.min(anchor, head)
+            const to = math.max(anchor, head)
+            decorations.push(Decoration.inline(from, to, { style: `background-color: ${user.color}70` }))
+          }
+        }
+      })
+      return DecorationSet.create(state.doc, decorations)
+    }
+  },
+  view: view => {
+    const ystate = prosemirrorPluginKey.getState(view.state)
+    const y = ystate.y
+    const awarenessListener = () => {
+      view.updateState(view.state)
+    }
+    const updateCursorInfo = () => {
+      const current = y.getLocalAwarenessInfo()
+      if (view.hasFocus() && ystate.binding !== null) {
+        const anchor = absolutePositionToRelativePosition(view.state.selection.anchor, ystate.type, ystate.binding.mapping)
+        const head = absolutePositionToRelativePosition(view.state.selection.head, ystate.type, ystate.binding.mapping)
+        if (current.cursor == null || !YPos.equal(current.cursor.anchor, anchor) || !YPos.equal(current.cursor.head, head)) {
+          y.setAwarenessField('cursor', {
+            anchor, head
+          })
+        }
+      } else if (current.cursor !== null) {
+        y.setAwarenessField('cursor', null)
+      }
+    }
+    y.on('awareness', awarenessListener)
+    view.dom.addEventListener('focusin', updateCursorInfo)
+    view.dom.addEventListener('focusout', updateCursorInfo)
+    return {
+      update: updateCursorInfo,
+      destroy: () => {
+        const y = prosemirrorPluginKey.getState(view.state).y
+        y.setAwarenessField('cursor', null)
+        y.off('awareness', awarenessListener)
+      }
+    }
+  }
+})
+
+/**
+ * Transforms a Prosemirror based absolute position to a Yjs based relative position.
+ *
+ * @param {number} pos
+ * @param {YXmlFragment} type
+ * @param {ProsemirrorMapping} mapping
+ * @return {any} relative position
+ */
+export const absolutePositionToRelativePosition = (pos, type, mapping) => {
+  if (pos === 0) {
+    return YPos.getRelativePosition(type, 0)
+  }
+  let n = type._first
+  if (n !== null) {
+    while (type !== n) {
+      const pNodeSize = (mapping.get(n) || { nodeSize: 0 }).nodeSize
+      if (n.constructor === YText) {
+        if (n.length >= pos) {
+          return YPos.getRelativePosition(n, pos)
+        } else {
+          pos -= n.length
+        }
+        if (n._next !== null) {
+          n = n._next
+        } else {
+          do {
+            n = n._parent
+            pos--
+          } while (n._next === null && n !== type)
+          if (n !== type) {
+            n = n._next
+          }
+        }
+      } else if (n._first !== null && pos < pNodeSize) {
+        n = n._first
+        pos--
+      } else {
+        if (pos === 1 && n.length === 0 && pNodeSize > 1) {
+          // edge case, should end in this paragraph
+          return ['endof', n._id.user, n._id.clock, null, null]
+        }
+        pos -= pNodeSize
+        if (n._next !== null) {
+          n = n._next
+        } else {
+          if (pos === 0) {
+            n = n._parent
+            return ['endof', n._id.user, n._id.clock || null, n._id.name || null, n._id.type || null]
+          }
+          do {
+            n = n._parent
+            pos--
+          } while (n._next === null && n !== type)
+          if (n !== type) {
+            n = n._next
+          }
+        }
+      }
+      if (pos === 0 && n.constructor !== YText && n !== type) { // TODO: set to <= 0
+        return [n._id.user, n._id.clock]
+      }
+    }
+  }
+  return YPos.getRelativePosition(type, type.length)
+}
+
+/**
+ * @param {YXmlFragment} yDoc Top level type that is bound to pView
+ * @param {any} relPos Encoded Yjs based relative position
+ * @param {ProsemirrorMapping} mapping
+ */
+export const relativePositionToAbsolutePosition = (yDoc, relPos, mapping) => {
+  const decodedPos = YPos.fromRelativePosition(yDoc._y, relPos)
+  if (decodedPos === null) {
+    return null
+  }
+  let type = decodedPos.type
+  let pos = 0
+  if (type.constructor === YText) {
+    pos = decodedPos.offset
+  } else if (!type._deleted) {
+    let n = type._first
+    let i = 0
+    while (i < type.length && i < decodedPos.offset && n !== null) {
+      i++
+      pos += mapping.get(n).nodeSize
+      n = n._next
+    }
+    pos += 1 // increase because we go out of n
+  }
+  while (type !== yDoc) {
+    const parent = type._parent
+    if (!parent._deleted) {
+      pos += 1 // the start tag
+      let n = parent._first
+      // now iterate until we found type
+      while (n !== null) {
+        if (n === type) {
+          break
+        }
+        pos += mapping.get(n).nodeSize
+        n = n._next
+      }
+    }
+    type = parent
+  }
+  return pos - 1 // we don't count the most outer tag, because it is a fragment
+}
+
+/**
+ * Binding for prosemirror.
+ *
+ * @protected
+ */
+export class ProsemirrorBinding {
+  /**
+   * @param {YXmlFragment} yXmlFragment The bind source
+   * @param {EditorView} prosemirrorView The target binding
+   */
+  constructor (yXmlFragment, prosemirrorView) {
+    this.type = yXmlFragment
+    this.prosemirrorView = prosemirrorView
+    this.mux = createMutex()
+    /**
+     * @type {ProsemirrorMapping}
+     */
+    this.mapping = new Map()
+    this._observeFunction = this._typeChanged.bind(this)
+    this.y = yXmlFragment._y
+    /**
+     * current selection as relative positions in the Yjs model
+     */
+    this._relSelection = null
+    this.y.on('beforeTransaction', e => {
+      this._relSelection = {
+        anchor: absolutePositionToRelativePosition(this.prosemirrorView.state.selection.anchor, yXmlFragment, this.mapping),
+        head: absolutePositionToRelativePosition(this.prosemirrorView.state.selection.head, yXmlFragment, this.mapping)
+      }
+    })
+    yXmlFragment.observeDeep(this._observeFunction)
+  }
+  _forceRerender () {
+    this.mapping = new Map()
+    this.mux(() => {
+      const fragmentContent = this.type.toArray().map(t => createNodeFromYElement(t, this.prosemirrorView.state.schema, this.mapping)).filter(n => n !== null)
+      const tr = this.prosemirrorView.state.tr.replace(0, this.prosemirrorView.state.doc.content.size, new PModel.Slice(new PModel.Fragment(fragmentContent), 0, 0))
+      this.prosemirrorView.dispatch(tr)
+    })
+  }
+  /**
+   *
+   * @param {*} snapshot
+   * @param {*} prevSnapshot
+   */
+  _renderSnapshot (snapshot, prevSnapshot) {
+    // clear mapping because we are going to rerender
+    this.mapping = new Map()
+    this.mux(() => {
+      const fragmentContent = this.type.toArray({ sm: snapshot.sm, ds: prevSnapshot.ds}).map(t => createNodeFromYElement(t, this.prosemirrorView.state.schema, new Map(), snapshot, prevSnapshot)).filter(n => n !== null)
+      const tr = this.prosemirrorView.state.tr.replace(0, this.prosemirrorView.state.doc.content.size, new PModel.Slice(new PModel.Fragment(fragmentContent), 0, 0))
+      this.prosemirrorView.dispatch(tr)
+    })
+  }
+  _typeChanged (events, transaction) {
+    if (events.length === 0 || prosemirrorPluginKey.getState(this.prosemirrorView.state).snapshot != null) {
+      // drop out if snapshot is active
+      return
+    }
+    console.info('new types:', transaction.newTypes.size, 'deleted types:', transaction.deletedStructs.size, transaction.newTypes, transaction.deletedStructs)
+    this.mux(() => {
+      const delStruct = (_, struct) => this.mapping.delete(struct)
+      transaction.deletedStructs.forEach(struct => this.mapping.delete(struct))
+      transaction.changedTypes.forEach(delStruct)
+      transaction.changedParentTypes.forEach(delStruct)
+      const fragmentContent = this.type.toArray().map(t => createNodeIfNotExists(t, this.prosemirrorView.state.schema, this.mapping)).filter(n => n !== null)
+      const tr = this.prosemirrorView.state.tr.replace(0, this.prosemirrorView.state.doc.content.size, new PModel.Slice(new PModel.Fragment(fragmentContent), 0, 0))
+      const relSel = this._relSelection
+      if (relSel !== null && relSel.anchor !== null && relSel.head !== null) {
+        const anchor = relativePositionToAbsolutePosition(this.type, relSel.anchor, this.mapping)
+        const head = relativePositionToAbsolutePosition(this.type, relSel.head, this.mapping)
+        if (anchor !== null && head !== null) {
+          tr.setSelection(TextSelection.create(tr.doc, anchor, head))
+        }
+      }
+      this.prosemirrorView.dispatch(tr)
+    })
+  }
+  _prosemirrorChanged (doc) {
+    this.mux(() => {
+      updateYFragment(this.type, doc.content, this.mapping)
+    })
+  }
+  destroy () {
+    this.type.unobserveDeep(this._observeFunction)
+  }
+}
+
+/**
+ * @private
+ * @param {YXmlElement} el
+ * @param {PModel.Schema} schema
+ * @param {ProsemirrorMapping} mapping
+ * @param {HistorySnapshot} [snapshot]
+ * @param {HistorySnapshot} [prevSnapshot]
+ * @return {PModel.Node}
+ */
+export const createNodeIfNotExists = (el, schema, mapping, snapshot, prevSnapshot) => {
+  const node = mapping.get(el)
+  if (node === undefined) {
+    return createNodeFromYElement(el, schema, mapping, snapshot, prevSnapshot)
+  }
+  return node
+}
+
+/**
+ * @private
+ * @param {YXmlElement} el
+ * @param {PModel.Schema} schema
+ * @param {ProsemirrorMapping} mapping
+ * @param {import('../protocols/history.js').HistorySnapshot} [snapshot]
+ * @param {import('../protocols/history.js').HistorySnapshot} [prevSnapshot]
+ * @return {PModel.Node | null} Returns node if node could be created. Otherwise it deletes the yjs type and returns null
+ */
+export const createNodeFromYElement = (el, schema, mapping, snapshot, prevSnapshot) => {
+  let _snapshot = snapshot
+  let _prevSnapshot = prevSnapshot
+  if (snapshot !== undefined && prevSnapshot !== undefined) {
+    if (!isVisible(el, snapshot)) {
+      // if this element is already rendered as deleted (ychange), then do not render children as deleted
+      _snapshot = {sm: snapshot.sm, ds: prevSnapshot.ds}
+      _prevSnapshot = _snapshot
+    } else if (!isVisible(el, prevSnapshot)) {
+      _prevSnapshot = _snapshot
+    }
+  }
+  const children = []
+  const createChildren = type => {
+    if (type.constructor === YXmlElement) {
+      const n = createNodeIfNotExists(type, schema, mapping, _snapshot, _prevSnapshot)
+      if (n !== null) {
+        children.push(n)
+      }
+    } else {
+      const ns = createTextNodesFromYText(type, schema, mapping, _snapshot, _prevSnapshot)
+      if (ns !== null) {
+        ns.forEach(textchild => {
+          if (textchild !== null) {
+            children.push(textchild)
+          }
+        })
+      }
+    }
+  }
+  if (snapshot === undefined || prevSnapshot === undefined) {
+    el.toArray().forEach(createChildren)
+  } else {
+    el.toArray({sm: snapshot.sm, ds: prevSnapshot.ds}).forEach(createChildren)
+  }
+  let node
+  try {
+    const attrs = el.getAttributes(_snapshot)
+    if (snapshot !== undefined) {
+      if (!isVisible(el, snapshot)) {
+        attrs.ychange = { user: el._id.user, state: 'removed' }
+      } else if (!isVisible(el, prevSnapshot)) {
+        attrs.ychange = { user: el._id.user, state: 'added' }
+      }
+    }
+    node = schema.node(el.nodeName.toLowerCase(), attrs, children)
+  } catch (e) {
+    // an error occured while creating the node. This is probably a result because of a concurrent action.
+    // ignore the node while rendering
+    /* do not delete anymore
+    el._y.transact(() => {
+      el._delete(el._y, true)
+    })
+    */
+    return null
+  }
+  mapping.set(el, node)
+  return node
+}
+
+/**
+ * @private
+ * @param {YText} text
+ * @param {PModel.Schema} schema
+ * @param {ProsemirrorMapping} mapping
+ * @param {HistorySnapshot} [snapshot]
+ * @param {HistorySnapshot} [prevSnapshot]
+ * @return {Array<PModel.Node>}
+ */
+export const createTextNodesFromYText = (text, schema, mapping, snapshot, prevSnapshot) => {
+  const nodes = []
+  const deltas = text.toDelta(snapshot, prevSnapshot)
+  try {
+    for (let i = 0; i < deltas.length; i++) {
+      const delta = deltas[i]
+      const marks = []
+      for (let markName in delta.attributes) {
+        marks.push(schema.mark(markName, delta.attributes[markName]))
+      }
+      nodes.push(schema.text(delta.insert, marks))
+    }
+    if (nodes.length > 0) {
+      mapping.set(text, nodes[0]) // only map to first child, all following children are also considered bound to this type
+    }
+  } catch (e) {
+    /*
+    text._y.transact(() => {
+      text._delete(text._y, true)
+    })
+    */
+    return null
+  }
+  return nodes
+}
+
+/**
+ * @private
+ * @param {PModel.Node} node
+ * @param {ProsemirrorMapping} mapping
+ * @return {YXmlElement | YText}
+ */
+export const createTypeFromNode = (node, mapping) => {
+  let type
+  if (node.isText) {
+    type = new YText()
+    const attrs = {}
+    node.marks.forEach(mark => {
+      if (mark.type.name !== 'ychange') {
+        attrs[mark.type.name] = mark.attrs
+      }
+    })
+    type.insert(0, node.text, attrs)
+  } else {
+    type = new YXmlElement(node.type.name)
+    for (let key in node.attrs) {
+      const val = node.attrs[key]
+      if (val !== null && key !== 'ychange') {
+        type.setAttribute(key, val)
+      }
+    }
+    const ins = []
+    for (let i = 0; i < node.childCount; i++) {
+      ins.push(createTypeFromNode(node.child(i), mapping))
+    }
+    type.insert(0, ins)
+  }
+  mapping.set(type, node)
+  return type
+}
+
+const equalAttrs = (pattrs, yattrs) => {
+  const keys = Object.keys(pattrs).filter(key => pattrs[key] === null)
+  let eq = keys.length === Object.keys(yattrs).filter(key => yattrs[key] === null).length
+  for (let i = 0; i < keys.length && eq; i++) {
+    const key = keys[i]
+    const l = pattrs[key]
+    const r = yattrs[key]
+    eq = key === 'ychange' || l === r || (typeof l === 'object' && typeof r === 'object' && equalAttrs(l, r))
+  }
+  return eq
+}
+
+const equalYTextPText = (ytext, ptext) => {
+  const d = ytext.toDelta()[0]
+  return d.insert === ptext.text && object.keys(d.attributes || {}).length === ptext.marks.length && ptext.marks.every(mark => equalAttrs(d.attributes[mark.type.name], mark.attrs))
+}
+
+const equalYTypePNode = (ytype, pnode) =>
+  ytype.constructor === YText
+    ? equalYTextPText(ytype, pnode)
+    : (matchNodeName(ytype, pnode) && ytype.length === pnode.childCount && equalAttrs(ytype.getAttributes(), pnode.attrs) && ytype.toArray().every((ychild, i) => equalYTypePNode(ychild, pnode.child(i))))
+
+const computeChildEqualityFactor = (ytype, pnode, mapping) => {
+  const yChildren = ytype.toArray()
+  const pChildCnt = pnode.childCount
+  const yChildCnt = yChildren.length
+  const minCnt = math.min(yChildCnt, pChildCnt)
+  let left = 0
+  let right = 0
+  let foundMappedChild = false
+  for (; left < minCnt; left++) {
+    const leftY = yChildren[left]
+    const leftP = pnode.child(left)
+    if (mapping.get(leftY) === leftP) {
+      foundMappedChild = true// definite (good) match!
+    } else if (!equalYTypePNode(leftY, leftP)) {
+      break
+    }
+  }
+  for (; left + right < minCnt; right++) {
+    const rightY = yChildren[yChildCnt - right - 1]
+    const rightP = pnode.child(pChildCnt - right - 1)
+    if (mapping.get(rightY) !== rightP) {
+      foundMappedChild = true
+    } else if (!equalYTypePNode(rightP, rightP)) {
+      break
+    }
+  }
+  return {
+    equalityFactor: left + right,
+    foundMappedChild
+  }
+}
+
+/**
+ * @private
+ * @param {YXmlFragment} yDomFragment
+ * @param {PModel.Node} pContent
+ * @param {ProsemirrorMapping} mapping
+ */
+const updateYFragment = (yDomFragment, pContent, mapping) => {
+  if (yDomFragment instanceof YXmlElement && yDomFragment.nodeName.toLowerCase() !== pContent.type.name) {
+    throw new Error('node name mismatch!')
+  }
+  mapping.set(yDomFragment, pContent)
+  // update attributes
+  if (yDomFragment instanceof YXmlElement) {
+    const yDomAttrs = yDomFragment.getAttributes()
+    const pAttrs = pContent.attrs
+    for (let key in pAttrs) {
+      if (pAttrs[key] !== null) {
+        if (yDomAttrs[key] !== pAttrs[key] && key !== 'ychange') {
+          yDomFragment.setAttribute(key, pAttrs[key])
+        }
+      } else {
+        yDomFragment.removeAttribute(key)
+      }
+    }
+    // remove all keys that are no longer in pAttrs
+    for (let key in yDomAttrs) {
+      if (pAttrs[key] === undefined) {
+        yDomFragment.removeAttribute(key)
+      }
+    }
+  }
+  // update children
+  const pChildCnt = pContent.childCount
+  const yChildren = yDomFragment.toArray()
+  const yChildCnt = yChildren.length
+  const minCnt = math.min(pChildCnt, yChildCnt)
+  let left = 0
+  let right = 0
+  // find number of matching elements from left
+  for (;left < minCnt; left++) {
+    const leftY = yChildren[left]
+    const leftP = pContent.child(left)
+    if (mapping.get(leftY) !== leftP) {
+      if (equalYTypePNode(leftY, leftP)) {
+        // update mapping
+        mapping.set(leftY, leftP)
+      } else {
+        break
+      }
+    }
+  }
+  // find number of matching elements from right
+  for (;right + left < minCnt; right++) {
+    const rightY = yChildren[yChildCnt - right - 1]
+    const rightP = pContent.child(pChildCnt - right - 1)
+    if (mapping.get(rightY) !== rightP) {
+      if (equalYTypePNode(rightY, rightP)) {
+        // update mapping
+        mapping.set(rightY, rightP)
+      } else {
+        break
+      }
+    }
+  }
+  yDomFragment._y.transact(() => {
+    // try to compare and update
+    while (yChildCnt - left - right > 0 && pChildCnt - left - right > 0) {
+      const leftY = yChildren[left]
+      const leftP = pContent.child(left)
+      const rightY = yChildren[yChildCnt - right - 1]
+      const rightP = pContent.child(pChildCnt - right - 1)
+      if (leftY.constructor === YText && leftP.isText) {
+        if (!equalYTextPText(leftY, leftP)) {
+          // try to apply diff. Only if attrs don't match, delete insert
+          // TODO: use a single ytext to hold all following Prosemirror Text nodes
+          const pattrs = {}
+          leftP.marks.forEach(mark => {
+            if (mark.type.name !== 'ychange') {
+              pattrs[mark.type.name] = mark.attrs
+            }
+          })
+          const delta = leftY.toDelta()
+          if (delta.length === 1 && delta[0].insert && equalAttrs(pattrs, delta[0].attributes || {})) {
+            const diff = simpleDiff(delta[0].insert, leftP.text)
+            leftY.delete(diff.pos, diff.remove)
+            leftY.insert(diff.pos, diff.insert)
+          } else {
+            yDomFragment.delete(left, 1)
+            yDomFragment.insert(left, [createTypeFromNode(leftP, mapping)])
+          }
+        }
+        left += 1
+      } else {
+        let updateLeft = matchNodeName(leftY, leftP)
+        let updateRight = matchNodeName(rightY, rightP)
+        if (updateLeft && updateRight) {
+          // decide which which element to update
+          const equalityLeft = computeChildEqualityFactor(leftY, leftP, mapping)
+          const equalityRight = computeChildEqualityFactor(rightY, rightP, mapping)
+          if (equalityLeft.foundMappedChild && !equalityRight.foundMappedChild) {
+            updateRight = false
+          } else if (!equalityLeft.foundMappedChild && equalityRight.foundMappedChild) {
+            updateLeft = false
+          } else if (equalityLeft.equalityFactor < equalityRight.equalityFactor) {
+            updateLeft = false
+          } else {
+            updateRight = false
+          }
+        }
+        if (updateLeft) {
+          updateYFragment(leftY, leftP, mapping)
+          left += 1
+        } else if (updateRight) {
+          updateYFragment(rightY, rightP, mapping)
+          right += 1
+        } else {
+          yDomFragment.delete(left, 1)
+          yDomFragment.insert(left, [createTypeFromNode(leftP, mapping)])
+          left += 1
+        }
+      }
+    }
+    const yDelLen = yChildCnt - left - right
+    if (yDelLen > 0) {
+      yDomFragment.delete(left, yDelLen)
+    }
+    if (left + right < pChildCnt) {
+      const ins = []
+      for (let i = left; i < pChildCnt - right; i++) {
+        ins.push(createTypeFromNode(pContent.child(i), mapping))
+      }
+      yDomFragment.insert(left, ins)
+    }
+  })
+}
+
+/**
+ * @function
+ * @param {YXmlElement} yElement
+ * @param {any} pNode Prosemirror Node
+ */
+const matchNodeName = (yElement, pNode) => yElement.nodeName === pNode.type.name.toUpperCase()

bindings/prosemirror.mjs

@@ -1,299 +0,0 @@
-/**
- * @module bindings/prosemirror
- */
-
-import { BindMapping } from '../utils/BindMapping.mjs'
-import { YText } from '../types/YText.mjs' // eslint-disable-line
-import { YXmlElement, YXmlFragment } from '../types/YXmlElement.mjs' // eslint-disable-line
-import { createMutex } from '../lib/mutex.mjs'
-import * as PModel from 'prosemirror-model'
-import { EditorView,  Decoration, DecorationSet } from 'prosemirror-view' // eslint-disable-line
-import { Plugin, PluginKey, EditorState } from 'prosemirror-state' // eslint-disable-line
-
-/**
- * @typedef {BindMapping<YText | YXmlElement, PModel.Node>} ProsemirrorMapping
- */
-
-/**
- * The unique prosemirror plugin key for prosemirrorPlugin.
- *
- * @public
- */
-export const prosemirrorPluginKey = new PluginKey('yjs')
-
-/**
- * This plugin listens to changes in prosemirror view and keeps yXmlState and view in sync.
- *
- * This plugin also keeps references to the type and the shared document so other plugins can access it.
- * @param {YXmlFragment} yXmlFragment
- * @return {Plugin} Returns a prosemirror plugin that binds to this type
- */
-export const prosemirrorPlugin = yXmlFragment => {
-  const pluginState = {
-    type: yXmlFragment,
-    y: yXmlFragment._y,
-    binding: null
-  }
-  const plugin = new Plugin({
-    key: prosemirrorPluginKey,
-    state: {
-      init: (initargs, state) => {
-        return pluginState
-      },
-      apply: (tr, pluginState) => {
-        return pluginState
-      }
-    },
-    view: view => {
-      const binding = new ProsemirrorBinding(yXmlFragment, view)
-      pluginState.binding = binding
-      return {
-        update: () => {
-          binding._prosemirrorChanged()
-        },
-        destroy: () => {
-          binding.destroy()
-        }
-      }
-    }
-  })
-  return plugin
-}
-
-/**
- * The unique prosemirror plugin key for cursorPlugin.
- *
- * @public
- */
-export const cursorPluginKey = new PluginKey('yjs-cursor')
-
-/**
- * A prosemirror plugin that listens to awareness information on Yjs.
- * This requires that a `prosemirrorPlugin` is also bound to the prosemirror.
- *
- * @public
- */
-export const cursorPlugin = new Plugin({
-  key: cursorPluginKey,
-  props: {
-    decorations: state => {
-      const y = prosemirrorPluginKey.getState(state).y
-      const awareness = y.getAwarenessInfo()
-      const decorations = []
-      awareness.forEach((state, userID) => {
-        if (state.cursor != null) {
-          const username = `User: ${userID}`
-          decorations.push(Decoration.widget(state.cursor.from, () => {
-            const cursor = document.createElement('span')
-            cursor.classList.add('ProseMirror-yjs-cursor')
-            const user = document.createElement('div')
-            user.insertBefore(document.createTextNode(username), null)
-            cursor.insertBefore(user, null)
-            return cursor
-          }, { key: username }))
-          decorations.push(Decoration.inline(state.cursor.from, state.cursor.to, { style: 'background-color: #ffa50070' }))
-        }
-      })
-      return DecorationSet.create(state.doc, decorations)
-    }
-  },
-  view: view => {
-    const y = prosemirrorPluginKey.getState(view.state).y
-    const awarenessListener = () => {
-      view.updateState(view.state)
-    }
-    y.on('awareness', awarenessListener)
-    return {
-      update: () => {
-        const y = prosemirrorPluginKey.getState(view.state).y
-        const from = view.state.selection.from
-        const to = view.state.selection.to
-        const current = y.getLocalAwarenessInfo()
-        if (current.cursor == null || current.cursor.to !== to || current.cursor.from !== from) {
-          y.setAwarenessField('cursor', {
-            from, to
-          })
-        }
-      },
-      destroy: () => {
-        const y = prosemirrorPluginKey.getState(view.state).y
-        y.setAwarenessField('cursor', null)
-        y.off('awareness', awarenessListener)
-      }
-    }
-  }
-})
-
-/**
- * Binding for prosemirror.
- *
- * @protected
- */
-export class ProsemirrorBinding {
-  /**
-   * @param {YXmlFragment} yXmlFragment The bind source
-   * @param {EditorView} prosemirrorView The target binding
-   */
-  constructor (yXmlFragment, prosemirrorView) {
-    this.type = yXmlFragment
-    this.prosemirrorView = prosemirrorView
-    this.mux = createMutex()
-    /**
-     * @type {ProsemirrorMapping}
-     */
-    this.mapping = new BindMapping()
-    this._observeFunction = this._typeChanged.bind(this)
-    yXmlFragment.observeDeep(this._observeFunction)
-  }
-  _typeChanged (events) {
-    if (events.length === 0) {
-      return
-    }
-    this.mux(() => {
-      events.forEach(event => {
-        // recompute node for each parent
-        // except main node, compute main node in the end
-        let target = event.target
-        if (target !== this.type) {
-          do {
-            if (target.constructor === YXmlElement) {
-              createNodeFromYElement(target, this.prosemirrorView.state.schema, this.mapping)
-            }
-            target = target._parent
-          } while (target._parent !== this.type)
-        }
-      })
-      const fragmentContent = this.type.toArray().map(t => createNodeIfNotExists(t, this.prosemirrorView.state.schema, this.mapping))
-      const tr = this.prosemirrorView.state.tr.replace(0, this.prosemirrorView.state.doc.content.size, new PModel.Slice(new PModel.Fragment(fragmentContent), 0, 0))
-      this.prosemirrorView.updateState(this.prosemirrorView.state.apply(tr))
-    })
-  }
-  _prosemirrorChanged () {
-    this.mux(() => {
-      updateYFragment(this.type, this.prosemirrorView.state, this.mapping)
-    })
-  }
-  destroy () {
-    this.type.unobserveDeep(this._observeFunction)
-  }
-}
-
-/**
- * @private
- * @param {Y.XmlElement} el
- * @param {PModel.Schema} schema
- * @param {ProsemirrorMapping} mapping
- * @return {PModel.Node}
- */
-export const createNodeIfNotExists = (el, schema, mapping) => {
-  const node = mapping.getY(el)
-  if (node === undefined) {
-    return createNodeFromYElement(el, schema, mapping)
-  }
-  return node
-}
-
-/**
- * @private
- * @param {Y.XmlElement} el
- * @param {PModel.Schema} schema
- * @param {ProsemirrorMapping} mapping
- * @return {PModel.Node}
- */
-export const createNodeFromYElement = (el, schema, mapping) => {
-  const children = []
-  el.toArray().forEach(type => {
-    if (type.constructor === YXmlElement) {
-      children.push(createNodeIfNotExists(type, schema, mapping))
-    } else {
-      children.concat(createTextNodesFromYText(type, schema, mapping)).forEach(textchild => children.push(textchild))
-    }
-  })
-  const node = schema.node(el.nodeName.toLowerCase(), el.getAttributes(), el.toArray().map(t => createNodeIfNotExists(t, schema, mapping)))
-  mapping.bind(el, node)
-  return node
-}
-
-/**
- * @private
- * @param {Y.Text} text
- * @param {PModel.Schema} schema
- * @param {ProsemirrorMapping} mapping
- * @return {Array<PModel.Node>}
- */
-export const createTextNodesFromYText = (text, schema, mapping) => {
-  const nodes = []
-  const deltas = text.toDelta()
-  for (let i = 0; i < deltas.length; i++) {
-    const delta = deltas[i]
-    const marks = []
-    for (let markName in delta.attributes) {
-      marks.push(schema.mark(markName, delta.attributes[markName]))
-    }
-    nodes.push(schema.text(delta.insert, marks))
-  }
-  if (nodes.length > 0) {
-    mapping.bind(text, nodes[0]) // only map to first child, all following children are also considered bound to this type
-  }
-  return nodes
-}
-
-/**
- * @private
- * @param {PModel.Node} node
- * @param {ProsemirrorMapping} mapping
- * @return {YXmlElement | YText}
- */
-export const createTypeFromNode = (node, mapping) => {
-  let type
-  if (node.isText) {
-    type = new YText()
-    const attrs = {}
-    node.marks.forEach(mark => { attrs[mark.type.name] = mark.attrs })
-    type.insert(0, node.text, attrs)
-  } else {
-    type = new YXmlElement(node.type.name)
-    for (let key in node.attrs) {
-      type.setAttribute(key, node.attrs[key])
-    }
-    type.insert(0, node.content.content.map(node => createTypeFromNode(node, mapping)))
-  }
-  mapping.bind(type, node)
-  return type
-}
-
-/**
- * @private
- * @param {YXmlFragment} yDomFragment
- * @param {EditorState} state
- * @param {BindMapping} mapping
- */
-const updateYFragment = (yDomFragment, state, mapping) => {
-  const pChildCnt = state.doc.content.childCount
-  const yChildren = yDomFragment.toArray()
-  const yChildCnt = yChildren.length
-  const minCnt = pChildCnt < yChildCnt ? pChildCnt : yChildCnt
-  let left = 0
-  let right = 0
-  // find number of matching elements from left
-  for (;left < minCnt; left++) {
-    if (state.doc.content.child(left) !== mapping.getY(yChildren[left])) {
-      break
-    }
-  }
-  // find number of matching elements from right
-  for (;right < minCnt; right++) {
-    if (state.doc.content.child(pChildCnt - right - 1) !== mapping.getY(yChildren[yChildCnt - right - 1])) {
-      break
-    }
-  }
-  if (left + right > pChildCnt) {
-    // nothing changed
-    return
-  }
-  yDomFragment._y.transact(() => {
-    // now update y to match editor state
-    yDomFragment.delete(left, yChildCnt - left - right)
-    yDomFragment.insert(left, state.doc.content.content.slice(left, pChildCnt - right).map(node => createTypeFromNode(node, mapping)))
-  })
-}

bindings/quill.js

@@ -2,7 +2,7 @@
  * @module bindings/quill
  */
 
-import { createMutex } from '../lib/mutex.mjs'
+import { createMutex } from '../lib/mutex.js'
 
 const typeObserver = function (event) {
   const quill = this.target
@@ -53,7 +53,6 @@ export class QuillBinding {
      * @private
      */
     this._mutualExclude = createMutex()
-    
     // Set initial value.
     quill.setContents(textType.toDelta(), 'yjs')
     // Observers are handled by this class.

bindings/textarea.js

@@ -2,9 +2,9 @@
  * @module bindings/textarea
  */
 
-import { simpleDiff } from '../lib/diff.mjs'
-import { getRelativePosition, fromRelativePosition } from '../utils/relativePosition.mjs'
-import { createMutex } from '../lib/mutex.mjs'
+import { simpleDiff } from '../lib/diff.js'
+import { getRelativePosition, fromRelativePosition } from '../utils/relativePosition.js'
+import { createMutex } from '../lib/mutex.js'
 
 function typeObserver () {
   this._mutualExclude(() => {

examples/codemirror.html

@@ -0,0 +1,70 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Yjs CodeMirror Example</title>
+    <link rel=stylesheet href="https://codemirror.net/lib/codemirror.css">
+    <style>
+      #container {
+        border: grey;
+        border-style: solid;
+        border-width: thin;
+      }
+    </style>
+  </head>
+<body>
+  <p>This example shows how to bind a YText type to <a href="https://codemirror.net/">CodeMirror</a> editor.</p>
+  <p>The content of this editor is shared with every client who visits this domain.</p>
+  <div class="code-html">
+<style>
+.remote-caret {
+  position: absolute;
+  border-left: black;
+  border-left-style: solid;
+  border-left-width: 2px;
+  height: 1em;
+}
+.remote-caret > div {
+  position: relative;
+  top: -1.05em;
+  font-size: 13px;
+  background-color: rgb(250, 129, 0);
+  font-family: serif;
+  font-style: normal;
+  font-weight: normal;
+  line-height: normal;
+  user-select: none;
+  color: white;
+  padding-left: 2px;
+  padding-right: 2px;
+}
+</style>
+<div id="container"></div>
+  </div>
+  <!-- The actual source file for the following code is found in ./codemirror.js. Run `npm run watch` to compile the files -->
+  <script class="code-js" src="./build/codemirror.js">
+import * as Y from 'yjs'
+import { WebsocketProvider } from 'yjs/provider/websocket.js'
+import { CodeMirrorBinding } from 'yjs/bindings/codemirror.js'
+
+import * as conf from './exampleConfig.js'
+
+import CodeMirror from 'codemirror'
+import 'codemirror/mode/javascript/javascript.js'
+
+const provider = new WebsocketProvider(conf.serverAddress)
+const ydocument = provider.get('codemirror')
+const ytext = ydocument.define('codemirror', Y.Text)
+
+const editor = new CodeMirror(document.querySelector('#container'), {
+    mode: 'javascript',
+    lineNumbers: true
+})
+
+const binding = new CodeMirrorBinding(ytext, editor)
+
+window.codemirrorExample = {
+  binding, editor, ytext, ydocument
+}
+  </script>
+</body>
+</html>

examples/codemirror.js

@@ -0,0 +1,23 @@
+import * as Y from '../index.js'
+import { WebsocketProvider } from '../provider/websocket.js'
+import { CodeMirrorBinding } from '../bindings/codemirror.js'
+
+import * as conf from './exampleConfig.js'
+
+import CodeMirror from 'codemirror'
+import 'codemirror/mode/javascript/javascript.js'
+
+const provider = new WebsocketProvider(conf.serverAddress)
+const ydocument = provider.get('codemirror')
+const ytext = ydocument.define('codemirror', Y.Text)
+
+const editor = new CodeMirror(document.querySelector('#container'), {
+  mode: 'javascript',
+  lineNumbers: true
+})
+
+const binding = new CodeMirrorBinding(ytext, editor)
+
+window.codemirrorExample = {
+  binding, editor, ytext, ydocument
+}

examples/dom.html

@@ -10,15 +10,21 @@
     </style>
 </head>
 <body>
+  <p>This example shows how to bind a YXmlFragment type to an arbitrary DOM element. We set the DOM element to contenteditable so it basically behaves like a very powerful rich-text editor.</p>
+  <p>The content of this editor is shared with every client who visits this domain.</p>
+  <hr>
   <div class="code-html">
-    <div id="content" contenteditable=""></div>
-  </div>
-  <script class="code-js" src="./build/dom.js">
-    import * as Y from 'yjs/index.mjs'
-    import { WebsocketProvider } from 'yjs/provider/websocket.mjs'
-    import { DomBinding } from 'yjs/bindings/dom.mjs'
 
-    const provider = new WebsocketProvider('wss://api.yjs.website')
+<div id="content" contenteditable=""></div>
+  </div>
+  <!-- The actual source file for the following code is found in ./dom.js. Run `npm run watch` to compile the files -->
+  <script class="code-js" src="./build/dom.js">
+    import * as Y from 'yjs/index.js'
+    import { WebsocketProvider } from 'yjs/provider/websocket.js'
+    import { DomBinding } from 'yjs/bindings/dom.js'
+    import * as conf from './exampleConfig.js'
+
+    const provider = new WebsocketProvider(conf.serverAddress)
     const ydocument = provider.get('dom')
     const type = ydocument.define('xml', Y.XmlFragment)
     const binding = new DomBinding(type, document.querySelector('#content'), { scrollingElement: document.scrollingElement })

examples/dom.js

@@ -1,8 +1,9 @@
-import * as Y from '../index.mjs'
-import { WebsocketProvider } from '../provider/websocket.mjs'
-import { DomBinding } from '../bindings/dom.mjs'
+import * as Y from '../index.js'
+import { WebsocketProvider } from '../provider/websocket.js'
+import { DomBinding } from '../bindings/dom.js'
+import * as conf from './exampleConfig.js'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
+const provider = new WebsocketProvider(conf.serverAddress)
 const ydocument = provider.get('dom')
 const type = ydocument.define('xml', Y.XmlFragment)
 const binding = new DomBinding(type, document.querySelector('#content'), { scrollingElement: document.scrollingElement })

examples/exampleConfig.js

@@ -0,0 +1,9 @@
+/* eslint-env browser */
+
+const isDeployed = location.hostname === 'yjs.website'
+
+if (!isDeployed) {
+  console.log('%cYjs: Start your local websocket server by running %c`npm run websocket-server`', 'color:blue', 'color: grey; font-weight: bold')
+}
+
+export const serverAddress = isDeployed ? 'wss://api.yjs.website' : 'ws://localhost:1234'

examples/examples.json

@@ -1,6 +1,9 @@
 {
+  "codemirror": {
+    "title": "CodeMirror Binding"
+  },
   "prosemirror": {
-    "title": "Prosemirror Binding"
+    "title": "ProseMirror Binding"
   },
   "textarea": {
     "title": "Textarea Binding"

examples/prosemirror-history.js

@@ -0,0 +1,159 @@
+
+import {Plugin} from 'prosemirror-state'
+import crel from 'crel'
+import * as Y from '../index.js'
+import { prosemirrorPluginKey } from '../bindings/prosemirror.js'
+import * as encoding from '../lib/encoding.js'
+import * as decoding from '../lib/decoding.js'
+import * as historyProtocol from '../protocols/history.js'
+
+const niceColors = ['#3cb44b', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#008080', '#9a6324', '#800000', '#808000', '#000075', '#808080']
+
+const createUserCSS = (userid, username, color = 'rgb(250, 129, 0)', color2 = 'rgba(250, 129, 0, .41)') => `
+  [ychange_state][ychange_user="${userid}"]:hover::before {
+    content: "${username}" !important;
+    background-color: ${color} !important;
+  }
+  [ychange_state="added"][ychange_user="${userid}"] {
+    background-color: ${color2} !important;
+  }
+  [ychange_state="removed"][ychange_user="${userid}"] {
+    color: ${color} !important;
+  }
+`
+
+export const noteHistoryPlugin = new Plugin({
+  state: {
+    init (initargs, state) {
+      return new NoteHistoryPlugin()
+    },
+    apply (tr, pluginState) {
+      return pluginState
+    }
+  },
+  view (editorView) {
+    const hstate = noteHistoryPlugin.getState(editorView.state)
+    hstate.init(editorView)
+    return {
+      destroy: hstate.destroy.bind(hstate)
+    }
+  }
+})
+
+const createWrapper = () => {
+  const wrapper = crel('div', { style: 'display: flex;' })
+  const historyContainer = crel('div', { style: 'align-self: baseline; flex-basis: 250px;', class: 'shared-history' })
+  wrapper.insertBefore(historyContainer, null)
+  const userStyleContainer = crel('style')
+  wrapper.insertBefore(userStyleContainer, null)
+  return { wrapper, historyContainer, userStyleContainer }
+}
+
+class NoteHistoryPlugin {
+  init (editorView) {
+    this.editorView = editorView
+    const { historyContainer, wrapper, userStyleContainer } = createWrapper()
+    this.userStyleContainer = userStyleContainer
+    this.wrapper = wrapper
+    this.historyContainer = historyContainer
+    const n = editorView.dom.parentNode.parentNode
+    n.parentNode.replaceChild(this.wrapper, n)
+    n.style['flex-grow'] = '1'
+    wrapper.insertBefore(n, this.wrapper.firstChild)
+    this.render()
+    const y = prosemirrorPluginKey.getState(this.editorView.state).y
+    const history = y.define('history', Y.Array)
+    history.observe(this.render.bind(this))
+  }
+  destroy () {
+    this.wrapper.parentNode.replaceChild(this.wrapper.firstChild, this.wrapper)
+    const y = prosemirrorPluginKey.getState(this.editorView.state).y
+    const history = y.define('history', Y.Array)
+    history.unobserve(this.render)
+  }
+  render () {
+    const y = prosemirrorPluginKey.getState(this.editorView.state).y
+    const history = y.define('history', Y.Array).toArray()
+    const fragment = document.createDocumentFragment()
+    const snapshotBtn = crel('button', { type: 'button' }, ['snapshot'])
+    fragment.insertBefore(snapshotBtn, null)
+    let _prevSnap = null // empty
+    snapshotBtn.addEventListener('click', () => {
+      const awareness = y.getAwarenessInfo()
+      const userMap = new Map()
+      const aw = y.getLocalAwarenessInfo()
+      userMap.set(y.userID, aw.name || 'unknown')
+      awareness.forEach((a, userID) => {
+        userMap.set(userID, a.name || 'Unknown')
+      })
+      this.snapshot(userMap)
+    })
+    history.forEach(buf => {
+      const decoder = decoding.createDecoder(buf)
+      const snapshot = historyProtocol.readHistorySnapshot(decoder)
+      const date = new Date(decoding.readUint32(decoder) * 1000)
+      const restoreBtn = crel('button', { type: 'button' }, ['restore'])
+      const a = crel('a', [
+        '• ' + date.toUTCString(), restoreBtn
+      ])
+      const el = crel('div', [ a ])
+      let prevSnapshot = _prevSnap // rebind to new variable
+      restoreBtn.addEventListener('click', event => {
+        if (prevSnapshot === null) {
+          prevSnapshot = { ds: snapshot.ds, sm: new Map() }
+        }
+        this.editorView.dispatch(this.editorView.state.tr.setMeta(prosemirrorPluginKey, { snapshot, prevSnapshot, restore: true }))
+        event.stopPropagation()
+      })
+      a.addEventListener('click', () => {
+        console.log('setting snapshot')
+        if (prevSnapshot === null) {
+          prevSnapshot = { ds: snapshot.ds, sm: new Map() }
+        }
+        this.renderSnapshot(snapshot, prevSnapshot)
+      })
+      fragment.insertBefore(el, null)
+      _prevSnap = snapshot
+    })
+    this.historyContainer.innerHTML = ''
+    this.historyContainer.insertBefore(fragment, null)
+  }
+  renderSnapshot (snapshot, prevSnapshot) {
+    this.editorView.dispatch(this.editorView.state.tr.setMeta(prosemirrorPluginKey, { snapshot, prevSnapshot }))
+    /**
+     * @type {Array<string|null>}
+     */
+    let colors = niceColors.slice()
+    let style = ''
+    snapshot.userMap.forEach((name, userid) => {
+      /**
+       * @type {any}
+       */
+      const randInt = name.split('').map(s => s.charCodeAt(0)).reduce((a, b) => a + b)
+      let color = null
+      let i = 0
+      for (; i < colors.length && color === null; i++) {
+        color = colors[(randInt + i) % colors.length]
+      }
+      if (color === null) {
+        colors = niceColors.slice()
+        i = 0
+        color = colors[randInt % colors.length]
+      }
+      colors[randInt % colors.length] = null
+      style += createUserCSS(userid, name, color, color + '69')
+    })
+    this.userStyleContainer.innerHTML = style
+  }
+  /**
+   * @param {Map<number, string>} [updatedUserMap] Maps from userid (yjs model) to account name (e.g. mail address)
+   */
+  snapshot (updatedUserMap = new Map()) {
+    const y = prosemirrorPluginKey.getState(this.editorView.state).y
+    const history = y.define('history', Y.Array)
+    const encoder = encoding.createEncoder()
+    historyProtocol.writeHistorySnapshot(encoder, y, updatedUserMap)
+    encoding.writeUint32(encoder, Math.floor(Date.now() / 1000))
+    history.push([encoding.toBuffer(encoder)])
+  }
+}

examples/prosemirror.html

@@ -2,7 +2,7 @@
 <html>
 <head>
     <title>Yjs Prosemirror Example</title>
-    <link rel=stylesheet href="https://prosemirror.net/css/editor.css">
+    <link rel=stylesheet href="./prosemirror.css">
     <style>
       placeholder {
         display: inline;
@@ -16,6 +16,13 @@
         font-weight: bold;
       }
       .ProseMirror img { max-width: 100px }
+      /* this is a rough fix for the first cursor position when the first paragraph is empty */
+      .ProseMirror > .ProseMirror-yjs-cursor:first-child {
+        margin-top: 16px;
+      }
+      .ProseMirror p:first-child, .ProseMirror h1:first-child, .ProseMirror h2:first-child, .ProseMirror h3:first-child, .ProseMirror h4:first-child, .ProseMirror h5:first-child, .ProseMirror h6:first-child {
+        margin-top: 16px
+      }
       .ProseMirror-yjs-cursor {
         position: absolute;
         border-left: black;
@@ -23,6 +30,7 @@
         border-left-width: 2px;
         border-color: orange;
         height: 1em;
+        word-break: normal;
       }
       .ProseMirror-yjs-cursor > div {
         position: relative;
@@ -38,17 +46,53 @@
         padding-left: 2px;
         padding-right: 2px;
       }
+      [ychange_state] {
+        position: relative;
+      }
+      [ychange_state]:hover::before {
+        content: attr(ychange_user);
+        background-color: #fa8100;
+        position: absolute;
+        top: -14px;
+        right: 0;
+        font-size: 12px;
+        padding: 0 2px;
+        border-radius: 3px 3px 0 0;
+        color: #fdfdfe;
+        user-select: none;
+        word-break: normal;
+      }
+      *[ychange_state='added'] {
+        background-color: #fa810069;
+      }
+      ychange[ychange_state='removed'] {
+        color: rgb(250, 129, 0);
+        text-decoration: line-through;
+      }
+      *:not(ychange)[ychange_state='removed'] {
+        background-color: #ff9494c9;
+        text-decoration: line-through;
+      }
+      img[ychange_state='removed'] {
+        padding: 2px;
+      }
     </style>
 </head>
 <body>
+  <p>This example shows how to bind a YXmlFragment type to a <a href="http://prosemirror.net">Prosemirror</a> editor.</p>
+  <p>The content of this editor is shared with every client who visits this domain.</p>
   <div class="code-html">
-    <div id="editor" style="margin-bottom: 23px"></div>
-    <div style="display: none" id="content"></div>
+
+<div id="editor" style="margin-bottom: 23px"></div>
+<div style="display: none" id="content"></div>
   </div>
+  <!-- The actual source file for the following code is found in ./prosemirror.js. Run `npm run watch` to compile the files -->
   <script class="code-js" src="./build/prosemirror.js">
 import * as Y from 'yjs'
-import { WebsocketProvider } from '../provider/websocket.mjs'
-import { prosemirrorPlugin, cursorPlugin } from '../bindings/prosemirror'
+import { WebsocketProvider } from 'yjs/provider/websocket.js'
+import { prosemirrorPlugin, cursorPlugin } from 'yjs/bindings/prosemirror'
+
+import * as conf from './exampleConfig.js'
 
 import { EditorState } from 'prosemirror-state'
 import { EditorView } from 'prosemirror-view'
@@ -56,7 +100,7 @@ import { DOMParser } from 'prosemirror-model'
 import { schema } from 'prosemirror-schema-basic'
 import { exampleSetup } from 'prosemirror-example-setup'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
+const provider = new WebsocketProvider(conf.serverAddress)
 const ydocument = provider.get('prosemirror')
 const type = ydocument.define('prosemirror', Y.XmlFragment)
 

examples/prosemirror.js

@@ -1,21 +1,24 @@
-import * as Y from '../index.mjs'
-import { WebsocketProvider } from '../provider/websocket.mjs'
-import { prosemirrorPlugin, cursorPlugin } from '../bindings/prosemirror'
+import * as Y from '../index.js'
+import { WebsocketProvider } from '../provider/websocket.js'
+import { prosemirrorPlugin, cursorPlugin } from '../bindings/prosemirror.js'
+
+import * as conf from './exampleConfig.js'
 
 import { EditorState } from 'prosemirror-state'
 import { EditorView } from 'prosemirror-view'
-import { DOMParser } from 'prosemirror-model'
-import { schema } from 'prosemirror-schema-basic'
+import { DOMParser, Schema } from 'prosemirror-model'
+import { schema } from './prosemirror-schema.js'
 import { exampleSetup } from 'prosemirror-example-setup'
+import { noteHistoryPlugin } from './prosemirror-history.js'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
-const ydocument = provider.get('prosemirror')
+const provider = new WebsocketProvider(conf.serverAddress)
+const ydocument = provider.get('prosemirror', { gc: false })
 const type = ydocument.define('prosemirror', Y.XmlFragment)
 
 const prosemirrorView = new EditorView(document.querySelector('#editor'), {
   state: EditorState.create({
     doc: DOMParser.fromSchema(schema).parse(document.querySelector('#content')),
-    plugins: exampleSetup({schema}).concat([prosemirrorPlugin(type), cursorPlugin])
+    plugins: exampleSetup({schema}).concat([prosemirrorPlugin(type), cursorPlugin /* noteHistoryPlugin */])
   })
 })
 

examples/quill.html

@@ -1,25 +1,29 @@
 <!DOCTYPE html>
 <html>
 <head>
-    <title>Yjs Prosemirror Example</title>
-    <link rel=stylesheet href="https://prosemirror.net/css/editor.css">
-</head>
+    <title>Yjs Quill Example</title>
+    <link rel="stylesheet" href="https://cdn.quilljs.com/1.3.6/quill.snow.css">
+  </head>
 <body>
-  <link href="https://cdn.quilljs.com/1.3.6/quill.snow.css" rel="stylesheet">
+  <p>This example shows how to bind a YText type to <a href="https://quilljs.com">Quill</a> editor.</p>
+  <p>The content of this editor is shared with every client who visits this domain.</p>
   <div class="code-html">
-    <div id="quill-container">
-      <div id="quill">
-      </div>
-    </div>
+<div id="quill-container">
+  <div id="quill">
   </div>
+</div>
+  </div>
+  <!-- The actual source file for the following code is found in ./quill.js. Run `npm run watch` to compile the files -->
   <script class="code-js" src="./build/quill.js">
 import * as Y from 'yjs'
-import { WebsocketProvider } from 'yjs/provider/websocket.mjs'
-import { QuillBinding } from 'yjs/bindings/quill.mjs'
+import { WebsocketProvider } from 'yjs/provider/websocket.js'
+import { QuillBinding } from 'yjs/bindings/quill.js'
+
+import * as conf from './exampleConfig.js'
 
 import Quill from 'quill'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
+const provider = new WebsocketProvider(conf.serverAddress)
 const ydocument = provider.get('quill')
 const ytext = ydocument.define('quill', Y.Text)
 

examples/quill.js

@@ -1,10 +1,12 @@
-import * as Y from '../index.mjs'
-import { WebsocketProvider } from '../provider/websocket.mjs'
-import { QuillBinding } from '../bindings/quill.mjs'
+import * as Y from '../index.js'
+import { WebsocketProvider } from '../provider/websocket.js'
+import { QuillBinding } from '../bindings/quill.js'
+
+import * as conf from './exampleConfig.js'
 
 import Quill from 'quill'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
+const provider = new WebsocketProvider(conf.serverAddress)
 const ydocument = provider.get('quill')
 const ytext = ydocument.define('quill', Y.Text)
 

examples/textarea.html

@@ -2,18 +2,23 @@
 <html>
 <head>
     <title>Yjs Textarea Example</title>
-    <link rel=stylesheet href="https://prosemirror.net/css/editor.css">
 </head>
 <body>
+  <p>This example shows how to bind a YText type to a DOM Textarea.</p>
+  <p>The content of this textarea is shared with every client who visits this domain.</p>
   <div class="code-html">
-    <textarea style="width:80%;" rows=40 autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
+
+<textarea style="width:80%;" rows=40 autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
   </div>
+  <!-- The actual source file for the following code is found in ./textarea.js. Run `npm run watch` to compile the files -->
   <script class="code-js" src="./build/textarea.js">
 import * as Y from 'yjs'
-import { WebsocketProvider } from 'yjs/provider/websocket.mjs'
-import { TextareaBinding } from 'yjs/bindings/textarea.mjs'
+import { WebsocketProvider } from 'yjs/provider/websocket.js'
+import { TextareaBinding } from 'yjs/bindings/textarea.js'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
+import * as conf from './exampleConfig.js'
+
+const provider = new WebsocketProvider(conf.serverAddress)
 const ydocument = provider.get('textarea')
 const type = ydocument.define('textarea', Y.Text)
 const textarea = document.querySelector('textarea')

examples/textarea.js

@@ -1,8 +1,10 @@
-import * as Y from '../index.mjs'
-import { WebsocketProvider } from '../provider/websocket.mjs'
-import { TextareaBinding } from '../bindings/textarea.mjs'
+import * as Y from '../index.js'
+import { WebsocketProvider } from '../provider/websocket.js'
+import { TextareaBinding } from '../bindings/textarea.js'
 
-const provider = new WebsocketProvider('wss://api.yjs.website')
+import * as conf from './exampleConfig.js'
+
+const provider = new WebsocketProvider(conf.serverAddress)
 const ydocument = provider.get('textarea')
 const type = ydocument.define('textarea', Y.Text)
 const textarea = document.querySelector('textarea')

examples_all/ace/index.html

@@ -25,7 +25,7 @@
 
     <div id="aceContainer"></div>
     <script src="../../y.js"></script>
-    <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+    <script src='../../../y-websockets-client/y-websockets-client.js'></script>
     <script src="../bower_components/ace-builds/src/ace.js"></script>
 
     <script src="./index.js"></script>

examples_all/chat/index.html

@@ -13,7 +13,7 @@
       <input type="submit" value="Send">
     </form>
     <script src="../../y.js"></script>
-    <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+    <script src='../../../y-websockets-client/y-websockets-client.js'></script>
     <script src="./index.js"></script>
   </body>
 </html>

examples_all/codemirror/index.html

@@ -6,7 +6,7 @@
     <div id="codeMirrorContainer"></div>
 
     <script src="../../y.js"></script>
-    <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+    <script src='../../../y-websockets-client/y-websockets-client.js'></script>
     <script src="../bower_components/codemirror/lib/codemirror.js"></script>
     <script src="../bower_components/codemirror/mode/javascript/javascript.js"></script>
     <link rel="stylesheet" href="../bower_components/codemirror/lib/codemirror.css">

examples_all/drawing/index.html

@@ -13,7 +13,7 @@
   <button type="button" id="clearDrawingCanvas">Clear Drawing</button>
   <svg id="drawingCanvas" viewbox="0 0 100 100" width="100%"></svg>
   <script src="../../y.js"></script>
-  <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+  <script src='../../../y-websockets-client/y-websockets-client.js'></script>
   <script src="../bower_components/d3/d3.min.js"></script>
   <script src="./index.js"></script>
 </body>

examples_all/html-editor-drawing-hook/index.html

@@ -2,7 +2,7 @@
 <html>
 </head>
   <script src="../../y.js"></script>
-  <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+  <script src='../../../y-websockets-client/y-websockets-client.js'></script>
   <script src="../bower_components/d3/d3.min.js"></script>
   <script src="./index.js"></script>
   <style>

examples_all/html-editor/index.js

@@ -1,12 +1,12 @@
 
-import YWebsocketsConnector from '../../src/Connectors/WebsocketsConnector/WebsocketsConnector.mjs'
-import Y from '../../src/Y.mjs'
-import DomBinding from '../../bindings/DomBinding/DomBinding.mjs'
-import UndoManager from '../../src/Util/UndoManager.mjs'
-import YXmlFragment from '../../src/Types/YXml/YXmlFragment.mjs'
-import YXmlText from '../../src/Types/YXml/YXmlText.mjs'
-import YXmlElement from '../../src/Types/YXml/YXmlElement.mjs'
-import YIndexdDBPersistence from '../../src/Persistences/IndexedDBPersistence.mjs'
+import YWebsocketsConnector from '../../src/Connectors/WebsocketsConnector/WebsocketsConnector.js'
+import Y from '../../src/Y.js'
+import DomBinding from '../../bindings/DomBinding/DomBinding.js'
+import UndoManager from '../../src/Util/UndoManager.js'
+import YXmlFragment from '../../src/Types/YXml/YXmlFragment.js'
+import YXmlText from '../../src/Types/YXml/YXmlText.js'
+import YXmlElement from '../../src/Types/YXml/YXmlElement.js'
+import YIndexdDBPersistence from '../../src/Persistences/IndexedDBPersistence.js'
 
 const connector = new YWebsocketsConnector()
 const persistence = new YIndexdDBPersistence()

examples_all/indexeddb/index.html

@@ -17,7 +17,7 @@
       }
     </style>
     <script src="../../y.js"></script>
-    <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+    <script src='../../../y-websockets-client/y-websockets-client.js'></script>
     <script src="./index.js"></script>
   </body>
 </html>

examples_all/jigsaw/index.html

@@ -17,7 +17,7 @@
 		</g>
   </svg>
   <script src="../../y.js"></script>
-  <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+  <script src='../../../y-websockets-client/y-websockets-client.js'></script>
   <script src="../bower_components/d3/d3.js"></script>
   <script src="./index.js"></script>
 </body>

examples_all/monaco/index.html

@@ -14,7 +14,7 @@
       }
     </style>
     <script src="../../y.js"></script>
-    <script src='../../../y-websockets-client/y-websockets-client.mjs'></script>
+    <script src='../../../y-websockets-client/y-websockets-client.js'></script>
     <script src="../node_modules/monaco-editor/min/vs/loader.js"></script>
     <script src="./index.js"></script>
   </body>

examples_all/notes/index.js

@@ -1,9 +1,9 @@
 /* eslint-env browser */
 
-import { createYdbClient } from '../../YdbClient/index.mjs'
-import Y from '../../src/Y.dist.mjs'
-import * as ydb from '../../YdbClient/YdbClient.mjs'
-import DomBinding from '../../bindings/DomBinding/DomBinding.mjs'
+import { createYdbClient } from '../../YdbClient/index.js'
+import Y from '../../src/Y.dist.js'
+import * as ydb from '../../YdbClient/YdbClient.js'
+import DomBinding from '../../bindings/DomBinding/DomBinding.js'
 
 const uuidv4 = () => 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {
   const r = Math.random() * 16 | 0

index.js

@@ -0,0 +1,57 @@
+
+import './structs/Item.js'
+import { Delete } from './structs/Delete.js'
+import { ItemJSON } from './structs/ItemJSON.js'
+import { ItemString } from './structs/ItemString.js'
+import { ItemFormat } from './structs/ItemFormat.js'
+import { ItemEmbed } from './structs/ItemEmbed.js'
+import { ItemBinary } from './structs/ItemBinary.js'
+import { GC } from './structs/GC.js'
+
+import { YArray } from './types/YArray.js'
+import { YMap } from './types/YMap.js'
+import { YText } from './types/YText.js'
+import { YXmlText } from './types/YXmlText.js'
+import { YXmlHook } from './types/YXmlHook.js'
+import { YXmlElement, YXmlFragment } from './types/YXmlElement.js'
+
+import { registerStruct } from './utils/structReferences.js'
+
+import * as decoding from './lib/decoding.js'
+import * as encoding from './lib/encoding.js'
+import * as awarenessProtocol from './protocols/awareness.js'
+import * as syncProtocol from './protocols/sync.js'
+import * as authProtocol from './protocols/auth.js'
+
+export { decoding, encoding, awarenessProtocol, syncProtocol, authProtocol }
+
+export { Y } from './utils/Y.js'
+export { UndoManager } from './utils/UndoManager.js'
+export { Transaction } from './utils/Transaction.js'
+
+export { YArray as Array } from './types/YArray.js'
+export { YMap as Map } from './types/YMap.js'
+export { YText as Text } from './types/YText.js'
+export { YXmlText as XmlText } from './types/YXmlText.js'
+export { YXmlHook as XmlHook } from './types/YXmlHook.js'
+export { YXmlElement as XmlElement, YXmlFragment as XmlFragment } from './types/YXmlElement.js'
+
+export { getRelativePosition, fromRelativePosition } from './utils/relativePosition.js'
+export { registerStruct } from './utils/structReferences.js'
+export * from './lib/mutex.js'
+
+registerStruct(0, GC)
+registerStruct(1, ItemJSON)
+registerStruct(2, ItemString)
+registerStruct(3, ItemFormat)
+registerStruct(4, Delete)
+
+registerStruct(5, YArray)
+registerStruct(6, YMap)
+registerStruct(7, YText)
+registerStruct(8, YXmlFragment)
+registerStruct(9, YXmlElement)
+registerStruct(10, YXmlText)
+registerStruct(11, YXmlHook)
+registerStruct(12, ItemEmbed)
+registerStruct(13, ItemBinary)

index.mjs

@@ -1,50 +0,0 @@
-
-import { Delete } from './structs/Delete.mjs'
-import { ItemJSON } from './structs/ItemJSON.mjs'
-import { ItemString } from './structs/ItemString.mjs'
-import { ItemFormat } from './structs/ItemFormat.mjs'
-import { ItemEmbed } from './structs/ItemEmbed.mjs'
-import { GC } from './structs/GC.mjs'
-
-import { YArray } from './types/YArray.mjs'
-import { YMap } from './types/YMap.mjs'
-import { YText } from './types/YText.mjs'
-import { YXmlText } from './types/YXmlText.mjs'
-import { YXmlHook } from './types/YXmlHook.mjs'
-import { YXmlElement, YXmlFragment } from './types/YXmlElement.mjs'
-
-import { registerStruct } from './utils/structReferences.mjs'
-
-export { Y } from './utils/Y.mjs'
-export { UndoManager } from './utils/UndoManager.mjs'
-export { Transaction } from './utils/Transaction.mjs'
-
-export { YArray as Array } from './types/YArray.mjs'
-export { YMap as Map } from './types/YMap.mjs'
-export { YText as Text } from './types/YText.mjs'
-export { YXmlText as XmlText } from './types/YXmlText.mjs'
-export { YXmlHook as XmlHook } from './types/YXmlHook.mjs'
-export { YXmlElement as XmlElement, YXmlFragment as XmlFragment } from './types/YXmlElement.mjs'
-
-export { getRelativePosition, fromRelativePosition } from './utils/relativePosition.mjs'
-export { registerStruct } from './utils/structReferences.mjs'
-export * from './protocols/syncProtocol.mjs'
-export * from './protocols/awarenessProtocol.mjs'
-export * from './lib/encoding.mjs'
-export * from './lib/decoding.mjs'
-export * from './lib/mutex.mjs'
-
-registerStruct(0, GC)
-registerStruct(1, ItemJSON)
-registerStruct(2, ItemString)
-registerStruct(3, ItemFormat)
-registerStruct(4, Delete)
-
-registerStruct(5, YArray)
-registerStruct(6, YMap)
-registerStruct(7, YText)
-registerStruct(8, YXmlFragment)
-registerStruct(9, YXmlElement)
-registerStruct(10, YXmlText)
-registerStruct(11, YXmlHook)
-registerStruct(12, ItemEmbed)

lib/binary.js

@@ -0,0 +1,40 @@
+/* eslint-env browser */
+
+/**
+ * @module binary
+ */
+
+import * as string from './string.js'
+import * as globals from './globals.js'
+
+export const BITS32 = 0xFFFFFFFF
+export const BITS21 = (1 << 21) - 1
+export const BITS16 = (1 << 16) - 1
+
+export const BIT26 = 1 << 26
+export const BIT32 = 1 << 32
+
+/**
+ * @param {Uint8Array} bytes
+ * @return {string}
+ */
+export const toBase64 = bytes => {
+  let s = ''
+  for (let i = 0; i < bytes.byteLength; i++) {
+    s += string.fromCharCode(bytes[i])
+  }
+  return btoa(s)
+}
+
+/**
+ * @param {string} s
+ * @return {Uint8Array}
+ */
+export const fromBase64 = s => {
+  const a = atob(s)
+  const bytes = globals.createUint8ArrayFromLen(a.length)
+  for (let i = 0; i < a.length; i++) {
+    bytes[i] = a.charCodeAt(i)
+  }
+  return bytes
+}

lib/binary.mjs

@@ -1,10 +0,0 @@
-/**
- * @module binary
- */
-
-export const BITS32 = 0xFFFFFFFF
-export const BITS21 = (1 << 21) - 1
-export const BITS16 = (1 << 16) - 1
-
-export const BIT26 = 1 << 26
-export const BIT32 = 1 << 32

lib/broadcastchannel.js

@@ -0,0 +1,72 @@
+/* eslint-env browser */
+
+import * as binary from './binary.js'
+import * as globals from './globals.js'
+
+/**
+ * @typedef {Object} Channel
+ * @property {Set<Function>} Channel.subs
+ * @property {BC} Channel.bc
+ */
+
+/**
+ * @type {Map<string, Channel>}
+ */
+const channels = new Map()
+
+class LocalStoragePolyfill {
+  constructor (room) {
+    this.room = room
+    this.onmessage = null
+    addEventListener('storage', e => e.key === room && this.onmessage !== null && this.onmessage({ data: binary.fromBase64(e.newValue) }))
+  }
+  /**
+   * @param {ArrayBuffer} data
+   */
+  postMessage (buf) {
+    if (typeof localStorage !== 'undefined') {
+      localStorage.setItem(this.room, binary.toBase64(globals.createUint8ArrayFromArrayBuffer(buf)))
+    }
+  }
+}
+
+// Use BroadcastChannel or Polyfill
+const BC = typeof BroadcastChannel === 'undefined' ? LocalStoragePolyfill : BroadcastChannel
+
+/**
+ * @param {string} room
+ * @return {Channel}
+ */
+const getChannel = room => {
+  let c = channels.get(room)
+  if (c === undefined) {
+    const subs = new Set()
+    const bc = new BC(room)
+    bc.onmessage = e => subs.forEach(sub => sub(e.data))
+    c = {
+      bc, subs
+    }
+    channels.set(room, c)
+  }
+  return c
+}
+
+/**
+ * @function
+ * @param {string} room
+ * @param {Function} f
+ */
+export const subscribe = (room, f) => getChannel(room).subs.add(f)
+
+/**
+ * Publish data to all subscribers (including subscribers on this tab)
+ *
+ * @function
+ * @param {string} room
+ * @param {ArrayBuffer} data
+ */
+export const publish = (room, data) => {
+  const c = getChannel(room)
+  c.bc.postMessage(data)
+  c.subs.forEach(sub => sub(data))
+}

lib/decoding.js

@@ -4,7 +4,7 @@
 
 /* global Buffer */
 
-import * as globals from './globals.mjs'
+import * as globals from './globals.js'
 
 /**
  * A Decoder handles the decoding of an ArrayBuffer.
@@ -148,6 +148,20 @@ export const readVarUint = decoder => {
   }
 }
 
+/**
+ * Look ahead and read varUint without incrementing position
+ *
+ * @function
+ * @param {Decoder} decoder
+ * @return {number}
+ */
+export const peekVarUint = decoder => {
+  let pos = decoder.pos
+  let s = readVarUint(decoder)
+  decoder.pos = pos
+  return s
+}
+
 /**
  * Read string of variable length
  * * varUint is used to store the length of the string

lib/encoding.js

@@ -1,7 +1,7 @@
 /**
  * @module encoding
  */
-import * as globals from './globals.mjs'
+import * as globals from './globals.js'
 
 const bits7 = 0b1111111
 const bits8 = 0b11111111

lib/encoding.test.js

@@ -1,4 +1,4 @@
-import * as encoding from './encoding.mjs'
+import * as encoding from './encoding.js'
 
 /**
  * Check if binary encoding is compatible with golang binary encoding - binary.PutVarUint.

lib/globals.js

@@ -58,8 +58,6 @@ export const until = (timeout, check) => createPromise((resolve, reject) => {
 
 export const error = description => new Error(description)
 
-export const max = (a, b) => a > b ? a : b
-
 /**
  * @param {number} t Time to wait
  * @return {Promise} Promise that is resolved after t ms

lib/idb.js

@@ -1,10 +1,10 @@
 /**
- * @module idb
+ * @module lib/idb
  */
 
 /* eslint-env browser */
 
-import * as globals from './globals.mjs'
+import * as globals from './globals.js'
 
 /*
  * IDB Request to Promise transformer

lib/idb.test.js

@@ -1,6 +1,6 @@
-import * as test from './testing.mjs'
-import * as idb from './idb.mjs'
-import * as logging from './logging.mjs'
+import * as test from './testing.js'
+import * as idb from './idb.js'
+import * as logging from './logging.js'
 
 const initTestDB = db => idb.createStores(db, [['test']])
 const testDBName = 'idb-test'

lib/logging.js

@@ -2,7 +2,7 @@
  * @module logging
  */
 
-import * as globals from './globals.mjs'
+import * as globals from './globals.js'
 
 let date = new Date().getTime()
 

lib/math.js

@@ -0,0 +1,28 @@
+/**
+ * @module math
+ */
+export const floor = Math.floor
+
+/**
+ * @function
+ * @param {number} a
+ * @param {number} b
+ * @return {number} The sum of a and b
+ */
+export const add = (a, b) => a + b
+
+/**
+ * @function
+ * @param {number} a
+ * @param {number} b
+ * @return {number} The smaller element of a and b
+ */
+export const min = (a, b) => a < b ? a : b
+
+/**
+ * @function
+ * @param {number} a
+ * @param {number} b
+ * @return {number} The bigger element of a and b
+ */
+export const max = (a, b) => a > b ? a : b

lib/math.mjs

@@ -1,4 +0,0 @@
-/**
- * @module math
- */
-export const floor = Math.floor

lib/object.js

@@ -0,0 +1,14 @@
+
+export const create = Object.create(null)
+
+export const keys = Object.keys
+
+export const equalFlat = (a, b) => {
+  const keys = Object.keys(a)
+  let eq = keys.length === Object.keys(b).length
+  for (let i = 0; i < keys.length && eq; i++) {
+    const key = keys[i]
+    eq = a[key] === b[key]
+  }
+  return eq
+}

lib/prng/PRNG/PRNG.tests.js

@@ -2,10 +2,10 @@
  * @module prng
  */
 
-import { Mt19937 } from './Mt19937.mjs'
-import { Xoroshiro128plus } from './Xoroshiro128plus.mjs'
-import { Xorshift32 } from './Xorshift32.mjs'
-import * as time from '../../time.mjs'
+import { Mt19937 } from './Mt19937.js'
+import { Xoroshiro128plus } from './Xoroshiro128plus.js'
+import { Xorshift32 } from './Xorshift32.js'
+import * as time from '../../time.js'
 
 const DIAMETER = 300
 const NUMBERS = 10000

lib/prng/PRNG/Xoroshiro128plus.js

@@ -2,7 +2,7 @@
  * @module prng
  */
 
-import { Xorshift32 } from './Xorshift32.mjs'
+import { Xorshift32 } from './Xorshift32.js'
 
 /**
  * This is a variant of xoroshiro128plus - the fastest full-period generator passing BigCrush without systematic failures.

lib/prng/prng.js

@@ -2,12 +2,12 @@
  * @module prng
  */
 
-import * as binary from '../binary.mjs'
-import { fromCharCode, fromCodePoint } from '../string.mjs'
-import { MAX_SAFE_INTEGER, MIN_SAFE_INTEGER } from '../number.mjs'
-import * as math from '../math.mjs'
+import * as binary from '../binary.js'
+import { fromCharCode, fromCodePoint } from '../string.js'
+import { MAX_SAFE_INTEGER, MIN_SAFE_INTEGER } from '../number.js'
+import * as math from '../math.js'
 
-import { Xoroshiro128plus as DefaultPRNG } from './PRNG/Xoroshiro128plus.mjs'
+import { Xoroshiro128plus as DefaultPRNG } from './PRNG/Xoroshiro128plus.js'
 
 /**
  * Description of the function

lib/prng/prng.mjs

@@ -1,134 +0,0 @@
-/**
- * @module prng
- */
-
-import * as binary from '../binary.mjs'
-import { fromCharCode, fromCodePoint } from '../string.mjs'
-import { MAX_SAFE_INTEGER, MIN_SAFE_INTEGER } from '../number.mjs'
-import * as math from '../math.mjs'
-
-import { Xoroshiro128plus as DefaultPRNG } from './PRNG/Xoroshiro128plus.mjs'
-
-/**
- * Description of the function
- *  @callback generatorNext
- *  @return {number} A 32bit integer
- */
-
-/**
- * A random type generator.
- *
- * @typedef {Object} PRNG
- * @property {generatorNext} next Generate new number
- */
-
-/**
- * Create a Xoroshiro128plus Pseudo-Random-Number-Generator.
- * This is the fastest full-period generator passing BigCrush without systematic failures.
- * But there are more PRNGs available in ./PRNG/.
- *
- * @param {number} seed A positive 32bit integer. Do not use negative numbers.
- * @return {PRNG}
- */
-export const createPRNG = seed => new DefaultPRNG(Math.floor(seed < 1 ? seed * binary.BITS32 : seed))
-
-/**
- * Generates a single random bool.
- *
- * @param {PRNG} gen A random number generator.
- * @return {Boolean} A random boolean
- */
-export const bool = gen => (gen.next() & 2) === 2 // brackets are non-optional!
-
-/**
- * Generates a random integer with 53 bit resolution.
- *
- * @param {PRNG} gen A random number generator.
- * @param {Number} [min = MIN_SAFE_INTEGER] The lower bound of the allowed return values (inclusive).
- * @param {Number} [max = MAX_SAFE_INTEGER] The upper bound of the allowed return values (inclusive).
- * @return {Number} A random integer on [min, max]
- */
-export const int53 = (gen, min = MIN_SAFE_INTEGER, max = MAX_SAFE_INTEGER) => math.floor(real53(gen) * (max + 1 - min) + min)
-
-/**
- * Generates a random integer with 32 bit resolution.
- *
- * @param {PRNG} gen A random number generator.
- * @param {Number} [min = MIN_SAFE_INTEGER] The lower bound of the allowed return values (inclusive).
- * @param {Number} [max = MAX_SAFE_INTEGER] The upper bound of the allowed return values (inclusive).
- * @return {Number} A random integer on [min, max]
- */
-export const int32 = (gen, min = MIN_SAFE_INTEGER, max = MAX_SAFE_INTEGER) => min + ((gen.next() >>> 0) % (max + 1 - min))
-
-/**
- * Generates a random real on [0, 1) with 32 bit resolution.
- *
- * @param {PRNG} gen A random number generator.
- * @return {Number} A random real number on [0, 1).
- */
-export const real32 = gen => (gen.next() >>> 0) / binary.BITS32
-
-/**
- * Generates a random real on [0, 1) with 53 bit resolution.
- *
- * @param {PRNG} gen A random number generator.
- * @return {Number} A random real number on [0, 1).
- */
-export const real53 = gen => (((gen.next() >>> 5) * binary.BIT26) + (gen.next() >>> 6)) / MAX_SAFE_INTEGER
-
-/**
- * Generates a random character from char code 32 - 126. I.e. Characters, Numbers, special characters, and Space:
- *
- * (Space)!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[/]^_`abcdefghijklmnopqrstuvwxyz{|}~
- */
-export const char = gen => fromCharCode(int32(gen, 32, 126))
-
-/**
- * @param {PRNG} gen
- * @return {string} A single letter (a-z)
- */
-export const letter = gen => fromCharCode(int32(gen, 97, 122))
-
-/**
- * @param {PRNG} gen
- * @return {string} A random word without spaces consisting of letters (a-z)
- */
-export const word = gen => {
-  const len = int32(gen, 0, 20)
-  let str = ''
-  for (let i = 0; i < len; i++) {
-    str += letter(gen)
-  }
-  return str
-}
-
-/**
- * TODO: this function produces invalid runes. Does not cover all of utf16!!
- */
-export const utf16Rune = gen => {
-  const codepoint = int32(gen, 0, 256)
-  return fromCodePoint(codepoint)
-}
-
-/**
- * @param {PRNG} gen
- * @param {number} [maxlen = 20]
- */
-export const utf16String = (gen, maxlen = 20) => {
-  const len = int32(gen, 0, maxlen)
-  let str = ''
-  for (let i = 0; i < len; i++) {
-    str += utf16Rune(gen)
-  }
-  return str
-}
-
-/**
- * Returns one element of a given array.
- *
- * @param {PRNG} gen A random number generator.
- * @param {Array<T>} array Non empty Array of possible values.
- * @return {T} One of the values of the supplied Array.
- * @template T
- */
-export const oneOf = (gen, array) => array[int32(gen, 0, array.length - 1)]

lib/prng/prng.test.js

@@ -4,14 +4,14 @@
 
 /**
  *TODO: enable tests
-import * as rt from '../rich-text/formatters.mjs''
-import { test } from '../test/test.mjs''
-import Xoroshiro128plus from './PRNG/Xoroshiro128plus.mjs''
-import Xorshift32 from './PRNG/Xorshift32.mjs''
-import MT19937 from './PRNG/Mt19937.mjs''
-import { generateBool, generateInt, generateInt32, generateReal, generateChar } from './random.mjs''
-import { MAX_SAFE_INTEGER } from '../number/constants.mjs''
-import { BIT32 } from '../binary/constants.mjs''
+import * as rt from '../rich-text/formatters.js''
+import { test } from '../test/test.js''
+import Xoroshiro128plus from './PRNG/Xoroshiro128plus.js''
+import Xorshift32 from './PRNG/Xorshift32.js''
+import MT19937 from './PRNG/Mt19937.js''
+import { generateBool, generateInt, generateInt32, generateReal, generateChar } from './random.js''
+import { MAX_SAFE_INTEGER } from '../number/constants.js''
+import { BIT32 } from '../binary/constants.js''
 
 function init (Gen) {
   return {

lib/testing.js

@@ -2,8 +2,8 @@
  * @module testing
  */
 
-import * as logging from './logging.mjs'
-import { simpleDiff } from './diff.mjs'
+import * as logging from './logging.js'
+import { simpleDiff } from './diff.js'
 
 export const run = async (name, f) => {
   console.log(`%cStart:%c   ${name}`, 'color:blue;', '')

package.json

@@ -1,22 +1,24 @@
 {
   "name": "yjs",
-  "version": "13.0.0-73",
+  "version": "13.0.0-77",
   "description": "A ",
-  "module": "./index.mjs'",
+  "main": "./build/yjs.js",
+  "module": "./index.js'",
   "sideEffects": false,
   "scripts": {
     "test": "npm run lint",
-    "build": "rm -rf build examples/build && rollup -c",
+    "build": "rm -rf build examples/build && PRODUCTION=1 rollup -c",
     "watch": "rollup -wc",
-    "debug": "concurrently 'rollup -wc' 'cutest-serve build/y.test.js -o'",
+    "debug": "concurrently 'npm run watch' 'cutest-serve build/y.test.js -o'",
     "lint": "standard **/*.js",
     "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.v13.md --package ./package.json || true",
     "serve-docs": "npm run docs && serve ./docs/",
     "postversion": "npm run build",
-    "websocket-server": "node --experimental-modules ./provider/websocket/server.mjs",
+    "websocket-server": "node ./provider/websocket/server.js",
     "now-start": "npm run websocket-server"
   },
   "files": [
+    "build/*",
     "bindings/*",
     "docs/*",
     "examples/*",
@@ -40,7 +42,7 @@
     "lib": "./"
   },
   "bin": {
-    "y-websockets": "provider/websocket/server.js"
+    "y-websocket-server": "provider/websocket/server.js"
   },
   "standard": {
     "ignore": [
@@ -65,6 +67,7 @@
   },
   "homepage": "http://y-js.org",
   "devDependencies": {
+    "@types/ws": "^6.0.1",
     "babel-cli": "^6.26.0",
     "babel-plugin-external-helpers": "^6.22.0",
     "babel-plugin-transform-regenerator": "^6.26.0",
@@ -93,9 +96,12 @@
     "rollup-regenerator-runtime": "^6.23.1",
     "rollup-watch": "^3.2.2",
     "standard": "^11.0.1",
-    "tui-jsdoc-template": "^1.2.2"
+    "tui-jsdoc-template": "^1.2.2",
+    "codemirror": "^5.42.0",
+    "crel": "^3.1.0"
   },
-  "dependencies": {
+  "optionalDependencies": {
+    "level": "^4.0.0",
     "ws": "^6.1.0"
   }
 }

persistence/FilePersistence.js

@@ -1,10 +1,10 @@
 /*
 import fs from 'fs'
 import path from 'path'
-import * as encoding from '../lib/encoding.mjs'
-import * as decoding from '../lib/decoding.mjs'
-import { createMutex } from '../lib/mutex.mjs'
-import { encodeUpdate, encodeStructsDS, decodePersisted } from './decodePersisted.mjs'
+import * as encoding from '../lib/encoding.js'
+import * as decoding from '../lib/decoding.js'
+import { createMutex } from '../lib/mutex.js'
+import { encodeUpdate, encodeStructsDS, decodePersisted } from './decodePersisted.js'
 
 function createFilePath (persistence, roomName) {
   // TODO: filename checking!

persistence/IndexedDBPersistence.js

@@ -0,0 +1,553 @@
+/*
+import { Y } from '../utils/Y.js'
+import { createMutex } from '../lib/mutex.js'
+import { decodePersisted, encodeStructsDS, encodeUpdate, PERSIST_STRUCTS_DS, PERSIST_UPDATE } from './decodePersisted.js'
+
+function rtop (request) {
+  return new Promise(function (resolve, reject) {
+    request.onerror = function (event) {
+      reject(new Error(event.target.error))
+    }
+    request.onblocked = function () {
+      location.reload()
+    }
+    request.onsuccess = function (event) {
+      resolve(event.target.result)
+    }
+  })
+}
+
+function openDB (room) {
+  return new Promise(function (resolve, reject) {
+    let request = indexedDB.open(room)
+    request.onupgradeneeded = function (event) {
+      const db = event.target.result
+      if (db.objectStoreNames.contains('updates')) {
+        db.deleteObjectStore('updates')
+      }
+      db.createObjectStore('updates', {autoIncrement: true})
+    }
+    request.onerror = function (event) {
+      reject(new Error(event.target.error))
+    }
+    request.onblocked = function () {
+      location.reload()
+    }
+    request.onsuccess = function (event) {
+      const db = event.target.result
+      db.onversionchange = function () { db.close() }
+      resolve(db)
+    }
+  })
+}
+
+function persist (room) {
+  let t = room.db.transaction(['updates'], 'readwrite')
+  let updatesStore = t.objectStore('updates')
+  return rtop(updatesStore.getAll())
+    .then(updates => {
+      // apply all previous updates before deleting them
+      room.mutex(() => {
+        updates.forEach(update => {
+          decodePersisted(y, new BinaryDecoder(update))
+        })
+      })
+      const encoder = new BinaryEncoder()
+      encodeStructsDS(y, encoder)
+      // delete all pending updates
+      rtop(updatesStore.clear()).then(() => {
+        // write current model
+        updatesStore.put(encoder.createBuffer())
+      })
+    })
+}
+
+function saveUpdate (room, updateBuffer) {
+  const db = room.db
+  if (db !== null) {
+    const t = db.transaction(['updates'], 'readwrite')
+    const updatesStore = t.objectStore('updates')
+    const updatePut = rtop(updatesStore.put(updateBuffer))
+    rtop(updatesStore.count()).then(cnt => {
+      if (cnt >= PREFERRED_TRIM_SIZE) {
+        persist(room)
+      }
+    })
+    return updatePut
+  }
+}
+
+function registerRoomInPersistence (documentsDB, roomName) {
+  return documentsDB.then(
+    db => Promise.all([
+      db,
+      rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
+    ])
+  ).then(
+    ([db, doc]) => {
+      if (doc === undefined) {
+        return rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: 0 }))
+      }
+    }
+  )
+}
+
+const PREFERRED_TRIM_SIZE = 400
+
+export class IndexedDBPersistence {
+  constructor () {
+    this._rooms = new Map()
+    this._documentsDB = new Promise(function (resolve, reject) {
+      let request = indexedDB.open('_yjs_documents')
+      request.onupgradeneeded = function (event) {
+        const db = event.target.result
+        if (db.objectStoreNames.contains('documents')) {
+          db.deleteObjectStore('documents')
+        }
+        db.createObjectStore('documents', { keyPath: "roomName" })
+      }
+      request.onerror = function (event) {
+        reject(new Error(event.target.error))
+      }
+      request.onblocked = function () {
+        location.reload()
+      }
+      request.onsuccess = function (event) {
+        const db = event.target.result
+        db.onversionchange = function () { db.close() }
+        resolve(db)
+      }
+    })
+    addEventListener('unload', () => {
+      // close everything when page unloads
+      this._rooms.forEach(room => {
+        if (room.db !== null) {
+          room.db.close()
+        } else {
+          room.dbPromise.then(db => db.close())
+        }
+      })
+      this._documentsDB.then(db => db.close())
+    })
+  }
+  getAllDocuments () {
+    return this._documentsDB.then(
+      db => rtop(db.transaction(['documents'], 'readonly').objectStore('documents').getAll())
+    )
+  }
+  setRemoteUpdateCounter (roomName, remoteUpdateCounter) {
+    this._documentsDB.then(
+      db => rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').put({ roomName, remoteUpdateCounter }))
+    )
+  }
+
+  _createYInstance (roomName) {
+    const room = this._rooms.get(roomName)
+    if (room !== undefined) {
+      return room.y
+    }
+    const y = new Y()
+    return openDB(roomName).then(
+      db => rtop(db.transaction(['updates'], 'readonly').objectStore('updates').getAll())
+    ).then(
+      updates =>
+        y.transact(() => {
+          updates.forEach(update => {
+            decodePersisted(y, new BinaryDecoder(update))
+          })
+        }, true)
+    ).then(() => Promise.resolve(y))
+  }
+
+  _persistStructsDS (roomName, structsDS) {
+    const encoder = new BinaryEncoder()
+    encoder.writeVarUint(PERSIST_STRUCTS_DS)
+    encoder.writeArrayBuffer(structsDS)
+    return openDB(roomName).then(db => {
+      const t = db.transaction(['updates'], 'readwrite')
+      const updatesStore = t.objectStore('updates')
+      return rtop(updatesStore.put(encoder.createBuffer()))
+    })
+  }
+
+  _persistStructs (roomName, structs) {
+    const encoder = new BinaryEncoder()
+    encoder.writeVarUint(PERSIST_UPDATE)
+    encoder.writeArrayBuffer(structs)
+    return openDB(roomName).then(db => {
+      const t = db.transaction(['updates'], 'readwrite')
+      const updatesStore = t.objectStore('updates')
+      return rtop(updatesStore.put(encoder.createBuffer()))
+    })
+  }
+
+  connectY (roomName, y) {
+    if (this._rooms.has(roomName)) {
+      throw new Error('A Y instance is already bound to this room!')
+    }
+    let room = {
+      db: null,
+      dbPromise: null,
+      channel: null,
+      mutex: createMutex(),
+      y
+    }
+    if (typeof BroadcastChannel !== 'undefined') {
+      room.channel = new BroadcastChannel('__yjs__' + roomName)
+      room.channel.addEventListener('message', e => {
+        room.mutex(function () {
+          decodePersisted(y, new BinaryDecoder(e.data))
+        })
+      })
+    }
+    y.on('destroyed', () => {
+      this.disconnectY(roomName, y)
+    })
+    y.on('afterTransaction', (y, transaction) => {
+      room.mutex(() => {
+        if (transaction.encodedStructsLen > 0) {
+          const encoder = new BinaryEncoder()
+          const update = new BinaryEncoder()
+          encodeUpdate(y, transaction.encodedStructs, update)
+          const updateBuffer = update.createBuffer()
+          if (room.channel !== null) {
+            room.channel.postMessage(updateBuffer)
+          }
+          if (transaction.encodedStructsLen > 0
+            import { Y } from '../utils/Y.js'
+            import { createMutex } from '../lib/mutex.js'
+            import { decodePersisted, encodeStructsDS, encodeUpdate, PERSIST_STRUCTS_DS, PERSIST_UPDATE } from './decodePersisted.js'
+
+            function rtop (request) {
+              return new Promise(function (resolve, reject) {
+                request.onerror = function (event) {
+                  reject(new Error(event.target.error))
+                }
+                request.onblocked = function () {
+                  location.reload()
+                }
+                request.onsuccess = function (event) {
+                  resolve(event.target.result)
+                }
+              })
+            }
+
+            function openDB (room) {
+              return new Promise(function (resolve, reject) {
+                let request = indexedDB.open(room)
+                request.onupgradeneeded = function (event) {
+                  const db = event.target.result
+                  if (db.objectStoreNames.contains('updates')) {
+                    db.deleteObjectStore('updates')
+                  }
+                  db.createObjectStore('updates', {autoIncrement: true})
+                }
+                request.onerror = function (event) {
+                  reject(new Error(event.target.error))
+                }
+                request.onblocked = function () {
+                  location.reload()
+                }
+                request.onsuccess = function (event) {
+                  const db = event.target.result
+                  db.onversionchange = function () { db.close() }
+                  resolve(db)
+                }
+              })
+            }
+
+            function persist (room) {
+              let t = room.db.transaction(['updates'], 'readwrite')
+              let updatesStore = t.objectStore('updates')
+              return rtop(updatesStore.getAll())
+                .then(updates => {
+                  // apply all previous updates before deleting them
+                  room.mutex(() => {
+                    updates.forEach(update => {
+                      decodePersisted(y, new BinaryDecoder(update))
+                    })
+                  })
+                  const encoder = new BinaryEncoder()
+                  encodeStructsDS(y, encoder)
+                  // delete all pending updates
+                  rtop(updatesStore.clear()).then(() => {
+                    // write current model
+                    updatesStore.put(encoder.createBuffer())
+                  })
+                })
+            }
+
+            function saveUpdate (room, updateBuffer) {
+              const db = room.db
+              if (db !== null) {
+                const t = db.transaction(['updates'], 'readwrite')
+                const updatesStore = t.objectStore('updates')
+                const updatePut = rtop(updatesStore.put(updateBuffer))
+                rtop(updatesStore.count()).then(cnt => {
+                  if (cnt >= PREFERRED_TRIM_SIZE) {
+                    persist(room)
+                  }
+                })
+                return updatePut
+              }
+            }
+
+            function registerRoomInPersistence (documentsDB, roomName) {
+              return documentsDB.then(
+                db => Promise.all([
+                  db,
+                  rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
+                ])
+              ).then(
+                ([db, doc]) => {
+                  if (doc === undefined) {
+                    return rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: 0 }))
+                  }
+                }
+              )
+            }
+
+            const PREFERRED_TRIM_SIZE = 400
+
+            export class IndexedDBPersistence {
+              constructor () {
+                this._rooms = new Map()
+                this._documentsDB = new Promise(function (resolve, reject) {
+                  let request = indexedDB.open('_yjs_documents')
+                  request.onupgradeneeded = function (event) {
+                    const db = event.target.result
+                    if (db.objectStoreNames.contains('documents')) {
+                      db.deleteObjectStore('documents')
+                    }
+                    db.createObjectStore('documents', { keyPath: "roomName" })
+                  }
+                  request.onerror = function (event) {
+                    reject(new Error(event.target.error))
+                  }
+                  request.onblocked = function () {
+                    location.reload()
+                  }
+                  request.onsuccess = function (event) {
+                    const db = event.target.result
+                    db.onversionchange = function () { db.close() }
+                    resolve(db)
+                  }
+                })
+                addEventListener('unload', () => {
+                  // close everything when page unloads
+                  this._rooms.forEach(room => {
+                    if (room.db !== null) {
+                      room.db.close()
+                    } else {
+                      room.dbPromise.then(db => db.close())
+                    }
+                  })
+                  this._documentsDB.then(db => db.close())
+                })
+              }
+              getAllDocuments () {
+                return this._documentsDB.then(
+                  db => rtop(db.transaction(['documents'], 'readonly').objectStore('documents').getAll())
+                )
+              }
+              setRemoteUpdateCounter (roomName, remoteUpdateCounter) {
+                this._documentsDB.then(
+                  db => rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').put({ roomName, remoteUpdateCounter }))
+                )
+              }
+
+              _createYInstance (roomName) {
+                const room = this._rooms.get(roomName)
+                if (room !== undefined) {
+                  return room.y
+                }
+                const y = new Y()
+                return openDB(roomName).then(
+                  db => rtop(db.transaction(['updates'], 'readonly').objectStore('updates').getAll())
+                ).then(
+                  updates =>
+                    y.transact(() => {
+                      updates.forEach(update => {
+                        decodePersisted(y, new BinaryDecoder(update))
+                      })
+                    }, true)
+                ).then(() => Promise.resolve(y))
+              }
+
+              _persistStructsDS (roomName, structsDS) {
+                const encoder = new BinaryEncoder()
+                encoder.writeVarUint(PERSIST_STRUCTS_DS)
+                encoder.writeArrayBuffer(structsDS)
+                return openDB(roomName).then(db => {
+                  const t = db.transaction(['updates'], 'readwrite')
+                  const updatesStore = t.objectStore('updates')
+                  return rtop(updatesStore.put(encoder.createBuffer()))
+                })
+              }
+
+              _persistStructs (roomName, structs) {
+                const encoder = new BinaryEncoder()
+                encoder.writeVarUint(PERSIST_UPDATE)
+                encoder.writeArrayBuffer(structs)
+                return openDB(roomName).then(db => {
+                  const t = db.transaction(['updates'], 'readwrite')
+                  const updatesStore = t.objectStore('updates')
+                  return rtop(updatesStore.put(encoder.createBuffer()))
+                })
+              }
+
+              connectY (roomName, y) {
+                if (this._rooms.has(roomName)) {
+                  throw new Error('A Y instance is already bound to this room!')
+                }
+                let room = {
+                  db: null,
+                  dbPromise: null,
+                  channel: null,
+                  mutex: createMutex(),
+                  y
+                }
+                if (typeof BroadcastChannel !== 'undefined') {
+                  room.channel = new BroadcastChannel('__yjs__' + roomName)
+                  room.channel.addEventListener('message', e => {
+                    room.mutex(function () {
+                      decodePersisted(y, new BinaryDecoder(e.data))
+                    })
+                  })
+                }
+                y.on('destroyed', () => {
+                  this.disconnectY(roomName, y)
+                })
+                y.on('afterTransaction', (y, transaction) => {
+                  room.mutex(() => {
+                    if (transaction.encodedStructsLen > 0) {
+                      const encoder = new BinaryEncoder()
+                      const update = new BinaryEncoder()
+                      encodeUpdate(y, transaction.encodedStructs, update)
+                      const updateBuffer = update.createBuffer()
+                      if (room.channel !== null) {
+                        room.channel.postMessage(updateBuffer)
+                      }
+                      if (transaction.encodedStructsLen > 0) {
+                        if (room.db !== null) {
+                          saveUpdate(room, updateBuffer)
+                        }
+                      }
+                    }
+                  })
+                })
+                // register document in documentsDB
+                this._documentsDB.then(
+                  db =>
+                    rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
+                      .then(
+                        doc => doc === undefined && rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: -1 }))
+                      )
+                )
+                // open room db and read existing data
+                return room.dbPromise = openDB(roomName)
+                  .then(db => {
+                    room.db = db
+                    const t = room.db.transaction(['updates'], 'readwrite')
+                    const updatesStore = t.objectStore('updates')
+                    // write current state as update
+                    const encoder = new BinaryEncoder()
+                    encodeStructsDS(y, encoder)
+                    return rtop(updatesStore.put(encoder.createBuffer())).then(() => {
+                      // read persisted state
+                      return rtop(updatesStore.getAll()).then(updates => {
+                        room.mutex(() => {
+                          y.transact(() => {
+                            updates.forEach(update => {
+                              decodePersisted(y, new BinaryDecoder(update))
+                            })
+                          }, true)
+                        })
+                      })
+                    })
+                  })
+              }
+              disconnectY (roomName) {
+                const {
+                  db, channel
+                } = this._rooms.get(roomName)
+                db.close()
+                if (channel !== null) {
+                  channel.close()
+                }
+                this._rooms.delete(roomName)
+              }
+
+              /**
+               * Remove all persisted data that belongs to a room.
+               * Automatically destroys all Yjs all Yjs instances that persist to
+               * the room. If `destroyYjsInstances = false` the persistence functionality
+               * will be removed from the Yjs instances.
+               *
+              removePersistedData (roomName, destroyYjsInstances = true) {
+                this.disconnectY(roomName)
+                return rtop(indexedDB.deleteDatabase(roomName))
+              }
+            }
+             {
+            if (room.db !== null) {
+              saveUpdate(room, updateBuffer)
+            }
+          }
+        }
+      })
+    })
+    // register document in documentsDB
+    this._documentsDB.then(
+      db =>
+        rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
+          .then(
+            doc => doc === undefined && rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: -1 }))
+          )
+    )
+    // open room db and read existing data
+    return room.dbPromise = openDB(roomName)
+      .then(db => {
+        room.db = db
+        const t = room.db.transaction(['updates'], 'readwrite')
+        const updatesStore = t.objectStore('updates')
+        // write current state as update
+        const encoder = new BinaryEncoder()
+        encodeStructsDS(y, encoder)
+        return rtop(updatesStore.put(encoder.createBuffer())).then(() => {
+          // read persisted state
+          return rtop(updatesStore.getAll()).then(updates => {
+            room.mutex(() => {
+              y.transact(() => {
+                updates.forEach(update => {
+                  decodePersisted(y, new BinaryDecoder(update))
+                })
+              }, true)
+            })
+          })
+        })
+      })
+  }
+  disconnectY (roomName) {
+    const {
+      db, channel
+    } = this._rooms.get(roomName)
+    db.close()
+    if (channel !== null) {
+      channel.close()
+    }
+    this._rooms.delete(roomName)
+  }
+
+  /**
+   * Remove all persisted data that belongs to a room.
+   * Automatically destroys all Yjs all Yjs instances that persist to
+   * the room. If `destroyYjsInstances = false` the persistence functionality
+   * will be removed from the Yjs instances.
+   *
+  removePersistedData (roomName, destroyYjsInstances = true) {
+    this.disconnectY(roomName)
+    return rtop(indexedDB.deleteDatabase(roomName))
+  }
+}
+*/

persistence/decodePersisted.js

@@ -1,7 +1,7 @@
 /*
-import { integrateRemoteStructs } from '../MessageHandler/integrateRemoteStructs.mjs'
-import { writeStructs } from '../MessageHandler/syncStep1.mjs'
-import { writeDeleteSet, readDeleteSet } from '../MessageHandler/deleteSet.mjs'
+import { integrateRemoteStructs } from '../MessageHandler/integrateRemoteStructs.js'
+import { writeStructs } from '../MessageHandler/syncStep1.js'
+import { writeDeleteSet, readDeleteSet } from '../MessageHandler/deleteSet.js'
 
 export const PERSIST_UPDATE = 0
 /**

persistence/leveldb.js

@@ -0,0 +1,125 @@
+/**
+ * @module persistence/leveldb
+ * This module re-uses the encoding of syncProtocol to store and read updates from leveldb.
+ */
+
+const level = require('level')
+const Y = require('../build/yjs.js')
+const mux = Y.createMutex()
+
+/*
+ * Improves the uniqueness of timestamps.
+ * We gamble with the fact that users won't create more than 10000 changes on a single document
+ * within one millisecond (also assuming clock works correctly).
+ */
+let timestampIterator = 0
+/**
+ * @return {string} A random, time-based string starting with "${roomName}:"
+ */
+const getNextTimestamp = () => {
+  timestampIterator = (timestampIterator + 1) % 10000
+  return `${Date.now()}${timestampIterator.toString().padStart(4, '0')}`
+}
+
+/**
+ * @param {string} docName
+ * @return {string}
+ */
+const generateEntryKey = docName => `${docName}#${getNextTimestamp()}`
+
+/**
+ *
+ * @param {any} db
+ * @param {string} docName
+ * @param {Uint8Array | ArrayBuffer} buf
+ */
+const writeEntry = (db, docName, buf) => db.put(generateEntryKey(docName), buf)
+
+/**
+ * @param {Uint8Array} arr
+ * @param {Y.Y} ydocument
+ */
+const readEntry = (arr, ydocument) => mux(() =>
+  Y.syncProtocol.readSyncMessage(Y.decoding.createDecoder(arr), Y.encoding.createEncoder(), ydocument)
+)
+
+/**
+ * @param {any} db
+ * @param {string} docName
+ * @param {Y.Y} ydocument
+ */
+const loadFromPersistence = (db, docName, ydocument) => new Promise((resolve, reject) =>
+  db.createReadStream({
+    gte: `${docName}#`,
+    lte: `${docName}#Z`,
+    keys: false,
+    values: true
+  })
+    .on('data', data => readEntry(data, ydocument))
+    .on('error', reject)
+    .on('end', resolve)
+    .on('close', resolve)
+)
+
+const persistState = (db, docName, ydocument) => {
+  const encoder = Y.encoding.createEncoder()
+  Y.syncProtocol.writeSyncStep2(encoder, ydocument, new Map())
+  const entryKey = generateEntryKey(docName)
+  const entryPromise = db.put(entryKey, Y.encoding.toBuffer(encoder))
+  const delOps = []
+  return new Promise((resolve, reject) => db.createKeyStream({
+    gte: `${docName}#`,
+    lt: entryKey
+  })
+    .on('data', key => delOps.push({ type: 'del', key }))
+    .on('error', reject)
+    .on('end', resolve)
+    .on('close', resolve)
+  ).then(() => entryPromise).then(() => db.batch(delOps))
+}
+
+/**
+ * Persistence layer for Leveldb.
+ */
+exports.LevelDbPersistence = class LevelDbPersistence {
+  /**
+   * @param {string} fpath Path to leveldb database
+   */
+  constructor (fpath) {
+    this.db = level(fpath, { valueEncoding: 'binary' })
+  }
+  /**
+   * Retrieve all data from LevelDB and automatically persist all document updates to leveldb.
+   *
+   * @param {string} docName
+   * @param {Y.Y} ydocument
+   */
+  bindState (docName, ydocument) {
+    // write all updates received from other clients
+    // - unless it is created by this persistence layer (e.g. loadFromPersistence, we we mux).
+    ydocument.on('afterTransaction', (y, transaction) => {
+      if (transaction.encodedStructsLen > 0) {
+        mux(() => {
+          const encoder = Y.encoding.createEncoder()
+          Y.syncProtocol.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
+          writeEntry(this.db, docName, Y.encoding.toBuffer(encoder))
+        })
+      }
+    })
+    // read all data from persistence
+    return loadFromPersistence(this.db, docName, ydocument).then(() =>
+      // write current state (just in case anything was added before state was bound)
+      this.writeState(docName, ydocument)
+    )
+  }
+  /**
+   * Write current state to persistence layer. Deletes all entries that were made before.
+   * Call this method at any time - the recommended time to call this method is before the ydocument is destroyed.
+   *
+   * @param {string} docName
+   * @param {Y.Y} ydocument
+   */
+  writeState (docName, ydocument) {
+    return persistState(this.db, docName, ydocument)
+  }
+}

persistences/IndexedDBPersistence.mjs

@@ -1,288 +0,0 @@
-
-/*
-import { Y } from '../utils/Y.mjs'
-import { createMutex } from '../lib/mutex.mjs'
-import { decodePersisted, encodeStructsDS, encodeUpdate, PERSIST_STRUCTS_DS, PERSIST_UPDATE } from './decodePersisted.mjs'
-
-function rtop (request) {
-  return new Promise(function (resolve, reject) {
-    request.onerror = function (event) {
-      reject(new Error(event.target.error))
-    }
-    request.onblocked = function () {
-      location.reload()
-    }
-    request.onsuccess = function (event) {
-      resolve(event.target.result)
-    }
-  })
-}
-
-function openDB (room) {
-  return new Promise(function (resolve, reject) {
-    let request = indexedDB.open(room)
-    request.onupgradeneeded = function (event) {
-      const db = event.target.result
-      if (db.objectStoreNames.contains('updates')) {
-        db.deleteObjectStore('updates')
-      }
-      db.createObjectStore('updates', {autoIncrement: true})
-    }
-    request.onerror = function (event) {
-      reject(new Error(event.target.error))
-    }
-    request.onblocked = function () {
-      location.reload()
-    }
-    request.onsuccess = function (event) {
-      const db = event.target.result
-      db.onversionchange = function () { db.close() }
-      resolve(db)
-    }
-  })
-}
-
-function persist (room) {
-  let t = room.db.transaction(['updates'], 'readwrite')
-  let updatesStore = t.objectStore('updates')
-  return rtop(updatesStore.getAll())
-    .then(updates => {
-      // apply all previous updates before deleting them
-      room.mutex(() => {
-        updates.forEach(update => {
-          decodePersisted(y, new BinaryDecoder(update))
-        })
-      })
-      const encoder = new BinaryEncoder()
-      encodeStructsDS(y, encoder)
-      // delete all pending updates
-      rtop(updatesStore.clear()).then(() => {
-        // write current model
-        updatesStore.put(encoder.createBuffer())
-      })
-    })
-}
-
-function saveUpdate (room, updateBuffer) {
-  const db = room.db
-  if (db !== null) {
-    const t = db.transaction(['updates'], 'readwrite')
-    const updatesStore = t.objectStore('updates')
-    const updatePut = rtop(updatesStore.put(updateBuffer))
-    rtop(updatesStore.count()).then(cnt => {
-      if (cnt >= PREFERRED_TRIM_SIZE) {
-        persist(room)
-      }
-    })
-    return updatePut
-  }
-}
-
-function registerRoomInPersistence (documentsDB, roomName) {
-  return documentsDB.then(
-    db => Promise.all([
-      db,
-      rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
-    ])
-  ).then(
-    ([db, doc]) => {
-      if (doc === undefined) {
-        return rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: 0 }))
-      }
-    }
-  )
-}
-
-const PREFERRED_TRIM_SIZE = 400
-
-export class IndexedDBPersistence {
-  constructor () {
-    this._rooms = new Map()
-    this._documentsDB = new Promise(function (resolve, reject) {
-      let request = indexedDB.open('_yjs_documents')
-      request.onupgradeneeded = function (event) {
-        const db = event.target.result
-        if (db.objectStoreNames.contains('documents')) {
-          db.deleteObjectStore('documents')
-        }
-        db.createObjectStore('documents', { keyPath: "roomName" })
-      }
-      request.onerror = function (event) {
-        reject(new Error(event.target.error))
-      }
-      request.onblocked = function () {
-        location.reload()EventListener' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:160:36: 'BinaryDecoder' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:167:25: 'BinaryEncoder' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:178:25: 'BinaryEncoder' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:203:34: 'BinaryDecoder' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:213:17: 'encoder' is assigned a value but never used.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:213:31: 'BinaryEncoder' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:214:30: 'BinaryEncoder' is not defined.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:230:12: Trailing spaces not allowed.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:237:5: Return statement should not contain assignment.
-        /home/dmonad/go/src/github.com/y-js/yjs/persistences/IndexedDBPersistence.js:243:29: 'BinaryEncoder' i
-      }
-      request.onsuccess = function (event) {
-        const db = event.target.result
-        db.onversionchange = function () { db.close() }
-        resolve(db)
-      }
-    })
-    addEventListener('unload', () => {
-      // close everything when page unloads
-      this._rooms.forEach(room => {
-        if (room.db !== null) {
-          room.db.close()
-        } else {
-          room.dbPromise.then(db => db.close())
-        }
-      })
-      this._documentsDB.then(db => db.close())
-    })
-  }
-  getAllDocuments () {
-    return this._documentsDB.then(
-      db => rtop(db.transaction(['documents'], 'readonly').objectStore('documents').getAll())
-    )
-  }
-  setRemoteUpdateCounter (roomName, remoteUpdateCounter) {
-    this._documentsDB.then(
-      db => rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').put({ roomName, remoteUpdateCounter }))
-    )
-  }
-
-  _createYInstance (roomName) {
-    const room = this._rooms.get(roomName)
-    if (room !== undefined) {
-      return room.y
-    }
-    const y = new Y()
-    return openDB(roomName).then(
-      db => rtop(db.transaction(['updates'], 'readonly').objectStore('updates').getAll())
-    ).then(
-      updates =>
-        y.transact(() => {
-          updates.forEach(update => {
-            decodePersisted(y, new BinaryDecoder(update))
-          })
-        }, true)
-    ).then(() => Promise.resolve(y))
-  }
-
-  _persistStructsDS (roomName, structsDS) {
-    const encoder = new BinaryEncoder()
-    encoder.writeVarUint(PERSIST_STRUCTS_DS)
-    encoder.writeArrayBuffer(structsDS)
-    return openDB(roomName).then(db => {
-      const t = db.transaction(['updates'], 'readwrite')
-      const updatesStore = t.objectStore('updates')
-      return rtop(updatesStore.put(encoder.createBuffer()))
-    })
-  }
-
-  _persistStructs (roomName, structs) {
-    const encoder = new BinaryEncoder()
-    encoder.writeVarUint(PERSIST_UPDATE)
-    encoder.writeArrayBuffer(structs)
-    return openDB(roomName).then(db => {
-      const t = db.transaction(['updates'], 'readwrite')
-      const updatesStore = t.objectStore('updates')
-      return rtop(updatesStore.put(encoder.createBuffer()))
-    })
-  }
-
-  connectY (roomName, y) {
-    if (this._rooms.has(roomName)) {
-      throw new Error('A Y instance is already bound to this room!')
-    }
-    let room = {
-      db: null,
-      dbPromise: null,
-      channel: null,
-      mutex: createMutex(),
-      y
-    }
-    if (typeof BroadcastChannel !== 'undefined') {
-      room.channel = new BroadcastChannel('__yjs__' + roomName)
-      room.channel.addEventListener('message', e => {
-        room.mutex(function () {
-          decodePersisted(y, new BinaryDecoder(e.data))
-        })
-      })
-    }
-    y.on('destroyed', () => {
-      this.disconnectY(roomName, y)
-    })
-    y.on('afterTransaction', (y, transaction) => {
-      room.mutex(() => {
-        if (transaction.encodedStructsLen > 0) {
-          const encoder = new BinaryEncoder()
-          const update = new BinaryEncoder()
-          encodeUpdate(y, transaction.encodedStructs, update)
-          const updateBuffer = update.createBuffer()
-          if (room.channel !== null) {
-            room.channel.postMessage(updateBuffer)
-          }
-          if (transaction.encodedStructsLen > 0) {
-            if (room.db !== null) {
-              saveUpdate(room, updateBuffer)
-            }
-          }
-        }
-      })
-    })
-    // register document in documentsDB
-    this._documentsDB.then(
-      db =>
-        rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
-          .then(
-            doc => doc === undefined && rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: -1 }))
-          )
-    )
-    // open room db and read existing data
-    return room.dbPromise = openDB(roomName)
-      .then(db => {
-        room.db = db
-        const t = room.db.transaction(['updates'], 'readwrite')
-        const updatesStore = t.objectStore('updates')
-        // write current state as update
-        const encoder = new BinaryEncoder()
-        encodeStructsDS(y, encoder)
-        return rtop(updatesStore.put(encoder.createBuffer())).then(() => {
-          // read persisted state
-          return rtop(updatesStore.getAll()).then(updates => {
-            room.mutex(() => {
-              y.transact(() => {
-                updates.forEach(update => {
-                  decodePersisted(y, new BinaryDecoder(update))
-                })
-              }, true)
-            })
-          })
-        })
-      })
-  }
-  disconnectY (roomName) {
-    const {
-      db, channel
-    } = this._rooms.get(roomName)
-    db.close()
-    if (channel !== null) {
-      channel.close()
-    }
-    this._rooms.delete(roomName)
-  }
-
-  /**
-   * Remove all persisted data that belongs to a room.
-   * Automatically destroys all Yjs all Yjs instances that persist to
-   * the room. If `destroyYjsInstances = false` the persistence functionality
-   * will be removed from the Yjs instances.
-   *
-  removePersistedData (roomName, destroyYjsInstances = true) {
-    this.disconnectY(roomName)
-    return rtop(indexedDB.deleteDatabase(roomName))
-  }
-}
-*/

protocols/auth.js

@@ -0,0 +1,33 @@
+
+import * as encoding from '../lib/encoding.js'
+import * as decoding from '../lib/decoding.js'
+import { Y } from '../utils/Y.js' // eslint-disable-line
+
+export const messagePermissionDenied = 0
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {string} reason
+ */
+export const writePermissionDenied = (encoder, reason) => {
+  encoding.writeVarUint(encoder, messagePermissionDenied)
+  encoding.writeVarString(encoder, reason)
+}
+
+/**
+ * @callback PermissionDeniedHandler
+ * @param {any} y
+ * @param {string} reason
+ */
+
+/**
+ *
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ * @param {PermissionDeniedHandler} permissionDeniedHandler
+ */
+export const readAuthMessage = (decoder, y, permissionDeniedHandler) => {
+  switch (decoding.readVarUint(decoder)) {
+    case messagePermissionDenied: permissionDeniedHandler(y, decoding.readVarString(decoder))
+  }
+}

protocols/awareness.js

@@ -2,16 +2,17 @@
  * @module awareness-protocol
  */
 
-import * as encoding from '../lib/encoding.mjs'
-import * as decoding from '../lib/decoding.mjs'
-import { Y } from '../utils/Y.mjs' // eslint-disable-line
+import * as encoding from '../lib/encoding.js'
+import * as decoding from '../lib/decoding.js'
+import { Y } from '../utils/Y.js' // eslint-disable-line
 
 const messageUsersStateChanged = 0
 
 /**
  * @typedef {Object} UserStateUpdate
  * @property {number} UserStateUpdate.userID
- * @property {Object} state
+ * @property {number} UserStateUpdate.clock
+ * @property {Object} UserStateUpdate.state
  */
 
 /**
@@ -23,8 +24,9 @@ export const writeUsersStateChange = (encoder, stateUpdates) => {
   encoding.writeVarUint(encoder, messageUsersStateChanged)
   encoding.writeVarUint(encoder, len)
   for (let i = 0; i < len; i++) {
-    const {userID, state} = stateUpdates[i]
+    const {userID, state, clock} = stateUpdates[i]
     encoding.writeVarUint(encoder, userID)
+    encoding.writeVarUint(encoder, clock)
     encoding.writeVarString(encoder, JSON.stringify(state))
   }
 }
@@ -36,21 +38,24 @@ export const readUsersStateChange = (decoder, y) => {
   const len = decoding.readVarUint(decoder)
   for (let i = 0; i < len; i++) {
     const userID = decoding.readVarUint(decoder)
+    const clock = decoding.readVarUint(decoder)
     const state = JSON.parse(decoding.readVarString(decoder))
-    if (userID !== y.userID) {
-      if (state === null) {
-        if (y.awareness.has(userID)) {
-          y.awareness.delete(userID)
-          removed.push(userID)
-        }
-      } else {
-        if (y.awareness.has(userID)) {
-          updated.push(userID)
-        } else {
-          added.push(userID)
-        }
-        y.awareness.set(userID, state)
+    const uClock = y.awarenessClock.get(userID) || 0
+    y.awarenessClock.set(userID, clock)
+    if (state === null) {
+      // only write if clock increases. cannot overwrite
+      if (y.awareness.has(userID) && uClock < clock) {
+        y.awareness.delete(userID)
+        removed.push(userID)
       }
+    } else if (uClock <= clock) { // allow to overwrite (e.g. when client was on, then offline)
+      if (y.awareness.has(userID)) {
+        updated.push(userID)
+      } else {
+        added.push(userID)
+      }
+      y.awareness.set(userID, state)
+      y.awarenessClock.set(userID, clock)
     }
   }
   if (added.length > 0 || updated.length > 0 || removed.length > 0) {
@@ -63,6 +68,7 @@ export const readUsersStateChange = (decoder, y) => {
 /**
  * @param {decoding.Decoder} decoder
  * @param {encoding.Encoder} encoder
+ * @return {Array<UserStateUpdate>}
  */
 export const forwardUsersStateChange = (decoder, encoder) => {
   const len = decoding.readVarUint(decoder)
@@ -71,10 +77,12 @@ export const forwardUsersStateChange = (decoder, encoder) => {
   encoding.writeVarUint(encoder, len)
   for (let i = 0; i < len; i++) {
     const userID = decoding.readVarUint(decoder)
+    const clock = decoding.readVarUint(decoder)
     const state = decoding.readVarString(decoder)
     encoding.writeVarUint(encoder, userID)
+    encoding.writeVarUint(encoder, clock)
     encoding.writeVarString(encoder, state)
-    updates.push({userID, state: JSON.parse(state)})
+    updates.push({userID, state: JSON.parse(state), clock})
   }
   return updates
 }
@@ -91,13 +99,23 @@ export const readAwarenessMessage = (decoder, y) => {
   }
 }
 
+/**
+ * @typedef {Object} UserState
+ * @property {number} UserState.userID
+ * @property {any} UserState.state
+ * @property {number} UserState.clock
+ */
+
 /**
  * @param {decoding.Decoder} decoder
  * @param {encoding.Encoder} encoder
+ * @return {Array<UserState>} Array of state updates
  */
 export const forwardAwarenessMessage = (decoder, encoder) => {
+  let s = []
   switch (decoding.readVarUint(decoder)) {
     case messageUsersStateChanged:
-      return forwardUsersStateChange(decoder, encoder)
+      s = forwardUsersStateChange(decoder, encoder)
   }
+  return s
 }

protocols/history.js

@@ -0,0 +1,46 @@
+
+import * as encoding from '../lib/encoding.js'
+import * as decoding from '../lib/decoding.js'
+import { Y } from '../utils/Y.js' // eslint-disable-line
+import { writeDeleteStore, readFreshDeleteStore, DeleteStore } from '../utils/DeleteStore.js' // eslint-disable-line
+import { writeStateMap, readStateMap } from '../utils/StateStore.js'
+
+/**
+ * @typedef {Object} HistorySnapshot
+ * @property {DeleteStore} HistorySnapshot.ds
+ * @property {Map<number,number>} HistorySnapshot.sm
+ * @property {Map<number,string>} HistorySnapshot.userMap
+ */
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {Y} y
+ * @param {Map<number, string>} userMap
+ */
+export const writeHistorySnapshot = (encoder, y, userMap) => {
+  writeDeleteStore(encoder, y.ds)
+  writeStateMap(encoder, y.ss.state)
+  encoding.writeVarUint(encoder, userMap.size)
+  userMap.forEach((accountname, userid) => {
+    encoding.writeVarUint(encoder, userid)
+    encoding.writeVarString(encoder, accountname)
+  })
+}
+
+/**
+ *
+ * @param {decoding.Decoder} decoder
+ * @return {HistorySnapshot}
+ */
+export const readHistorySnapshot = decoder => {
+  const ds = readFreshDeleteStore(decoder)
+  const sm = readStateMap(decoder)
+  const size = decoding.readVarUint(decoder)
+  const userMap = new Map()
+  for (let i = 0; i < size; i++) {
+    const userid = decoding.readVarUint(decoder)
+    const accountname = decoding.readVarString(decoder)
+    userMap.set(userid, accountname)
+  }
+  return { ds, sm, userMap }
+}

protocols/sync.js

@@ -0,0 +1,264 @@
+/**
+ * @module sync-protocol
+ */
+
+import * as encoding from '../lib/encoding.js'
+import * as decoding from '../lib/decoding.js'
+import * as ID from '../utils/ID.js'
+import { getStruct } from '../utils/structReferences.js'
+import { deleteItemRange } from '../utils/structManipulation.js'
+import { integrateRemoteStruct } from '../utils/integrateRemoteStructs.js'
+import { Y } from '../utils/Y.js' // eslint-disable-line
+import * as stringify from '../utils/structStringify.js'
+import { readStateMap, writeStateMap } from '../utils/StateStore.js'
+import { writeDeleteStore, readDeleteStore, stringifyDeleteStore } from '../utils/DeleteStore.js'
+
+/**
+ * @typedef {Map<number, number>} StateMap
+ */
+
+/**
+ * Core Yjs only defines three message types:
+ * • YjsSyncStep1: Includes the State Set of the sending client. When received, the client should reply with YjsSyncStep2.
+ * • YjsSyncStep2: Includes all missing structs and the complete delete set. When received, the the client is assured that
+ *   it received all information from the remote client.
+ *
+ * In a peer-to-peer network, you may want to introduce a SyncDone message type. Both parties should initiate the connection
+ * with SyncStep1. When a client received SyncStep2, it should reply with SyncDone. When the local client received both
+ * SyncStep2 and SyncDone, it is assured that it is synced to the remote client.
+ *
+ * In a client-server model, you want to handle this differently: The client should initiate the connection with SyncStep1.
+ * When the server receives SyncStep1, it should reply with SyncStep2 immediately followed by SyncStep1. The client replies
+ * with SyncStep2 when it receives SyncStep1. Optionally the server may send a SyncDone after it received SyncStep2, so the
+ * client knows that the sync is finished.  There are two reasons for this more elaborated sync model: 1. This protocol can
+ * easily be implemented on top of http and websockets. 2. The server shoul only reply to requests, and not initiate them.
+ * Therefore it is necesarry that the client initiates the sync.
+ *
+ * Construction of a message:
+ * [messageType : varUint, message definition..]
+ *
+ * Note: A message does not include information about the room name. This must to be handled by the upper layer protocol!
+ *
+ * stringify[messageType] stringifies a message definition (messageType is already read from the bufffer)
+ */
+
+export const messageYjsSyncStep1 = 0
+export const messageYjsSyncStep2 = 1
+export const messageYjsUpdate = 2
+
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ * @return {string}
+ */
+export const stringifyStructs = (decoder, y) => {
+  let str = ''
+  const len = decoding.readUint32(decoder)
+  for (let i = 0; i < len; i++) {
+    let reference = decoding.readVarUint(decoder)
+    let Constr = getStruct(reference)
+    let struct = new Constr()
+    let missing = struct._fromBinary(y, decoder)
+    let logMessage = '  ' + struct._logString()
+    if (missing.length > 0) {
+      logMessage += ' .. missing: ' + missing.map(stringify.stringifyItemID).join(', ')
+    }
+    str += logMessage + '\n'
+  }
+  return str
+}
+
+/**
+ * Write all Items that are not not included in ss to
+ * the encoder object.
+ *
+ * @param {encoding.Encoder} encoder
+ * @param {Y} y
+ * @param {StateMap} ss State Set received from a remote client. Maps from client id to number of created operations by client id.
+ */
+export const writeStructs = (encoder, y, ss) => {
+  const lenPos = encoding.length(encoder)
+  encoding.writeUint32(encoder, 0)
+  let len = 0
+  for (let user of y.ss.state.keys()) {
+    let clock = ss.get(user) || 0
+    if (user !== ID.RootFakeUserID) {
+      const minBound = ID.createID(user, clock)
+      const overlappingLeft = y.os.findPrev(minBound)
+      const rightID = overlappingLeft === null ? null : overlappingLeft._id
+      if (rightID !== null && rightID.user === user && rightID.clock + overlappingLeft._length > clock) {
+        // TODO: only write partial content (only missing content)
+        // const struct = overlappingLeft._clonePartial(clock - rightID.clock)
+        const struct = overlappingLeft
+        struct._toBinary(encoder)
+        len++
+      }
+      y.os.iterate(minBound, ID.createID(user, Number.MAX_VALUE), struct => {
+        struct._toBinary(encoder)
+        len++
+      })
+    }
+  }
+  encoding.setUint32(encoder, lenPos, len)
+}
+
+/**
+ * Read structs and delete operations from decoder and apply them on a shared document.
+ *
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ */
+export const readStructs = (decoder, y) => {
+  const len = decoding.readUint32(decoder)
+  for (let i = 0; i < len; i++) {
+    integrateRemoteStruct(decoder, y)
+  }
+}
+
+/**
+ * Read SyncStep1 and return it as a readable string.
+ *
+ * @param {decoding.Decoder} decoder
+ * @return {string}
+ */
+export const stringifySyncStep1 = (decoder) => {
+  let s = 'SyncStep1: '
+  const len = decoding.readUint32(decoder)
+  for (let i = 0; i < len; i++) {
+    const user = decoding.readVarUint(decoder)
+    const clock = decoding.readVarUint(decoder)
+    s += `(${user}:${clock})`
+  }
+  return s
+}
+
+/**
+ * Create a sync step 1 message based on the state of the current shared document.
+ *
+ * @param {encoding.Encoder} encoder
+ * @param {Y} y
+ */
+export const writeSyncStep1 = (encoder, y) => {
+  encoding.writeVarUint(encoder, messageYjsSyncStep1)
+  writeStateMap(encoder, y.ss.state)
+}
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {Y} y
+ * @param {Map<number, number>} ss
+ */
+export const writeSyncStep2 = (encoder, y, ss) => {
+  encoding.writeVarUint(encoder, messageYjsSyncStep2)
+  writeStructs(encoder, y, ss)
+  writeDeleteStore(encoder, y.ds)
+}
+
+/**
+ * Read SyncStep1 message and reply with SyncStep2.
+ *
+ * @param {decoding.Decoder} decoder The reply to the received message
+ * @param {encoding.Encoder} encoder The received message
+ * @param {Y} y
+ */
+export const readSyncStep1 = (decoder, encoder, y) =>
+  writeSyncStep2(encoder, y, readStateMap(decoder))
+
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ * @return {string}
+ */
+export const stringifySyncStep2 = (decoder, y) => {
+  let str = '  == Sync step 2:\n'
+  str += ' + Structs:\n'
+  str += stringifyStructs(decoder, y)
+  // write DS to string
+  str += ' + Delete Set:\n'
+  str += stringifyDeleteStore(decoder)
+  return str
+}
+
+/**
+ * Read and apply Structs and then DeleteStore to a y instance.
+ *
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ */
+export const readSyncStep2 = (decoder, y) => {
+  readStructs(decoder, y)
+  readDeleteStore(decoder, y)
+}
+
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ * @return {string}
+ */
+export const stringifyUpdate = (decoder, y) =>
+  '  == Update:\n' + stringifyStructs(decoder, y)
+
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {number} numOfStructs
+ * @param {encoding.Encoder} updates
+ */
+export const writeUpdate = (encoder, numOfStructs, updates) => {
+  encoding.writeVarUint(encoder, messageYjsUpdate)
+  encoding.writeUint32(encoder, numOfStructs)
+  encoding.writeBinaryEncoder(encoder, updates)
+}
+
+export const readUpdate = readStructs
+
+/**
+ * @param {decoding.Decoder} decoder
+ * @param {Y} y
+ * @return {string} The message converted to string
+ */
+export const stringifySyncMessage = (decoder, y) => {
+  const messageType = decoding.readVarUint(decoder)
+  let stringifiedMessage
+  let stringifiedMessageType
+  switch (messageType) {
+    case messageYjsSyncStep1:
+      stringifiedMessageType = 'YjsSyncStep1'
+      stringifiedMessage = stringifySyncStep1(decoder)
+      break
+    case messageYjsSyncStep2:
+      stringifiedMessageType = 'YjsSyncStep2'
+      stringifiedMessage = stringifySyncStep2(decoder, y)
+      break
+    case messageYjsUpdate:
+      stringifiedMessageType = 'YjsUpdate'
+      stringifiedMessage = stringifyStructs(decoder, y)
+      break
+    default:
+      stringifiedMessageType = 'Unknown'
+      stringifiedMessage = 'Unknown'
+  }
+  return `Message ${stringifiedMessageType}:\n${stringifiedMessage}`
+}
+
+/**
+ * @param {decoding.Decoder} decoder A message received from another client
+ * @param {encoding.Encoder} encoder The reply message. Will not be sent if empty.
+ * @param {Y} y
+ */
+export const readSyncMessage = (decoder, encoder, y) => {
+  const messageType = decoding.readVarUint(decoder)
+  switch (messageType) {
+    case messageYjsSyncStep1:
+      readSyncStep1(decoder, encoder, y)
+      break
+    case messageYjsSyncStep2:
+      y.transact(() => readSyncStep2(decoder, y), true)
+      break
+    case messageYjsUpdate:
+      y.transact(() => readUpdate(decoder, y), true)
+      break
+    default:
+      throw new Error('Unknown message type')
+  }
+  return messageType
+}

protocols/syncProtocol.mjs

@@ -1,492 +0,0 @@
-/**
- * @module sync-protocol
- */
-
-import * as encoding from '../lib/encoding.mjs'
-import * as decoding from '../lib/decoding.mjs'
-import * as ID from '../utils/ID.mjs'
-import { getStruct } from '../utils/structReferences.mjs'
-import { deleteItemRange } from '../utils/structManipulation.mjs'
-import { integrateRemoteStruct } from '../utils/integrateRemoteStructs.mjs'
-import { Y } from '../utils/Y.mjs' // eslint-disable-line
-import { Item } from '../structs/Item.mjs'
-
-/**
- * @typedef {Map<number, number>} StateSet
- */
-
-/**
- * Core Yjs only defines three message types:
- * • YjsSyncStep1: Includes the State Set of the sending client. When received, the client should reply with YjsSyncStep2.
- * • YjsSyncStep2: Includes all missing structs and the complete delete set. When received, the the client is assured that
- *   it received all information from the remote client.
- *
- * In a peer-to-peer network, you may want to introduce a SyncDone message type. Both parties should initiate the connection
- * with SyncStep1. When a client received SyncStep2, it should reply with SyncDone. When the local client received both
- * SyncStep2 and SyncDone, it is assured that it is synced to the remote client.
- *
- * In a client-server model, you want to handle this differently: The client should initiate the connection with SyncStep1.
- * When the server receives SyncStep1, it should reply with SyncStep2 immediately followed by SyncStep1. The client replies
- * with SyncStep2 when it receives SyncStep1. Optionally the server may send a SyncDone after it received SyncStep2, so the
- * client knows that the sync is finished.  There are two reasons for this more elaborated sync model: 1. This protocol can
- * easily be implemented on top of http and websockets. 2. The server shoul only reply to requests, and not initiate them.
- * Therefore it is necesarry that the client initiates the sync.
- *
- * Construction of a message:
- * [messageType : varUint, message definition..]
- *
- * Note: A message does not include information about the room name. This must to be handled by the upper layer protocol!
- *
- * stringify[messageType] stringifies a message definition (messageType is already read from the bufffer)
- */
-
-const messageYjsSyncStep1 = 0
-const messageYjsSyncStep2 = 1
-const messageYjsUpdate = 2
-
-/**
- * Stringifies a message-encoded Delete Set.
- *
- * @param {decoding.Decoder} decoder
- * @return {string}
- */
-export const stringifyDeleteSet = (decoder) => {
-  let str = ''
-  const dsLength = decoding.readUint32(decoder)
-  for (let i = 0; i < dsLength; i++) {
-    str += ' -' + decoding.readVarUint(decoder) + ':\n' // decodes user
-    const dvLength = decoding.readUint32(decoder)
-    for (let j = 0; j < dvLength; j++) {
-      str += `clock: ${decoding.readVarUint(decoder)}, length: ${decoding.readVarUint(decoder)}, gc: ${decoding.readUint8(decoder) === 1}\n`
-    }
-  }
-  return str
-}
-
-/**
- * Write the DeleteSet of a shared document to an Encoder.
- *
- * @param {encoding.Encoder} encoder
- * @param {Y} y
- */
-export const writeDeleteSet = (encoder, y) => {
-  let currentUser = null
-  let currentLength
-  let lastLenPos
-  let numberOfUsers = 0
-  const laterDSLenPus = encoding.length(encoder)
-  encoding.writeUint32(encoder, 0)
-  y.ds.iterate(null, null, n => {
-    const user = n._id.user
-    const clock = n._id.clock
-    const len = n.len
-    const gc = n.gc
-    if (currentUser !== user) {
-      numberOfUsers++
-      // a new user was foundimport { StateSet } from '../Store/StateStore.mjs' // eslint-disable-line
-
-      if (currentUser !== null) { // happens on first iteration
-        encoding.setUint32(encoder, lastLenPos, currentLength)
-      }
-      currentUser = user
-      encoding.writeVarUint(encoder, user)
-      // pseudo-fill pos
-      lastLenPos = encoding.length(encoder)
-      encoding.writeUint32(encoder, 0)
-      currentLength = 0
-    }
-    encoding.writeVarUint(encoder, clock)
-    encoding.writeVarUint(encoder, len)
-    encoding.writeUint8(encoder, gc ? 1 : 0)
-    currentLength++
-  })
-  if (currentUser !== null) { // happens on first iteration
-    encoding.setUint32(encoder, lastLenPos, currentLength)
-  }
-  encoding.setUint32(encoder, laterDSLenPus, numberOfUsers)
-}
-
-/**
- * Read delete set from Decoder and apply it to a shared document.
- *
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- */
-export const readDeleteSet = (decoder, y) => {
-  const dsLength = decoding.readUint32(decoder)
-  for (let i = 0; i < dsLength; i++) {
-    const user = decoding.readVarUint(decoder)
-    const dv = []
-    const dvLength = decoding.readUint32(decoder)
-    for (let j = 0; j < dvLength; j++) {
-      const from = decoding.readVarUint(decoder)
-      const len = decoding.readVarUint(decoder)
-      const gc = decoding.readUint8(decoder) === 1
-      dv.push({from, len, gc})
-    }
-    if (dvLength > 0) {
-      const deletions = []
-      let pos = 0
-      let d = dv[pos]
-      y.ds.iterate(ID.createID(user, 0), ID.createID(user, Number.MAX_VALUE), n => {
-        // cases:
-        // 1. d deletes something to the right of n
-        //  => go to next n (break)
-        // 2. d deletes something to the left of n
-        //  => create deletions
-        //  => reset d accordingly
-        //  *)=> if d doesn't delete anything anymore, go to next d (continue)
-        // 3. not 2) and d deletes something that also n deletes
-        //  => reset d so that it doesn't contain n's deletion
-        //  *)=> if d does not delete anything anymore, go to next d (continue)
-        while (d != null) {
-          var diff = 0 // describe the diff of length in 1) and 2)
-          if (n._id.clock + n.len <= d.from) {
-            // 1)
-            break
-          } else if (d.from < n._id.clock) {
-            // 2)
-            // delete maximum the len of d
-            // else delete as much as possible
-            diff = Math.min(n._id.clock - d.from, d.len)
-            // deleteItemRange(y, user, d.from, diff, true)
-            deletions.push([user, d.from, diff])
-          } else {
-            // 3)
-            diff = n._id.clock + n.len - d.from // never null (see 1)
-            if (d.gc && !n.gc) {
-              // d marks as gc'd but n does not
-              // then delete either way
-              // deleteItemRange(y, user, d.from, Math.min(diff, d.len), true)
-              deletions.push([user, d.from, Math.min(diff, d.len)])
-            }
-          }
-          if (d.len <= diff) {
-            // d doesn't delete anything anymore
-            d = dv[++pos]
-          } else {
-            d.from = d.from + diff // reset pos
-            d.len = d.len - diff // reset length
-          }
-        }
-      })
-      // TODO: It would be more performant to apply the deletes in the above loop
-      // Adapt the Tree implementation to support delete while iterating
-      for (let i = deletions.length - 1; i >= 0; i--) {
-        const del = deletions[i]
-        deleteItemRange(y, del[0], del[1], del[2], true)
-      }
-      // for the rest.. just apply it
-      for (; pos < dv.length; pos++) {
-        d = dv[pos]
-        deleteItemRange(y, user, d.from, d.len, true)
-        // deletions.push([user, d.from, d.len, d.gc)
-      }
-    }
-  }
-}
-
-/**
- * Read a StateSet from Decoder and return it as string.
- *
- * @param {decoding.Decoder} decoder
- * @return {string}
- */
-export const stringifyStateSet = decoder => {
-  let s = 'State Set: '
-  readStateSet(decoder).forEach((clock, user) => {
-    s += `(${user}: ${clock}), `
-  })
-  return s
-}
-
-/**
- * Write StateSet to Encoder
- *
- * @param {encoding.Encoder} encoder
- * @param {Y} y
- */
-export const writeStateSet = (encoder, y) => {
-  const state = y.ss.state
-  // write as fixed-size number to stay consistent with the other encode functions.
-  // => anytime we write the number of objects that follow, encode as fixed-size number.
-  encoding.writeUint32(encoder, state.size)
-  state.forEach((clock, user) => {
-    encoding.writeVarUint(encoder, user)
-    encoding.writeVarUint(encoder, clock)
-  })
-}
-
-/**
- * Read StateSet from Decoder and return as Map
- *
- * @param {decoding.Decoder} decoder
- * @return {StateSet}
- */
-export const readStateSet = decoder => {
-  const ss = new Map()
-  const ssLength = decoding.readUint32(decoder)
-  for (let i = 0; i < ssLength; i++) {
-    const user = decoding.readVarUint(decoder)
-    const clock = decoding.readVarUint(decoder)
-    ss.set(user, clock)
-  }
-  return ss
-}
-
-/**
- * Stringify an item id.
- *
- * @param {ID.ID | ID.RootID} id
- * @return {string}
- */
-export const stringifyID = id => id instanceof ID.ID ? `(${id.user},${id.clock})` : `(${id.name},${id.type})`
-
-/**
- * Stringify an item as ID. HHere, an item could also be a Yjs instance (e.g. item._parent).
- *
- * @param {Item | Y | null} item
- * @return {string}
- */
-export const stringifyItemID = item => {
-  let result
-  if (item === null) {
-    result = '()'
-  } else if (item instanceof Item) {
-    result = stringifyID(item._id)
-  } else {
-    // must be a Yjs instance
-    // Don't include Y in this module, so we prevent circular dependencies.
-    result = 'y'
-  }
-  return result
-}
-
-/**
- * Helper utility to convert an item to a readable format.
- *
- * @param {String} name The name of the item class (YText, ItemString, ..).
- * @param {Item} item The item instance.
- * @param {String} [append] Additional information to append to the returned
- *                          string.
- * @return {String} A readable string that represents the item object.
- *
- */
-export const logItemHelper = (name, item, append) => {
-  const left = item._left !== null ? stringifyID(item._left._lastId) : '()'
-  const origin = item._origin !== null ? stringifyID(item._origin._lastId) : '()'
-  return `${name}(id:${stringifyItemID(item)},left:${left},origin:${origin},right:${stringifyItemID(item._right)},parent:${stringifyItemID(item._parent)},parentSub:${item._parentSub}${append !== undefined ? ' - ' + append : ''})`
-}
-
-/**
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- * @return {string}
- */
-export const stringifyStructs = (decoder, y) => {
-  let str = ''
-  const len = decoding.readUint32(decoder)
-  for (let i = 0; i < len; i++) {
-    let reference = decoding.readVarUint(decoder)
-    let Constr = getStruct(reference)
-    let struct = new Constr()
-    let missing = struct._fromBinary(y, decoder)
-    let logMessage = '  ' + struct._logString()
-    if (missing.length > 0) {
-      logMessage += ' .. missing: ' + missing.map(stringifyItemID).join(', ')
-    }
-    str += logMessage + '\n'
-  }
-  return str
-}
-
-/**
- * Write all Items that are not not included in ss to
- * the encoder object.
- *
- * @param {encoding.Encoder} encoder
- * @param {Y} y
- * @param {StateSet} ss State Set received from a remote client. Maps from client id to number of created operations by client id.
- */
-export const writeStructs = (encoder, y, ss) => {
-  const lenPos = encoding.length(encoder)
-  encoding.writeUint32(encoder, 0)
-  let len = 0
-  for (let user of y.ss.state.keys()) {
-    let clock = ss.get(user) || 0
-    if (user !== ID.RootFakeUserID) {
-      const minBound = ID.createID(user, clock)
-      const overlappingLeft = y.os.findPrev(minBound)
-      const rightID = overlappingLeft === null ? null : overlappingLeft._id
-      if (rightID !== null && rightID.user === user && rightID.clock + overlappingLeft._length > clock) {
-        // TODO: only write partial content (only missing content)
-        // const struct = overlappingLeft._clonePartial(clock - rightID.clock)
-        const struct = overlappingLeft
-        struct._toBinary(encoder)
-        len++
-      }
-      y.os.iterate(minBound, ID.createID(user, Number.MAX_VALUE), struct => {
-        struct._toBinary(encoder)
-        len++
-      })
-    }
-  }
-  encoding.setUint32(encoder, lenPos, len)
-}
-
-/**
- * Read structs and delete operations from decoder and apply them on a shared document.
- *
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- */
-export const readStructs = (decoder, y) => {
-  const len = decoding.readUint32(decoder)
-  for (let i = 0; i < len; i++) {
-    integrateRemoteStruct(decoder, y)
-  }
-}
-
-/**
- * Read SyncStep1 and return it as a readable string.
- *
- * @param {decoding.Decoder} decoder
- * @return {string}
- */
-export const stringifySyncStep1 = (decoder) => {
-  let s = 'SyncStep1: '
-  const len = decoding.readUint32(decoder)
-  for (let i = 0; i < len; i++) {
-    const user = decoding.readVarUint(decoder)
-    const clock = decoding.readVarUint(decoder)
-    s += `(${user}:${clock})`
-  }
-  return s
-}
-
-/**
- * Create a sync step 1 message based on the state of the current shared document.
- *
- * @param {encoding.Encoder} encoder
- * @param {Y} y
- */
-export const writeSyncStep1 = (encoder, y) => {
-  encoding.writeVarUint(encoder, messageYjsSyncStep1)
-  writeStateSet(encoder, y)
-}
-
-/**
- * Read SyncStep1 message and reply with SyncStep2.
- *
- * @param {decoding.Decoder} decoder The reply to the received message
- * @param {encoding.Encoder} encoder The received message
- * @param {Y} y
- */
-export const readSyncStep1 = (decoder, encoder, y) => {
-  // read sync step 1 message
-  const ss = readStateSet(decoder)
-  // write sync step 2
-  encoding.writeVarUint(encoder, messageYjsSyncStep2)
-  writeStructs(encoder, y, ss)
-  writeDeleteSet(encoder, y)
-}
-
-/**
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- * @return {string}
- */
-export const stringifySyncStep2 = (decoder, y) => {
-  let str = '  == Sync step 2:\n'
-  str += ' + Structs:\n'
-  str += stringifyStructs(decoder, y)
-  // write DS to string
-  str += ' + Delete Set:\n'
-  str += stringifyDeleteSet(decoder)
-  return str
-}
-
-/**
- * Read and apply Structs and then DeleteSet to a y instance.
- *
- * @param {decoding.Decoder} decoder
- * @param {encoding.Encoder} encoder
- * @param {Y} y
- */
-export const readSyncStep2 = (decoder, encoder, y) => {
-  readStructs(decoder, y)
-  readDeleteSet(decoder, y)
-}
-
-/**
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- * @return {string}
- */
-export const stringifyUpdate = (decoder, y) =>
-  '  == Update:\n' + stringifyStructs(decoder, y)
-
-/**
- * @param {encoding.Encoder} encoder
- * @param {number} numOfStructs
- * @param {encoding.Encoder} updates
- */
-export const writeUpdate = (encoder, numOfStructs, updates) => {
-  encoding.writeVarUint(encoder, messageYjsUpdate)
-  encoding.writeUint32(encoder, numOfStructs)
-  encoding.writeBinaryEncoder(encoder, updates)
-}
-
-export const readUpdate = readStructs
-
-/**
- * @param {decoding.Decoder} decoder
- * @param {Y} y
- * @return {string} The message converted to string
- */
-export const stringifySyncMessage = (decoder, y) => {
-  const messageType = decoding.readVarUint(decoder)
-  let stringifiedMessage
-  let stringifiedMessageType
-  switch (messageType) {
-    case messageYjsSyncStep1:
-      stringifiedMessageType = 'YjsSyncStep1'
-      stringifiedMessage = stringifySyncStep1(decoder)
-      break
-    case messageYjsSyncStep2:
-      stringifiedMessageType = 'YjsSyncStep2'
-      stringifiedMessage = stringifySyncStep2(decoder, y)
-      break
-    case messageYjsUpdate:
-      stringifiedMessageType = 'YjsUpdate'
-      stringifiedMessage = stringifyStructs(decoder, y)
-      break
-    default:
-      stringifiedMessageType = 'Unknown'
-      stringifiedMessage = 'Unknown'
-  }
-  return `Message ${stringifiedMessageType}:\n${stringifiedMessage}`
-}
-
-/**
- * @param {decoding.Decoder} decoder A message received from another client
- * @param {encoding.Encoder} encoder The reply message. Will not be sent if empty.
- * @param {Y} y
- */
-export const readSyncMessage = (decoder, encoder, y) => {
-  const messageType = decoding.readVarUint(decoder)
-  switch (messageType) {
-    case messageYjsSyncStep1:
-      readSyncStep1(decoder, encoder, y)
-      break
-    case messageYjsSyncStep2:
-      y.transact(() => readSyncStep2(decoder, encoder, y), true)
-      break
-    case messageYjsUpdate:
-      y.transact(() => readUpdate(decoder, y), true)
-      break
-    default:
-      throw new Error('Unknown message type')
-  }
-  return messageType
-}

provider/websocket.js

@@ -0,0 +1,5 @@
+/**
+ * @module provider/websocket
+ */
+
+export * from './websocket/WebSocketProvider.js'

provider/websocket.mjs

@@ -1,5 +0,0 @@
-/**
- * @module provider/websocket
- */
-
-export * from './websocket/WebSocketProvider.mjs'

provider/websocket/WebSocketProvider.js

@@ -0,0 +1,183 @@
+/*
+Unlike stated in the LICENSE file, it is not necessary to include the copyright notice and permission notice when you copy code from this file.
+*/
+
+/**
+ * @module provider/websocket
+ */
+
+/* eslint-env browser */
+
+import * as Y from '../../index.js'
+import * as bc from '../../lib/broadcastchannel.js'
+
+const messageSync = 0
+const messageAwareness = 1
+const messageAuth = 2
+
+const reconnectTimeout = 3000
+
+/**
+ * @param {WebsocketsSharedDocument} doc
+ * @param {string} reason
+ */
+const permissionDeniedHandler = (doc, reason) => console.warn(`Permission denied to access ${doc.url}.\n${reason}`)
+
+/**
+ * @param {WebsocketsSharedDocument} doc
+ * @param {ArrayBuffer} buf
+ * @return {Y.encoding.Encoder}
+ */
+const readMessage = (doc, buf) => {
+  const decoder = Y.decoding.createDecoder(buf)
+  const encoder = Y.encoding.createEncoder()
+  const messageType = Y.decoding.readVarUint(decoder)
+  switch (messageType) {
+    case messageSync:
+      Y.encoding.writeVarUint(encoder, messageSync)
+      doc.mux(() =>
+        Y.syncProtocol.readSyncMessage(decoder, encoder, doc)
+      )
+      break
+    case messageAwareness:
+      Y.awarenessProtocol.readAwarenessMessage(decoder, doc)
+      break
+    case messageAuth:
+      Y.authProtocol.readAuthMessage(decoder, doc, permissionDeniedHandler)
+  }
+  return encoder
+}
+
+const setupWS = (doc, url) => {
+  const websocket = new WebSocket(url)
+  websocket.binaryType = 'arraybuffer'
+  doc.ws = websocket
+  websocket.onmessage = event => {
+    const encoder = readMessage(doc, event.data)
+    if (Y.encoding.length(encoder) > 1) {
+      websocket.send(Y.encoding.toBuffer(encoder))
+    }
+  }
+  websocket.onclose = () => {
+    doc.ws = null
+    doc.wsconnected = false
+    doc.emit('status', {
+      status: 'connected'
+    })
+    setTimeout(setupWS, reconnectTimeout, doc, url)
+  }
+  websocket.onopen = () => {
+    doc.wsconnected = true
+    doc.emit('status', {
+      status: 'disconnected'
+    })
+    // always send sync step 1 when connected
+    const encoder = Y.encoding.createEncoder()
+    Y.encoding.writeVarUint(encoder, messageSync)
+    Y.syncProtocol.writeSyncStep1(encoder, doc)
+    websocket.send(Y.encoding.toBuffer(encoder))
+    // force send stored awareness info
+    doc.setAwarenessField(null, null)
+  }
+}
+
+const broadcastUpdate = (y, transaction) => {
+  if (transaction.encodedStructsLen > 0) {
+    y.mux(() => {
+      const encoder = Y.encoding.createEncoder()
+      Y.encoding.writeVarUint(encoder, messageSync)
+      Y.syncProtocol.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
+      const buf = Y.encoding.toBuffer(encoder)
+      if (y.wsconnected) {
+        y.ws.send(buf)
+      }
+      bc.publish(y.url, buf)
+    })
+  }
+}
+
+class WebsocketsSharedDocument extends Y.Y {
+  constructor (url, opts) {
+    super(opts)
+    this.url = url
+    this.wsconnected = false
+    this.mux = Y.createMutex()
+    this.ws = null
+    this._localAwarenessState = {}
+    this.awareness = new Map()
+    this.awarenessClock = new Map()
+    setupWS(this, url)
+    this.on('afterTransaction', broadcastUpdate)
+    this._bcSubscriber = data => {
+      const encoder = readMessage(this, data) // already muxed
+      this.mux(() => {
+        if (Y.encoding.length(encoder) > 1) {
+          bc.publish(url, Y.encoding.toBuffer(encoder))
+        }
+      })
+    }
+    bc.subscribe(url, this._bcSubscriber)
+    // send sync step1 to bc
+    this.mux(() => {
+      const encoder = Y.encoding.createEncoder()
+      Y.encoding.writeVarUint(encoder, messageSync)
+      Y.syncProtocol.writeSyncStep1(encoder, this)
+      bc.publish(url, Y.encoding.toBuffer(encoder))
+    })
+  }
+  getLocalAwarenessInfo () {
+    return this._localAwarenessState
+  }
+  getAwarenessInfo () {
+    return this.awareness
+  }
+  setAwarenessField (field, value) {
+    if (field !== null) {
+      this._localAwarenessState[field] = value
+    }
+    if (this.wsconnected) {
+      const clock = (this.awarenessClock.get(this.userID) || 0) + 1
+      this.awarenessClock.set(this.userID, clock)
+      const encoder = Y.encoding.createEncoder()
+      Y.encoding.writeVarUint(encoder, messageAwareness)
+      Y.awarenessProtocol.writeUsersStateChange(encoder, [{ userID: this.userID, state: this._localAwarenessState, clock }])
+      const buf = Y.encoding.toBuffer(encoder)
+      this.ws.send(buf)
+    }
+  }
+}
+
+/**
+ * Websocket Provider for Yjs. Creates a single websocket connection to each document.
+ * The document name is attached to the provided url. I.e. the following example
+ * creates a websocket connection to http://localhost:1234/my-document-name
+ *
+ * @example
+ *   import { WebsocketProvider } from 'yjs/provider/websocket/client.js'
+ *   const provider = new WebsocketProvider('http://localhost:1234')
+ *   const ydocument = provider.get('my-document-name')
+ */
+export class WebsocketProvider {
+  constructor (url) {
+    // ensure that url is always ends with /
+    while (url[url.length - 1] === '/') {
+      url = url.slice(0, url.length - 1)
+    }
+    this.url = url + '/'
+    /**
+     * @type {Map<string, WebsocketsSharedDocument>}
+     */
+    this.docs = new Map()
+  }
+  /**
+   * @param {string} name
+   * @return {WebsocketsSharedDocument}
+   */
+  get (name, opts) {
+    let doc = this.docs.get(name)
+    if (doc === undefined) {
+      doc = new WebsocketsSharedDocument(this.url + name, opts)
+    }
+    return doc
+  }
+}

provider/websocket/WebSocketProvider.mjs

@@ -1,125 +0,0 @@
-/**
- * @module provider/websocket
- */
-
-/* eslint-env browser */
-
-import * as Y from '../../index.mjs'
-export * from '../../index.mjs'
-
-const messageSync = 0
-const messageAwareness = 1
-
-const reconnectTimeout = 100
-
-const setupWS = (doc, url) => {
-  const websocket = new WebSocket(url)
-  websocket.binaryType = 'arraybuffer'
-  doc.ws = websocket
-  websocket.onmessage = event => {
-    const decoder = Y.createDecoder(event.data)
-    const encoder = Y.createEncoder()
-    const messageType = Y.readVarUint(decoder)
-    switch (messageType) {
-      case messageSync:
-        Y.writeVarUint(encoder, messageSync)
-        doc.mux(() =>
-          Y.readSyncMessage(decoder, encoder, doc)
-        )
-        break
-      case messageAwareness:
-        Y.readAwarenessMessage(decoder, doc)
-        break
-    }
-    if (Y.length(encoder) > 1) {
-      websocket.send(Y.toBuffer(encoder))
-    }
-  }
-  websocket.onclose = () => {
-    doc.ws = null
-    doc.wsconnected = false
-    doc.emit('status', {
-      status: 'connected'
-    })
-    setTimeout(setupWS, reconnectTimeout, doc, url)
-  }
-  websocket.onopen = () => {
-    doc.wsconnected = true
-    doc.emit('status', {
-      status: 'disconnected'
-    })
-    // always send sync step 1 when connected
-    const encoder = Y.createEncoder()
-    Y.writeVarUint(encoder, messageSync)
-    Y.writeSyncStep1(encoder, doc)
-    websocket.send(Y.toBuffer(encoder))
-    // force send stored awareness info
-    doc.setAwarenessField(null, null)
-  }
-}
-
-const broadcastUpdate = (y, transaction) => {
-  if (y.wsconnected && transaction.encodedStructsLen > 0) {
-    y.mux(() => {
-      const encoder = Y.createEncoder()
-      Y.writeVarUint(encoder, messageSync)
-      Y.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
-      y.ws.send(Y.toBuffer(encoder))
-    })
-  }
-}
-
-class WebsocketsSharedDocument extends Y.Y {
-  constructor (url) {
-    super()
-    this.wsconnected = false
-    this.mux = Y.createMutex()
-    this.ws = null
-    this._localAwarenessState = {}
-    this.awareness = new Map()
-    setupWS(this, url)
-    this.on('afterTransaction', broadcastUpdate)
-  }
-  getLocalAwarenessInfo () {
-    return this._localAwarenessState
-  }
-  getAwarenessInfo () {
-    return this.awareness
-  }
-  setAwarenessField (field, value) {
-    if (field !== null) {
-      this._localAwarenessState[field] = value
-    }
-    if (this.wsconnected) {
-      const encoder = Y.createEncoder()
-      Y.writeVarUint(encoder, messageAwareness)
-      Y.writeUsersStateChange(encoder, [{ userID: this.userID, state: this._localAwarenessState }])
-      this.ws.send(Y.toBuffer(encoder))
-    }
-  }
-}
-
-export class WebsocketProvider {
-  constructor (url) {
-    // ensure that url is always ends with /
-    while (url[url.length - 1] === '/') {
-      url = url.slice(0, url.length - 1)
-    }
-    this.url = url + '/'
-    /**
-     * @type {Map<string, WebsocketsSharedDocument>}
-     */
-    this.docs = new Map()
-  }
-  /**
-   * @param {string} name
-   * @return {WebsocketsSharedDocument}
-   */
-  get (name) {
-    let doc = this.docs.get(name)
-    if (doc === undefined) {
-      doc = new WebsocketsSharedDocument(this.url + name)
-    }
-    return doc
-  }
-}

provider/websocket/index.js

@@ -0,0 +1,2 @@
+
+export * from './WebSocketProvider.js'

provider/websocket/server.js

@@ -0,0 +1,155 @@
+/*
+Unlike stated in the LICENSE file, it is not necessary to include the copyright notice and permission notice when you copy code from this file.
+*/
+
+/**
+ * @module provider/websocket/server
+ */
+
+const Y = require('../../build/yjs.js')
+const WebSocket = require('ws')
+const http = require('http')
+
+const port = process.env.PORT || 1234
+
+// disable gc when using snapshots!
+const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0'
+const persistenceDir = process.env.YPERSISTENCE
+let persistence = null
+if (typeof persistenceDir === 'string') {
+  const LevelDbPersistence = require('../../persistence/leveldb.js').LevelDbPersistence
+  persistence = new LevelDbPersistence(persistenceDir)
+}
+
+const server = http.createServer((req, res) => {
+  res.writeHead(200, { 'Content-Type': 'text/plain' })
+  res.end('okay')
+})
+
+const wss = new WebSocket.Server({ noServer: true })
+
+const docs = new Map()
+
+const messageSync = 0
+const messageAwareness = 1
+const messageAuth = 2
+
+const afterTransaction = (doc, transaction) => {
+  if (transaction.encodedStructsLen > 0) {
+    const encoder = Y.encoding.createEncoder()
+    Y.encoding.writeVarUint(encoder, messageSync)
+    Y.syncProtocol.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
+    const message = Y.encoding.toBuffer(encoder)
+    doc.conns.forEach((_, conn) => conn.send(message))
+  }
+}
+
+class WSSharedDoc extends Y.Y {
+  constructor () {
+    super({ gc: gcEnabled })
+    this.mux = Y.createMutex()
+    /**
+     * Maps from conn to set of controlled user ids. Delete all user ids from awareness when this conn is closed
+     * @type {Map<Object, Set<number>>}
+     */
+    this.conns = new Map()
+    this.awareness = new Map()
+    this.awarenessClock = new Map()
+    this.on('afterTransaction', afterTransaction)
+  }
+}
+
+const messageListener = (conn, doc, message) => {
+  const encoder = Y.encoding.createEncoder()
+  const decoder = Y.decoding.createDecoder(message)
+  const messageType = Y.decoding.readVarUint(decoder)
+  switch (messageType) {
+    case messageSync:
+      Y.encoding.writeVarUint(encoder, messageSync)
+      Y.syncProtocol.readSyncMessage(decoder, encoder, doc)
+      if (Y.encoding.length(encoder) > 1) {
+        conn.send(Y.encoding.toBuffer(encoder))
+      }
+      break
+    case messageAwareness: {
+      Y.encoding.writeVarUint(encoder, messageAwareness)
+      const updates = Y.awarenessProtocol.forwardAwarenessMessage(decoder, encoder)
+      updates.forEach(update => {
+        doc.awareness.set(update.userID, update.state)
+        doc.awarenessClock.set(update.userID, update.clock)
+        doc.conns.get(conn).add(update.userID)
+      })
+      const buff = Y.encoding.toBuffer(encoder)
+      doc.conns.forEach((_, c) => {
+        c.send(buff)
+      })
+      break
+    }
+  }
+}
+
+const setupConnection = (conn, req) => {
+  conn.binaryType = 'arraybuffer'
+  // get doc, create if it does not exist yet
+  const docName = req.url.slice(1)
+  let doc = docs.get(docName)
+  if (doc === undefined) {
+    doc = new WSSharedDoc()
+    if (persistence !== null) {
+      persistence.bindState(docName, doc)
+    }
+    docs.set(docName, doc)
+  }
+  doc.conns.set(conn, new Set())
+  // listen and reply to events
+  conn.on('message', message => messageListener(conn, doc, message))
+  conn.on('close', () => {
+    const controlledIds = doc.conns.get(conn)
+    doc.conns.delete(conn)
+    const encoder = Y.encoding.createEncoder()
+    Y.encoding.writeVarUint(encoder, messageAwareness)
+    Y.awarenessProtocol.writeUsersStateChange(encoder, Array.from(controlledIds).map(userID => {
+      const clock = (doc.awarenessClock.get(userID) || 0) + 1
+      doc.awareness.delete(userID)
+      doc.awarenessClock.delete(userID)
+      return { userID, state: null, clock }
+    }))
+    const buf = Y.encoding.toBuffer(encoder)
+    doc.conns.forEach((_, conn) => conn.send(buf))
+    if (doc.conns.size === 0 && persistence !== null) {
+      // if persisted, we store state and destroy ydocument
+      persistence.writeState(docName, doc).then(() => {
+        doc.destroy()
+      })
+      docs.delete(docName)
+    }
+  })
+  // send sync step 1
+  const encoder = Y.encoding.createEncoder()
+  Y.encoding.writeVarUint(encoder, messageSync)
+  Y.syncProtocol.writeSyncStep1(encoder, doc)
+  conn.send(Y.encoding.toBuffer(encoder))
+  if (doc.awareness.size > 0) {
+    const encoder = Y.encoding.createEncoder()
+    const userStates = []
+    doc.awareness.forEach((state, userID) => {
+      userStates.push({ state, userID, clock: (doc.awarenessClock.get(userID) || 0) })
+    })
+    Y.encoding.writeVarUint(encoder, messageAwareness)
+    Y.awarenessProtocol.writeUsersStateChange(encoder, userStates)
+    conn.send(Y.encoding.toBuffer(encoder))
+  }
+}
+
+wss.on('connection', setupConnection)
+
+server.on('upgrade', (request, socket, head) => {
+  // You may check auth of request here..
+  wss.handleUpgrade(request, socket, head, ws => {
+    wss.emit('connection', ws, request)
+  })
+})
+
+server.listen(port)
+
+console.log('running on port', port)

provider/websocket/server.mjs

@@ -1,126 +0,0 @@
-/**
- * @module provider/websocket
- */
-
-import * as Y from '../../index.mjs'
-import WebSocket from 'ws'
-import http from 'http'
-
-const port = process.env.PORT || 1234
-
-const server = http.createServer((req, res) => {
-  res.writeHead(200, { 'Content-Type': 'text/plain' })
-  res.end('okay')
-})
-
-const wss = new WebSocket.Server({ noServer: true })
-
-const docs = new Map()
-
-const messageSync = 0
-const messageAwareness = 1
-
-const afterTransaction = (doc, transaction) => {
-  if (transaction.encodedStructsLen > 0) {
-    const encoder = Y.createEncoder()
-    Y.writeVarUint(encoder, messageSync)
-    Y.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
-    const message = Y.toBuffer(encoder)
-    doc.conns.forEach((_, conn) => conn.send(message))
-  }
-}
-
-class WSSharedDoc extends Y.Y {
-  constructor () {
-    super()
-    this.mux = Y.createMutex()
-    /**
-     * Maps from conn to set of controlled user ids. Delete all user ids from awareness when this conn is closed
-     * @type {Map<Object, Set<number>>}
-     */
-    this.conns = new Map()
-    this.awareness = new Map()
-    this.on('afterTransaction', afterTransaction)
-  }
-}
-
-const messageListener = (conn, doc, message) => {
-  const encoder = Y.createEncoder()
-  const decoder = Y.createDecoder(message)
-  const messageType = Y.readVarUint(decoder)
-  switch (messageType) {
-    case messageSync:
-      Y.writeVarUint(encoder, messageSync)
-      Y.readSyncMessage(decoder, encoder, doc)
-      if (Y.length(encoder) > 1) {
-        conn.send(Y.toBuffer(encoder))
-      }
-      break
-    case messageAwareness: {
-      Y.writeVarUint(encoder, messageAwareness)
-      const updates = Y.forwardAwarenessMessage(decoder, encoder)
-      updates.forEach(update => {
-        doc.awareness.set(update.userID, update.state)
-        doc.conns.get(conn).add(update.userID)
-      })
-      const buff = Y.toBuffer(encoder)
-      doc.conns.forEach((_, c) => {
-        c.send(buff)
-      })
-      break
-    }
-  }
-}
-
-const setupConnection = (conn, req) => {
-  conn.binaryType = 'arraybuffer'
-  // get doc, create if it does not exist yet
-  let doc = docs.get(req.url.slice(1))
-  if (doc === undefined) {
-    doc = new WSSharedDoc()
-    docs.set(req.url.slice(1), doc)
-  }
-  doc.conns.set(conn, new Set())
-  // listen and reply to events
-  conn.on('message', message => messageListener(conn, doc, message))
-  conn.on('close', () => {
-    const controlledIds = doc.conns.get(conn)
-    doc.conns.delete(conn)
-    const encoder = Y.createEncoder()
-    Y.writeVarUint(encoder, messageAwareness)
-    Y.writeUsersStateChange(encoder, Array.from(controlledIds).map(userID => {
-      doc.awareness.delete(userID)
-      return { userID, state: null }
-    }))
-    const buf = Y.toBuffer(encoder)
-    doc.conns.forEach((_, conn) => conn.send(buf))
-  })
-  // send sync step 1
-  const encoder = Y.createEncoder()
-  Y.writeVarUint(encoder, messageSync)
-  Y.writeSyncStep1(encoder, doc)
-  conn.send(Y.toBuffer(encoder))
-  if (doc.awareness.size > 0) {
-    const encoder = Y.createEncoder()
-    const userStates = []
-    doc.awareness.forEach((state, userID) => {
-      userStates.push({ state, userID })
-    })
-    Y.writeVarUint(encoder, messageAwareness)
-    Y.writeUsersStateChange(encoder, userStates)
-    conn.send(Y.toBuffer(encoder))
-  }
-}
-
-wss.on('connection', setupConnection)
-
-server.on('upgrade', (request, socket, head) => {
-  // You may check auth of request here..
-  wss.handleUpgrade(request, socket, head, ws => {
-    wss.emit('connection', ws, request)
-  })
-})
-
-server.listen(port)
-
-console.log('running on port', port)

provider/ydb/NamedEventHandler.js

@@ -2,7 +2,7 @@
  * @module provider/ydb
  */
 
-import * as globals from './globals.mjs'
+import * as globals from './globals.js'
 
 export const Class = class NamedEventHandler {
   constructor () {

provider/ydb/YdbClient.js

@@ -3,19 +3,19 @@
  */
 
 /* eslint-env browser */
-import * as idbactions from './idbactions.mjs'
-import * as globals from '../../lib/globals.mjs'
-import * as message from './message.mjs'
-import * as bc from './broadcastchannel.mjs'
-import * as encoding from '../../lib/encoding.mjs'
-import * as logging from '../../lib/logging.mjs'
-import * as idb from '../../lib/idb.mjs'
-import * as decoding from '../../lib/decoding.mjs'
-import { Y } from '../../utils/Y.mjs'
-import { integrateRemoteStruct } from '../MessageHandler/integrateRemoteStructs.mjs'
-import { createMutualExclude } from '../../lib/mutualExclude.mjs'
+import * as idbactions from './idbactions.js'
+import * as globals from '../../lib/globals.js'
+import * as message from './message.js'
+import * as bc from './broadcastchannel.js'
+import * as encoding from '../../lib/encoding.js'
+import * as logging from '../../lib/logging.js'
+import * as idb from '../../lib/idb.js'
+import * as decoding from '../../lib/decoding.js'
+import { Y } from '../../utils/Y.js'
+import { integrateRemoteStruct } from '../MessageHandler/integrateRemoteStructs.js'
+import { createMutualExclude } from '../../lib/mutualExclude.js'
 
-import * as NamedEventHandler from './NamedEventHandler.mjs'
+import * as NamedEventHandler from './NamedEventHandler.js'
 
 /**
  * @typedef RoomState

provider/ydb/YdbClient.test.js

@@ -4,11 +4,11 @@
 
 /* eslint-env browser */
 
-import * as test from './test.mjs'
-import * as ydbClient from './YdbClient.mjs'
-import * as globals from './globals.mjs'
-import * as idbactions from './idbactions.mjs'
-import * as logging from './logging.mjs'
+import * as test from './test.js'
+import * as ydbClient from './YdbClient.js'
+import * as globals from './globals.js'
+import * as idbactions from './idbactions.js'
+import * as logging from './logging.js'
 
 const wsUrl = 'ws://127.0.0.1:8899/ws'
 const testRoom = 'testroom'

provider/ydb/broadcastchannel.js

@@ -4,12 +4,13 @@
 
 /* eslint-env browser */
 
-import * as decoding from '../../lib/decoding.mjs'
-import * as encoding from '../../lib/encoding.mjs'
-import * as globals from '../../lib/globals.mjs'
-import * as NamedEventHandler from './NamedEventHandler.mjs'
+import * as decoding from '../../lib/decoding.js'
+import * as encoding from '../../lib/encoding.js'
+import * as globals from '../../lib/globals.js'
+import * as NamedEventHandler from './NamedEventHandler.js'
 
 const bc = new BroadcastChannel('ydb-client')
+
 /**
  * @type {Map<string, Set<Function>>}
  */

provider/ydb/idbactions.js

@@ -33,11 +33,11 @@
  *   - A client may update a room when the room is in either US or Co
  */
 
-import * as encoding from '../../lib/encoding.mjs'
-import * as decoding from '../../lib/decoding.mjs'
-import * as idb from '../../lib/idb.mjs'
-import * as globals from '../../lib/globals.mjs'
-import * as message from './message.mjs'
+import * as encoding from '../../lib/encoding.js'
+import * as decoding from '../../lib/decoding.js'
+import * as idb from '../../lib/idb.js'
+import * as globals from '../../lib/globals.js'
+import * as message from './message.js'
 
 /**
  * Get 'client-unconfirmed' store from transaction
@@ -262,7 +262,7 @@ export const getRoomMetas = t => {
       result.push({
         room: metakey.slice(5),
         rsid,
-        offset: keys.reduce((cur, key) => globals.max(decodeHUKey(key).offset, cur), offset)
+        offset: keys.reduce((cur, key) => math.max(decodeHUKey(key).offset, cur), offset)
       })
     })
   ).then(() => globals.presolve(result))

provider/ydb/idbactions.test.js

@@ -1,6 +1,6 @@
-import * as globals from '../../lib/globals.mjs'
-import * as idbactions from './idbactions.mjs'
-import * as test from '../../lib/testing.mjs'
+import * as globals from '../../lib/globals.js'
+import * as idbactions from './idbactions.js'
+import * as test from '../../lib/testing.js'
 
 idbactions.deleteDB().then(() => idbactions.openDB()).then(db => {
   test.run('update lifetime 1', async (testname) => {

provider/ydb/index.js

@@ -2,7 +2,7 @@
  * @module provider/ydb
  */
 
-import * as ydbclient from './YdbClient.mjs'
+import * as ydbclient from './YdbClient.js'
 
 /**
  * @param {string} url

provider/ydb/message.js

@@ -2,11 +2,11 @@
  * @module provider/ydb
  */
 
-import * as encoding from './encoding.mjs'
-import * as decoding from './decoding.mjs'
-import * as idbactions from './idbactions.mjs'
-import * as logging from './logging.mjs'
-import * as bc from './broadcastchannel.mjs'
+import * as encoding from './encoding.js'
+import * as decoding from './decoding.js'
+import * as idbactions from './idbactions.js'
+import * as logging from './logging.js'
+import * as bc from './broadcastchannel.js'
 
 /* make sure to update message.go in ydb when updating these values.. */
 export const MESSAGE_UPDATE = 0 // TODO: rename host_unconfirmed?

rollup.config.js

@@ -3,8 +3,11 @@ import commonjs from 'rollup-plugin-commonjs'
 import babel from 'rollup-plugin-babel'
 import uglify from 'rollup-plugin-uglify-es'
 
+// set this to [] to disable obfuscation
+const minificationPlugins = process.env.PRODUCTION ? [babel(), uglify()] : []
+
 export default [{
-  input: './index.mjs',
+  input: './index.js',
   output: [{
     name: 'Y',
     file: 'build/yjs.js',
@@ -12,9 +15,9 @@ export default [{
     sourcemap: true
   }]
 }, {
-  input: 'tests/index.mjs',
+  input: 'tests/index.js',
   output: {
-    file: 'build/y.test.mjs',
+    file: 'build/y.test.js',
     format: 'iife',
     name: 'ytests',
     sourcemap: true
@@ -27,7 +30,22 @@ export default [{
     commonjs()
   ]
 }, {
-  input: './examples/prosemirror.mjs',
+  input: './examples/codemirror.js',
+  output: {
+    name: 'codemirror',
+    file: 'examples/build/codemirror.js',
+    format: 'iife',
+    sourcemap: true
+  },
+  plugins: [
+    nodeResolve({
+      sourcemap: true,
+      module: true
+    }),
+    commonjs()
+  ].concat(minificationPlugins)
+}, {
+  input: './examples/prosemirror.js',
   output: {
     name: 'prosemirror',
     file: 'examples/build/prosemirror.js',
@@ -39,36 +57,28 @@ export default [{
       sourcemap: true,
       module: true
     }),
-    commonjs(),
-    babel(),
-    uglify()
-  ]
+    commonjs()
+  ].concat(minificationPlugins)
 }, {
-  input: './examples/dom.mjs',
+  input: './examples/dom.js',
   output: {
     name: 'dom',
     file: 'examples/build/dom.js',
     format: 'iife',
     sourcemap: true
   },
-  plugins: [
-    babel(),
-    uglify()
-  ]
+  plugins: minificationPlugins
 }, {
-  input: './examples/textarea.mjs',
+  input: './examples/textarea.js',
   output: {
     name: 'textarea',
     file: 'examples/build/textarea.js',
     format: 'iife',
     sourcemap: true
   },
-  plugins: [
-    babel(),
-    uglify()
-  ]
+  plugins: minificationPlugins
 }, {
-  input: './examples/quill.mjs',
+  input: './examples/quill.js',
   output: {
     name: 'textarea',
     file: 'examples/build/quill.js',
@@ -80,8 +90,6 @@ export default [{
       sourcemap: true,
       module: true
     }),
-    commonjs(),
-    babel(),
-    uglify()
-  ]
+    commonjs()
+  ].concat(minificationPlugins)
 }]

structs/Delete.js

@@ -2,15 +2,15 @@
  * @module structs
  */
 
-import { getStructReference } from '../utils/structReferences.mjs'
-import * as ID from '../utils/ID.mjs'
-import { stringifyID } from '../protocols/syncProtocol.mjs'
-import { writeStructToTransaction } from '../utils/Transaction.mjs'
-import * as decoding from '../lib/decoding.mjs'
-import * as encoding from '../lib/encoding.mjs'
-import { Item } from './Item.mjs' // eslint-disable-line
-import { Y } from '../utils/Y.mjs' // eslint-disable-line
-import { deleteItemRange } from '../utils/structManipulation.mjs'
+import { getStructReference } from '../utils/structReferences.js'
+import * as ID from '../utils/ID.js'
+import { writeStructToTransaction } from '../utils/structEncoding.js'
+import * as decoding from '../lib/decoding.js'
+import * as encoding from '../lib/encoding.js'
+// import { Item } from './Item.js' // eslint-disable-line
+// import { Y } from '../utils/Y.js' // eslint-disable-line
+import { deleteItemRange } from '../utils/structManipulation.js'
+import * as stringify from '../utils/structStringify.js'
 
 /**
  * @private
@@ -99,6 +99,6 @@ export class Delete {
    * @private
    */
   _logString () {
-    return `Delete - target: ${stringifyID(this._targetID)}, len: ${this._length}`
+    return `Delete - target: ${stringify.stringifyID(this._targetID)}, len: ${this._length}`
   }
 }