13.4.0

Amend typing of YEvent.changes, fixes #232

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: dmonad
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/nodejs.yml

@@ -0,0 +1,31 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [10.x, 12.x, 13.x]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm ci
+    - run: npm run lint
+    - run: npm run test-extensive
+      env:
+        CI: true

.jsdoc.json

@@ -17,10 +17,13 @@
     "useCollapsibles": true,
     "collapse": true,
     "resources": {
-      "yjs.dev": "Yjs website"
+      "yjs.dev": "Website",
+      "docs.yjs.dev": "Docs",
+      "discuss.yjs.dev": "Forum",
+      "https://gitter.im/Yjs/community": "Chat"
     },
     "logo": {
-      "url": "https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png",
+      "url": "https://yjs.dev/images/logo/yjs-512x512.png",
       "width": "162px",
       "height": "162px",
       "link": "/"
@@ -35,7 +38,7 @@
     ],
     "default": {
       "staticFiles": {
-          "include": ["examples/"]
+          "include": []
       }
     }
   },
@@ -44,7 +47,6 @@
     "encoding": "utf8",
     "private": false,
     "recurse": true,
-    "template": "./node_modules/tui-jsdoc-template",
-    "tutorials": "./examples"
+    "template": "./node_modules/tui-jsdoc-template"
   }
 }

INTERNALS.md

@@ -0,0 +1,179 @@
+# Yjs Internals
+
+This document roughly explains how Yjs works internally. There is a complete
+walkthrough of the Yjs codebase available as a recording:
+https://youtu.be/0l5XgnQ6rB4
+
+The Yjs CRDT algorithm is described in the [YATA
+paper](https://www.researchgate.net/publication/310212186_Near_Real-Time_Peer-to-Peer_Shared_Editing_on_Extensible_Data_Types)
+from 2016. For an algorithmic view of how it works, the paper is a reasonable
+place to start. There are a handful of small improvements implemented in Yjs
+which aren't described in the paper. The most notable is that items have an
+`originRight` as well as an `origin` property, which improves performance when
+many concurrent inserts happen after the same character.
+
+At it heart, Yjs is a list CRDT. Everything is squeezed into a list in order to
+reuse the CRDT resolution algorithm:
+
+- Arrays are easy - they're lists of arbitrary items.
+- Text is a list of characters, optionally punctuated by formatting markers and
+  embeds for rich text support. Several characters can be wrapped in a single
+linked list `Item` (this is also known as the compound representation of
+CRDTs). More information about this in [this blog
+article](https://blog.kevinjahns.de/are-crdts-suitable-for-shared-editing/).
+- Maps are lists of entries. The last inserted entry for each key is used, and
+  all other duplicates for each key are flagged as deleted.
+
+Each client is assigned a unique *clientID* property on first insert. This is a
+random 53-bit integer (53 bits because that fits in the javascript safe integer
+range).
+
+## List items
+
+Each item in a Yjs list is made up of two objects:
+
+- An `Item` (*src/structs/Item.js*). This is used to relate the item to other
+  adjacent items.
+- An object in the `AbstractType` heirachy (subclasses of
+  *src/types/AbstractType.js* - eg `YText`). This stores the actual content in
+the Yjs document.
+
+The item and type object pair have a 1-1 mapping. The item's `content` field
+references the AbstractType object and the AbstractType object's `_item` field
+references the item.
+
+Everything inserted in a Yjs document is given a unique ID, formed from a
+*ID(clientID, clock)* pair (also known as a [Lamport
+Timestamp](https://en.wikipedia.org/wiki/Lamport_timestamp)). The clock counts
+up from 0 with the first inserted character or item a client makes. This is
+similar to automerge's operation IDs, but note that the clock is only
+incremented by inserts. Deletes are handled in a very different way (see
+below).
+
+If a run of characters is inserted into a document (eg `"abc"`), the clock will
+be incremented for each character (eg 3 times here). But Yjs will only add a
+single `Item` into the list. This has no effect on the core CRDT algorithm, but
+the optimization dramatically decreases the number of javascript objects
+created during normal text editing. This optimization only applies if the
+characters share the same clientID, they're inserted in order, and all
+characters have either been deleted or all characters are not deleted. The item
+will be split if the run is interrupted for any reason (eg a character in the
+middle of the run is deleted).
+
+When an item is created, it stores a reference to the IDs of the preceeding and
+succeeding item. These are stored in the item's `origin` and `originRight`
+fields, respectively. These are used when peers concurrently insert at the same
+location in a document. Though quite rare in practice, Yjs needs to make sure
+the list items always resolve to the same order on all peers. The actual logic
+is relatively simple - its only a couple dozen lines of code and it lives in
+the `Item#integrate()` method. The YATA paper has much more detail on the this
+algorithm.
+
+### Item Storage
+
+The items themselves are stored in two data structures and a cache:
+
+- The items are stored in a tree of doubly-linked lists in *document order*.
+  Each item has `left` and `right` properties linking to its siblings in the
+document. Items also have a `parent` property to reference their parent in the
+document tree (null at the root). (And you can access an item's children, if
+any, through `item.content`).
+- All items are referenced in *insertion order* inside the struct store
+  (*src/utils/StructStore.js*). This references the list of items inserted by
+for each client, in chronological order. This is used to find an item in the
+tree with a given ID (using a binary search). It is also used to efficiently
+gather the operations a peer is missing during sync (more on this below).
+
+When a local insert happens, Yjs needs to map the insert position in the
+document (eg position 1000) to an ID. With just the linked list, this would
+require a slow O(n) linear scan of the list. But when editing a document, most
+inserts are either at the same position as the last insert, or nearby. To
+improve performance, Yjs stores a cache of the 10 most recently looked up
+insert positions in the document. This is consulted and updated when a position
+is looked up to improve performance in the average case. The cache is updated
+using a heuristic that is still changing (currently, it is updated when a new
+position significantly diverges from existing markers in the cache). Internally
+this is referred to as the skip list / fast search marker.
+
+### Deletions
+
+Deletions in Yjs are treated very differently from insertions. Insertions are
+implemented as a sequential operation based CRDT, but deletions are treated as
+a simpler state based CRDT.
+
+When an item has been deleted by any peer, at any point in history, it is
+flagged as deleted on the item. (Internally Yjs uses the `info` bitfield.) Yjs
+does not record metadata about a deletion:
+
+- No data is kept on *when* an item was deleted, or which user deleted it.
+- The struct store does not contain deletion records
+- The clientID's clock is not incremented
+
+If garbage collection is enabled in Yjs, when an object is deleted its content
+is discarded. If a deleted object contains children (eg a field is deleted in
+an object), the content is replaced with a `GC` object (*src/structs/GC.js*).
+This is a very lightweight structure - it only stores the length of the removed
+content.
+
+Yjs has some special logic to share which content in a document has been
+deleted:
+
+- When a delete happens, as well as marking the item, the deleted IDs are
+  listed locally within the transaction. (See below for more information about
+transactions.) When a transaction has been committed locally, the set of
+deleted items is appended to a transaction's update message.
+- A snapshot (a marked point in time in the Yjs history) is specified using
+  both the set of (clientID, clock) pairs *and* the set of all deleted item
+IDs. The deleted set is O(n), but because deletions usually happen in runs,
+this data set is usually tiny in practice. (The real world editing trace from
+the B4 benchmark document contains 182k inserts and 77k deleted characters. The
+deleted set size in a snapshot is only 4.5Kb).
+
+## Transactions
+
+All updates in Yjs happen within a *transaction*. (Defined in
+*src/utils/Transaction.js*.)
+
+The transaction collects a set of updates to the Yjs document to be applied on
+remote peers atomically. Once a transaction has been committed locally, it
+generates a compressed *update message* which is broadcast to synchronized
+remote peers to notify them of the local change. The update message contains:
+
+- The set of newly inserted items
+- The set of items deleted within the transaction.
+
+## Network protocol
+
+The network protocol is not really a part of Yjs. There are a few relevant
+concepts that can be used to create a custom network protocol:
+
+* `update`: The Yjs document can be encoded to an *update* object that can be
+  parsed to reconstruct the document. Also every change on the document fires
+an incremental document updates that allows clients to sync with each other.
+The update object is an Uint8Array that efficiently encodes `Item` objects and
+the delete set.
+* `state vector`: A state vector defines the know state of each user (a set of
+  tubles `(client, clock)`). This object is also efficiently encoded as a
+Uint8Array.
+
+The client can ask a remote client for missing document updates by sending
+their state vector (often referred to as *sync step 1*). The remote peer can
+compute the missing `Item` objects using the `clocks` of the respective clients
+and compute a minimal update message that reflects all missing updates (sync
+step 2).
+
+An implementation of the syncing process is in
+[y-protocols](https://github.com/yjs/y-protocols).
+
+## Snapshots
+
+A snapshot can be used to restore an old document state. It is a `state vector`
++ `delete set`. I client can restore an old document state by iterating through
+the sequence CRDT and ignoring all Items that have an `id.clock >
+stateVector[id.client].clock`. Instead of using `item.deleted` the client will
+use the delete set to find out if an item was deleted or not.
+
+It is not recommended to restore an old document state using snapshots,
+although that would certainly be possible. Instead, the old state should be
+computed by iterating through the newest state and using the additional
+information from the state vector.

README.md

@@ -15,13 +15,55 @@ suited for even large documents.
 
 * Demos: [https://github.com/yjs/yjs-demos](https://github.com/yjs/yjs-demos)
 * Discuss: [https://discuss.yjs.dev](https://discuss.yjs.dev)
-* Benchmarks:
+* Benchmark Yjs vs. Automerge:
   [https://github.com/dmonad/crdt-benchmarks](https://github.com/dmonad/crdt-benchmarks)
 * Podcast [**"Yjs Deep Dive into real time collaborative editing solutions":**](https://www.tag1consulting.com/blog/deep-dive-real-time-collaborative-editing-solutions-tagteamtalk-001-0)
 * Podcast [**"Google Docs-style editing in Gutenberg with the YJS framework":**](https://publishpress.com/blog/yjs/)
 
-:construction_worker_woman: If you are looking for professional support to build
-collaborative or distributed applications ping us at <yjs@tag1consulting.com>.
+:construction_worker_woman: If you are looking for professional (paid) support to
+build collaborative or distributed applications ping us at
+<yjs@tag1consulting.com>. Otherwise you can find help on our
+[discussion board](https://discuss.yjs.dev).
+
+## Sponsors
+
+I'm currently looking for sponsors that allow me to be less dependent on
+contracting work. These awesome backers already fund further development of
+Yjs:
+
+[![Vincent Waller](https://github.com/vwall.png?size=60)](https://github.com/vwall)
+[<img src="https://user-images.githubusercontent.com/5553757/83337333-a7bcb380-a2ba-11ea-837b-e404eb35d318.png"
+height="60px" />](https://input.com/)
+[![Duane Johnson](https://github.com/canadaduane.png?size=60)](https://github.com/canadaduane)
+[![Joe Reeve](https://github.com/ISNIT0.png?size=60)](https://github.com/ISNIT0)
+[<img src="https://room.sh/img/icons/android-chrome-192x192.png" height="60px" />](https://room.sh/)
+[![JourneyApps](https://github.com/journeyapps.png?size=60)](https://github.com/journeyapps)
+[![Adam Brunnmeier](https://github.com/adabru.png?size=60)](https://github.com/adabru)
+[![Nathanael Anderson](https://github.com/NathanaelA.png?size=60)](https://github.com/NathanaelA)
+[![Gremloon](https://github.com/gremloon.png?size=60)](https://github.com/gremloon)
+[![ifiokjr](https://github.com/ifiokjr.png?size=60)](https://github.com/ifiokjr)
+[![mrfambo](https://github.com/mrfambo.png?size=60)](https://github.com/mrfambo)
+
+Sponsorship also comes with special perks! [![Become a Sponsor](https://img.shields.io/static/v1?label=Become%20a%20Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=d42f2d)](https://github.com/sponsors/dmonad)
+
+## Who is using Yjs
+
+* [Relm](http://www.relm.us/) A collaborative gameworld for teamwork and
+  community. :star2:
+* [Input](https://input.com/) A collaborative note taking app. :star2:
+* [Room.sh](https://room.sh/) A meeting application with integrated
+  collaborative drawing, editing, and coding tools. :star:
+* [http://coronavirustechhandbook.com/](https://coronavirustechhandbook.com/)
+  A collaborative wiki that is edited by thousands of different people to work
+  on a rapid and sophisticated response to the coronavirus outbreak and
+  subsequent impacts. :star:
+* [Nimbus Note](https://nimbusweb.me/note.php) A note-taking app designed by
+  Nimbus Web.
+* [JoeDocs](https://joedocs.com/) An open collaborative wiki.
+* [Pluxbox RadioManager](https://pluxbox.com/) A web-based app to
+  collaboratively organize radio broadcasts.
+* [Cattaz](http://cattaz.io/) A wiki that can run custom applications in the
+  wiki pages.
 
 ## Table of Contents
 
@@ -35,15 +77,7 @@ collaborative or distributed applications ping us at <yjs@tag1consulting.com>.
   * [Document Updates](#Document-Updates)
   * [Relative Positions](#Relative-Positions)
   * [Y.UndoManager](#YUndoManager)
-* [Miscellaneous](#Miscellaneous)
-  * [Typescript Declarations](#Typescript-Declarations)
 * [Yjs CRDT Algorithm](#Yjs-CRDT-Algorithm)
-* [Evaluation](#Evaluation)
-  * [Existing shared editing libraries](#Exisisting-Javascript-Libraries)
-  * [CRDT Algorithms](#CRDT-Algorithms)
-  * [Comparison of CRDT with OT](#Comparing-CRDT-with-OT)
-  * [Comparison of CRDT Algorithms](#Comparing-CRDT-Algorithms)
-  * [Comparison of Yjs with other Implementations](#Comparing-Yjs-with-other-Implementations)
 * [License and Author](#License-and-Author)
 
 ## Overview
@@ -56,13 +90,10 @@ are implemented in separate modules.
 
 | Name | Cursors | Binding |  Demo |
 |---|:-:|---|---|
-| [ProseMirror](https://prosemirror.net/) &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; | ✔ | [y-prosemirror](http://github.com/yjs/y-prosemirror) | [demo](https://yjs-demos.now.sh/prosemirror/) |
-| [Quill](https://quilljs.com/) | ✔ | [y-quill](http://github.com/yjs/y-quill) | [demo](https://yjs-demos.now.sh/quill/) |
-| [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/yjs/y-codemirror) | [demo](https://yjs-demos.now.sh/codemirror/) |
-| [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/yjs/y-monaco) | [demo](https://yjs-demos.now.sh/monaco/) |
-| [Ace](https://ace.c9.io/) | | [y-ace](http://github.com/yjs/y-ace) | [demo](https://yjs-demos.now.sh/ace/) |
-| [Textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) | | [y-textarea](http://github.com/yjs/y-textarea) | [demo](https://yjs-demos.now.sh/textarea/) |
-| [DOM](https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model) | | [y-dom](http://github.com/yjs/y-dom) | [demo](https://yjs-demos.now.sh/dom/) |
+| [ProseMirror](https://prosemirror.net/) &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; | ✔ | [y-prosemirror](http://github.com/yjs/y-prosemirror) | [demo](https://demos.yjs.dev/prosemirror/prosemirror.html) |
+| [Quill](https://quilljs.com/) | ✔ | [y-quill](http://github.com/yjs/y-quill) | [demo](https://demos.yjs.dev/quill/quill.html) |
+| [CodeMirror](https://codemirror.net/) | ✔ | [y-codemirror](http://github.com/yjs/y-codemirror) | [demo](https://demos.yjs.dev/codemirror/codemirror.html) |
+| [Monaco](https://microsoft.github.io/monaco-editor/) | ✔ | [y-monaco](http://github.com/yjs/y-monaco) | [demo](https://demos.yjs.dev/monaco/monaco.html) |
 
 ### Providers
 
@@ -85,6 +116,12 @@ document private.
 A module that contains a simple websocket backend and a websocket client that
 connects to that backend. The backend can be extended to persist updates in a
 leveldb database.
+  </dd>
+  <dt><a href="http://github.com/yjs/y-indexeddb">y-indexeddb</a></dt>
+  <dd>
+Efficiently persists document updates to the browsers indexeddb database.
+The document is immediately available and only diffs need to be synced through the
+network provider.
   </dd>
   <dt><a href="http://github.com/yjs/y-dat">y-dat</a></dt>
   <dd>
@@ -138,6 +175,54 @@ Now you understand how types are defined on a shared document. Next you can jump
 to the [demo repository](https://github.com/yjs/yjs-demos) or continue reading
 the API docs.
 
+### Example: Using and combining providers
+
+Any of the Yjs providers can be combined with each other. So you can sync data
+over different network technologies.
+
+In most cases you want to use a network provider (like y-websocket or y-webrtc)
+in combination with a persistence provider (y-indexeddb in the browser).
+Persistence allows you to load the document faster and to persist data that is
+created while offline.
+
+For the sake of this demo we combine two different network providers with a
+persistence provider.
+
+```js
+import * as Y from 'yjs'
+import { WebrtcProvider } from 'y-webrtc'
+import { WebsocketProvider } from 'y-websocket'
+import { IndexeddbPersistence } from 'y-indexeddb'
+
+const ydoc = new Y.Doc()
+
+// this allows you to instantly get the (cached) documents data
+const indexeddbProvider = new IndexeddbPersistence('count-demo', ydoc)
+indexeddbProvider.whenSynced.then(() => {
+  console.log('loaded data from indexed db')
+})
+
+// Sync clients with the y-webrtc provider.
+const webrtcProvider = new WebrtcProvider('count-demo', ydoc)
+
+// Sync clients with the y-websocket provider
+const websocketProvider = new WebsocketProvider(
+  'wss://demos.yjs.dev', 'count-demo', ydoc
+)
+
+// array of numbers which produce a sum
+const yarray = ydoc.getArray('count')
+
+// observe changes of the sum
+yarray.observe(event => {
+  // print updates when the data changes
+  console.log('new sum: ' + yarray.toArray().reduce((a,b) => a + b))
+})
+
+// add 1 to the sum
+yarray.push([1]) // => "new sum: 1"
+```
+
 ## API
 
 ```js
@@ -159,11 +244,13 @@ necessary.
     <b><code>insert(index:number, content:Array&lt;object|boolean|Array|string|number|Uint8Array|Y.Type&gt;)</code></b>
     <dd>
 Insert content at <var>index</var>. Note that content is an array of elements.
-I.e. <code>array.insert(0, [1]</code> splices the list and inserts 1 at
+I.e. <code>array.insert(0, [1])</code> splices the list and inserts 1 at
 position 0.
     </dd>
     <b><code>push(Array&lt;Object|boolean|Array|string|number|Uint8Array|Y.Type&gt;)</code></b>
     <dd></dd>
+    <b><code>unshift(Array&lt;Object|boolean|Array|string|number|Uint8Array|Y.Type&gt;)</code></b>
+    <dd></dd>
     <b><code>delete(index:number, length:number)</code></b>
     <dd></dd>
     <b><code>get(index:number)</code></b>
@@ -309,8 +396,12 @@ YTextEvents compute changes as deltas.
     <dd></dd>
     <b><code>format(index:number, length:number, formattingAttributes:Object&lt;string,string&gt;)</code></b>
     <dd>Assign formatting attributes to a range in the text</dd>
-    <b><code>applyDelta(delta)</code></b>
-    <dd>See <a href="https://quilljs.com/docs/delta/">Quill Delta</a></dd>
+    <b><code>applyDelta(delta, opts:Object&lt;string,any&gt;)</code></b>
+    <dd>
+        See <a href="https://quilljs.com/docs/delta/">Quill Delta</a>
+        Can set options for preventing remove ending newLines, default is true.
+        <pre>ytext.applyDelta(delta, { sanitize: false })</pre>
+    </dd>
     <b><code>length:number</code></b>
     <dd></dd>
     <b><code>toString():string</code></b>
@@ -464,6 +555,12 @@ const doc = new Y.Doc()
 <dl>
   <b><code>clientID</code></b>
   <dd>A unique id that identifies this client. (readonly)</dd>
+  <b><code>gc</code></b>
+  <dd>
+Whether garbage collection is enabled on this doc instance. Set `doc.gc = false`
+in order to disable gc and be able to restore old content. See https://github.com/yjs/yjs#yjs-crdt-algorithm
+for more information about gc in Yjs.
+  </dd>
   <b><code>transact(function(Transaction):void [, origin:any])</code></b>
   <dd>
 Every change on the shared document happens in a transaction. Observer calls and
@@ -473,6 +570,10 @@ calls. I.e. <code>doc.transact(() => { yarray.insert(..); ymap.set(..) })</code>
 triggers a single change event. <br>You can specify an optional <code>origin</code>
 parameter that is stored on <code>transaction.origin</code> and
 <code>on('update', (update, origin) => ..)</code>.
+  </dd>
+  <b><code>toJSON():any</code></b>
+  <dd>
+Converts the entire document into a js object, recursively traversing each yjs type.
   </dd>
   <b><code>get(string, Y.[TypeClass]):[Type]</code></b>
   <dd>Define a shared type.</dd>
@@ -500,6 +601,13 @@ peers. You can apply document updates in any order and multiple times.
   <dd>Emitted before each transaction.</dd>
   <b><code>on('afterTransaction', function(Y.Transaction, Y.Doc):void)</code></b>
   <dd>Emitted after each transaction.</dd>
+  <b><code>on('beforeAllTransactions', function(Y.Doc):void)</code></b>
+  <dd>
+Transactions can be nested (e.g. when an event within a transaction calls another
+transaction). Emitted before the first transaction.
+  </dd>
+  <b><code>on('afterAllTransactions', function(Y.Doc, Array&lt;Y.Transaction&gt;):void)</code></b>
+  <dd>Emitted after the last transaction is cleaned up.</dd>
 </dl>
 
 ### Document Updates
@@ -640,7 +748,7 @@ Yjs ships with an Undo/Redo manager for selective undo/redo of of changes on a
 Yjs type. The changes can be optionally scoped to transaction origins.
 
 ```js
-const ytext = doc.getArray('array')
+const ytext = doc.getText('text')
 const undoManager = new Y.UndoManager(ytext)
 
 ytext.insert(0, 'abc')
@@ -651,8 +759,8 @@ ytext.toString() // => 'abc'
 ```
 
 <dl>
-  <b><code>constructor(scope:Y.AbstractType|Array&lt;Y.AbstractType&gt;,
-  [[{captureTimeout:number,trackedOrigins:Set&lt;any&gt;,deleteFilter:function(item):boolean}]])</code></b>
+  <b><code>constructor(scope:Y.AbstractType|Array&lt;Y.AbstractType&gt;
+  [, {captureTimeout:number,trackedOrigins:Set&lt;any&gt;,deleteFilter:function(item):boolean}])</code></b>
   <dd>Accepts either single type as scope or an array of types.</dd>
   <b><code>undo()</code></b>
   <dd></dd>
@@ -692,28 +800,30 @@ StackItem won't be merged.
 // without stopCapturing
 ytext.insert(0, 'a')
 ytext.insert(1, 'b')
-um.undo()
+undoManager.undo()
 ytext.toString() // => '' (note that 'ab' was removed)
 // with stopCapturing
 ytext.insert(0, 'a')
-um.stopCapturing()
+undoManager.stopCapturing()
 ytext.insert(0, 'b')
-um.undo()
+undoManager.undo()
 ytext.toString() // => 'a' (note that only 'b' was removed)
 ```
 
 #### Example: Specify tracked origins
 
 Every change on the shared document has an origin. If no origin was specified,
-it defaults to `null`. By specifying `trackedTransactionOrigins` you can
+it defaults to `null`. By specifying `trackedOrigins` you can
 selectively specify which changes should be tracked by `UndoManager`. The
-UndoManager instance is always added to `trackedTransactionOrigins`.
+UndoManager instance is always added to `trackedOrigins`.
 
 ```js
 class CustomBinding {}
 
-const ytext = doc.getArray('array')
-const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding]))
+const ytext = doc.getText('text')
+const undoManager = new Y.UndoManager(ytext, {
+  trackedOrigins: new Set([42, CustomBinding])
+})
 
 ytext.insert(0, 'abc')
 undoManager.undo()
@@ -750,8 +860,10 @@ additional meta information like the cursor location or the view on the
 document. You can assign meta-information to Undo-/Redo-StackItems.
 
 ```js
-const ytext = doc.getArray('array')
-const undoManager = new Y.UndoManager(ytext, new Set([42, CustomBinding]))
+const ytext = doc.getText('text')
+const undoManager = new Y.UndoManager(ytext, {
+  trackedOrigins: new Set([42, CustomBinding])
+})
 
 undoManager.on('stack-item-added', event => {
   // save the current cursor location on the stack-item
@@ -764,24 +876,6 @@ undoManager.on('stack-item-popped', event => {
 })
 ```
 
-## Miscellaneous
-
-### Typescript Declarations
-
-Yjs has type descriptions. But until [this
-ticket](https://github.com/Microsoft/TypeScript/issues/7546) is fixed, this is
-how you can make use of Yjs type declarations.
-
-```json
-{
-  "compilerOptions": {
-    "allowJs": true,
-    "checkJs": true,
-  },
-  "maxNodeModuleJsDepth": 5
-}
-```
-
 ## Yjs CRDT Algorithm
 
 *Conflict-free replicated data types* (CRDT) for collaborative editing are an
@@ -798,11 +892,14 @@ do not require a central source of truth.
 
 Yjs implements a modified version of the algorithm described in [this
 paper](https://www.researchgate.net/publication/310212186_Near_Real-Time_Peer-to-Peer_Shared_Editing_on_Extensible_Data_Types).
-I will eventually publish a paper that describes why this approach works so well
-in practice. Note: Since operations make up the document structure, we prefer
-the term *struct* now.
+This [article](https://blog.kevinjahns.de/are-crdts-suitable-for-shared-editing/)
+explains a simple optimization on the CRDT model and
+gives more insight about the performance characteristics in Yjs.
+More information about the specific implementation is available in
+[INTERNALS.md](./INTERNALS.md) and in
+[this walkthrough of the Yjs codebase](https://youtu.be/0l5XgnQ6rB4).
 
-CRDTs suitable for shared text editing suffer from the fact that they only grow
+CRDTs that suitable for shared text editing suffer from the fact that they only grow
 in size. There are CRDTs that do not grow in size, but they do not have the
 characteristics that are benificial for shared text editing (like intention
 preservation). Yjs implements many improvements to the original algorithm that
@@ -855,5 +952,6 @@ Yjs and all related projects are [**MIT licensed**](./LICENSE).
 Yjs is based on my research as a student at the [RWTH
 i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time.
 
-Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or
-hiring [me](https://github.com/dmonad) for professional support.
+Fund this project by donating on [GitHub Sponsors](https://github.com/sponsors/dmonad)
+or hiring [me](https://github.com/dmonad) as a contractor for your collaborative
+app.

README.v12.md

@@ -1,305 +0,0 @@
-
-# ![Yjs](https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png)
-
-Yjs is a framework for offline-first p2p shared editing on structured data like
-text, richtext, json, or XML. It is fairly easy to get started, as Yjs hides
-most of the complexity of concurrent editing. For additional information, demos,
-and tutorials visit [y-js.org](http://y-js.org/).
-
-:warning: Checkout the [v13 docs](./README.md) for the upcoming release :warning:
-
-### Extensions
-Yjs only knows how to resolve conflicts on shared data. You have to choose a ..
-
-* *Connector* - a communication protocol that propagates changes to the clients
-* *Database* - a database to store your changes
-* one or more *Types* - that represent the shared data
-
-Connectors, Databases, and Types are available as modules that extend Yjs. Here
-is a list of the modules we know of:
-
-##### Connectors
-
-|Name            | Description               |
-|----------------|-----------------------------------|
-|[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC|
-|[websockets](https://github.com/y-js/y-websockets-client) | Set up [a central server](https://github.com/y-js/y-websockets-client), and connect to it via websockets |
-|[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))|
-|[ipfs](https://github.com/ipfs-labs/y-ipfs-connector) | Connector for the [Interplanetary File System](https://ipfs.io/)!|
-|[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios|
-
-##### Database adapters
-
-|Name            | Description               |
-|----------------|-----------------------------------|
-|[memory](https://github.com/y-js/y-memory) | In-memory storage. |
-|[indexeddb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser |
-|[leveldb](https://github.com/y-js/y-leveldb) | Persistent storage for node apps |
-
-##### Types
-
-| Name     | Description       |
-|----------|-------------------|
-|[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object |
-|[array](https://github.com/y-js/y-array) | A shared Array implementation |
-|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects |
-|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to the [Ace Editor](https://ace.c9.io), [CodeMirror](https://codemirror.net/), [Monaco](https://github.com/Microsoft/monaco-editor), textareas, input elements, and HTML elements (e.g. <*h1*>, or <*p*>) |
-|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)|
-
-##### Other
-
-| Name      | Description       |
-|-----------|-------------------|
-|[y-element](http://y-js.org/y-element/) | Yjs Polymer Element |
-
-## Use it!
-Install Yjs, and its modules with [bower](http://bower.io/), or
-[npm](https://www.npmjs.org/package/yjs).
-
-### Bower
-
-```
-bower install --save yjs y-array % add all y-* modules you want to use
-```
-You only need to include the `y.js` file. Yjs is able to automatically require
-missing modules.  
-```
-<script src="./bower_components/yjs/y.js"></script>
-```
-
-### CDN
-
-```
-<script src="https://cdn.jsdelivr.net/npm/yjs@12/dist/y.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/y-array@10/dist/y-array.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/y-websockets-client@8/dist/y-websockets-client.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/y-memory@8/dist/y-memory.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/y-map@10/dist/y-map.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/y-text@9/dist/y-text.js"></script>
-// ..
-// do the same for all modules you want to use
-```
-
-### Npm
-
-```
-npm install --save yjs % add all y-* modules you want to use
-```
-
-If you don't include via script tag, you have to explicitly include all modules!
-(Same goes for other module systems)
-```
-var Y = require('yjs')
-require('y-array')(Y) // add the y-array type to Yjs
-require('y-websockets-client')(Y)
-require('y-memory')(Y)
-require('y-map')(Y)
-require('y-text')(Y)
-// ..
-// do the same for all modules you want to use
-```
-
-### ES6 Syntax
-
-```
-import Y from 'yjs'
-import yArray from 'y-array'
-import yWebsocketsClient from 'y-webrtc'
-import yMemory from 'y-memory'
-import yMap from 'y-map'
-import yText from 'y-text'
-// ..
-Y.extend(yArray, yWebsocketsClient, yMemory, yArray, yMap, yText /*, .. */)
-```
-
-# Text editing example
-
-Install dependencies
-```
-bower i yjs y-memory y-webrtc y-array y-text
-```
-
-Here is a simple example of a shared textarea
-```HTML
-  <!DOCTYPE html>
-  <html>
-    <body>
-      <script src="./bower_components/yjs/y.js"></script>
-      <!-- Yjs automatically includes all missing dependencies (browser only) -->
-      <script>
-        Y({
-          db: {
-            name: 'memory' // use memory database adapter.
-            // name: 'indexeddb' // use indexeddb database adapter instead for offline apps
-          },
-          connector: {
-            name: 'webrtc', // use webrtc connector
-            // name: 'websockets-client'
-            // name: 'xmpp'
-            room: 'my-room' // clients connecting to the same room share data
-          },
-          sourceDir: './bower_components', // location of the y-* modules (browser only)
-          share: {
-            textarea: 'Text' // y.share.textarea is of type y-text
-          }
-        }).then(function (y) {
-          // The Yjs instance `y` is available
-          // y.share.* contains the shared types
-
-          // Bind `y.share.textarea` to `<textarea/>`
-          y.share.textarea.bind(document.querySelector('textarea'))
-        })
-      </script>
-      <textarea></textarea>
-    </body>
-  </html>
-```
-
-## Get Help & Give Help
-There are some friendly people on [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/y-js/yjs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) who are eager to help, and answer questions. Please join!
-
-Report _any_ issues to the
-[Github issue page](https://github.com/y-js/yjs/issues)! I try to fix them very
-soon, if possible.
-
-# API
-
-### Y(options)
-* Y.extend(module1, module2, ..)
-  * Add extensions to Y
-  * `Y.extend(require('y-webrtc'))` has the same semantics as
-    `require('y-webrtc')(Y)`
-* options.db
-  * Will be forwarded to the database adapter. Specify the database adaper on
-    `options.db.name`.
-  * Have a look at the used database adapter repository to see all available
-    options.
-* options.connector
-  * Will be forwarded to the connector adapter. Specify the connector adaper on
-    `options.connector.name`.
-  * All our connectors implement a `room` property. Clients that specify the
-    same room share the same data.
-  * All of our connectors specify an `url` property that defines the connection
-    endpoint of the used connector.
-    * All of our connectors also have a default connection endpoint that you can
-      use for development.
-  * Set `options.connector.generateUserId = true` in order to genenerate a
-    userid, instead of receiving one from the server. This way the `Y(..)` is
-    immediately going to be resolved, without waiting for any confirmation from
-    the server. Use with caution.
-  * Have a look at the used connector repository to see all available options.
-  * *Only if you know what you are doing:* Set
-    `options.connector.preferUntransformed = true` in order receive the shared
-    data untransformed. This is very efficient as the database content is simply
-    copied to this client. This does only work if this client receives content
-    from only one client.
-* options.sourceDir (browser only)
-  * Path where all y-* modules are stored
-  * Defaults to `/bower_components`
-  * Not required when running on `nodejs` / `iojs`
-  * When using nodejs you need to manually extend Yjs:
-```
-var Y = require('yjs')
-// you have to require a db, connector, and *all* types you use!
-require('y-memory')(Y)
-require('y-webrtc')(Y)
-require('y-map')(Y)
-// ..
-```
-* options.share
-  * Specify on `options.share[arbitraryName]` types that are shared among all
-    users.
-  * E.g. Specify `options.share[arbitraryName] = 'Array'` to require y-array and
-    create an y-array type on `y.share[arbitraryName]`.
-  * If userA doesn't specify `options.share[arbitraryName]`, it won't be
-    available for userA.
-  * If userB specifies `options.share[arbitraryName]`, it still won't be
-    available for userA. But all the updates are send from userB to userA.
-  * In contrast to y-map, types on `y.share.*` cannot be overwritten or deleted.
-    Instead, they are merged among all users. This feature is only available on
-    `y.share.*`
-  * Weird behavior: It is supported that two users specify different types with
-    the same property name.
-     E.g. userA specifies `options.share.x = 'Array'`, and userB specifies
-     `options.share.x = 'Text'`. But they only share data if they specified the
-     same type with the same property name
-* options.type (browser only)
-  * Array of modules that Yjs needs to require, before instantiating a shared
-    type.
-  * By default Yjs requires the specified database adapter, the specified
-    connector, and all modules that are used in `options.share.*`
-  * Put all types here that you intend to use, but are not used in y.share.*
-
-### Instantiated Y object (y)
-`Y(options)` returns a promise that is fulfilled when..
-
-* All modules are loaded
-  * The specified database adapter is loaded
-  * The specified connector is loaded
-  * All types are included
-* The connector is initialized, and a unique user id is set (received from the
-  server)
-  * Note: When using y-indexeddb, a retrieved user id is stored on `localStorage`
-
-The promise returns an instance of Y. We denote it with a lower case `y`.
-
-* y.share.*
-  * Instances of the types you specified on options.share.*
-  * y.share.* can only be defined once when you instantiate Y!
-* y.connector is an instance of Y.AbstractConnector
-* y.connector.onUserEvent(function (event) {..})
-  * Observe user events (event.action is either 'userLeft' or 'userJoined')
-* y.connector.whenSynced(listener)
-  * `listener` is executed when y synced with at least one user.
-  * `listener` is not called when no other user is in the same room.
-  * y-websockets-client aways waits to sync with the server
-* y.connector.disconnect()
-  * Force to disconnect this instance from the other instances
-* y.connector.connect()
-  * Try to reconnect to the other instances (needs to be supported by the
-    connector)
-  * Not supported by y-xmpp
-* y.close()
-  * Destroy this object.
-  * Destroys all types (they will throw weird errors if you still use them)
-  * Disconnects from the other instances (via connector)
-  * Returns a promise
-* y.destroy()
-  * calls y.close()
-  * Removes all data from the database
-  * Returns a promise
-* y.db.stopGarbageCollector()
-  * Stop the garbage collector. Call y.db.garbageCollect() to continue garbage
-    collection
-* y.db.gc :: Boolean
-  * Whether gc is turned on
-* y.db.gcTimeout :: Number (defaults to 50000 ms)
-  * Time interval between two garbage collect cycles
-  * It is required that all instances exchanged all messages after two garbage
-    collect cycles (after 100000 ms per default)
-* y.db.userId :: String
-  * The used user id for this client. **Never overwrite this**
-
-### Logging
-Yjs uses [debug](https://github.com/visionmedia/debug) for logging. The flag
-`y*` enables logging for all y-* components. You can selectively remove
-components you are not interested in: E.g. The flag `y*,-y:connector-message`
-will not log the long `y:connector-message` messages.
-
-##### Enable logging in Node.js
-```sh
-DEBUG=y* node app.js
-```
-
-Remove the colors in order to log to a file:
-```sh
-DEBUG_COLORS=0 DEBUG=y* node app.js > log
-```
-
-##### Enable logging in the browser
-```js
-localStorage.debug = 'y*'
-```
-
-## License
-Yjs is licensed under the [MIT License](./LICENSE).

package.json

@@ -1,21 +1,25 @@
 {
   "name": "yjs",
-  "version": "13.0.0",
+  "version": "13.4.0",
   "description": "Shared Editing Library",
   "main": "./dist/yjs.cjs",
-  "module": "./src/index.js",
+  "module": "./dist/yjs.mjs",
+  "unpkg": "./dist/yjs.mjs",
   "types": "./dist/src/index.d.ts",
   "sideEffects": false,
+  "funding": {
+    "type": "GitHub Sponsors ❤",
+    "url": "https://github.com/sponsors/dmonad"
+  },
   "scripts": {
     "test": "npm run dist && node ./dist/tests.cjs --repitition-time 50",
-    "test-exhaustive": "npm run lint && npm run dist && node ./dist/tests.cjs --production --repitition-time 10000",
+    "test-extensive": "npm run lint && npm run dist && node ./dist/tests.cjs --production --repitition-time 10000",
     "dist": "rm -rf dist && rollup -c && tsc",
     "watch": "rollup -wc",
     "lint": "markdownlint README.md && standard && tsc",
     "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true",
     "serve-docs": "npm run docs && http-server ./docs/",
-    "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000",
-    "postversion": "git push && git push --tags",
+    "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000 && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs",
     "debug": "concurrently 'http-server -o test.html' 'npm run watch'",
     "trace-deopt": "clear && rollup -c  && node --trace-deopt dist/test.cjs",
     "trace-opt": "clear && rollup -c  && node --trace-opt dist/test.cjs"
@@ -57,20 +61,20 @@
   },
   "homepage": "https://yjs.dev",
   "dependencies": {
-    "lib0": "^0.2.7"
+    "lib0": "^0.2.33"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^11.0.1",
-    "@rollup/plugin-node-resolve": "^7.0.0",
+    "@rollup/plugin-commonjs": "^11.1.0",
+    "@rollup/plugin-node-resolve": "^7.1.3",
     "concurrently": "^3.6.1",
-    "http-server": "^0.12.1",
-    "jsdoc": "^3.6.3",
-    "markdownlint-cli": "^0.19.0",
-    "rollup": "^1.29.1",
+    "http-server": "^0.12.3",
+    "jsdoc": "^3.6.5",
+    "markdownlint-cli": "^0.23.2",
+    "rollup": "^1.32.1",
     "rollup-cli": "^1.0.9",
-    "standard": "^14.0.0",
+    "standard": "^14.3.4",
     "tui-jsdoc-template": "^1.2.2",
-    "typescript": "^3.7.5",
-    "y-protocols": "^0.2.0"
+    "typescript": "^3.9.7",
+    "y-protocols": "^0.2.3"
   }
 }

rollup.config.js

@@ -51,6 +51,15 @@ export default [{
     }
   },
   external: id => /^lib0\//.test(id)
+}, {
+  input: './src/index.js',
+  output: {
+    name: 'Y',
+    file: 'dist/yjs.mjs',
+    format: 'esm',
+    sourcemap: true
+  },
+  external: id => /^lib0\//.test(id)
 }, {
   input: './tests/index.js',
   output: {
@@ -62,7 +71,6 @@ export default [{
   plugins: [
     debugResolve,
     nodeResolve({
-      sourcemap: true,
       mainFields: ['module', 'browser', 'main']
     }),
     commonjs()
@@ -78,7 +86,6 @@ export default [{
   plugins: [
     debugResolve,
     nodeResolve({
-      sourcemap: true,
       mainFields: ['module', 'main']
     }),
     commonjs()

src/index.js

@@ -12,6 +12,7 @@ export {
   YXmlEvent,
   YMapEvent,
   YArrayEvent,
+  YTextEvent,
   YEvent,
   Item,
   AbstractStruct,
@@ -26,6 +27,7 @@ export {
   ContentType,
   AbstractType,
   RelativePosition,
+  getTypeChildren,
   createRelativePositionFromTypeIndex,
   createRelativePositionFromJSON,
   createAbsolutePositionFromRelativePosition,
@@ -45,15 +47,29 @@ export {
   findRootTypeKey,
   typeListToArraySnapshot,
   typeMapGetSnapshot,
+  createDocFromSnapshot,
   iterateDeletedStructs,
   applyUpdate,
+  applyUpdateV2,
+  readUpdate,
+  readUpdateV2,
   encodeStateAsUpdate,
+  encodeStateAsUpdateV2,
   encodeStateVector,
+  encodeStateVectorV2,
   UndoManager,
   decodeSnapshot,
   encodeSnapshot,
+  decodeSnapshotV2,
+  encodeSnapshotV2,
+  decodeStateVector,
+  decodeStateVectorV2,
   isDeleted,
   isParentOf,
   equalSnapshots,
-  PermanentUserData // @TODO experimental
+  PermanentUserData, // @TODO experimental
+  tryGc,
+  transact,
+  AbstractConnector,
+  logType
 } from './internals.js'

src/internals.js

@@ -1,10 +1,14 @@
 
+export * from './utils/AbstractConnector.js'
 export * from './utils/DeleteSet.js'
 export * from './utils/Doc.js'
+export * from './utils/UpdateDecoder.js'
+export * from './utils/UpdateEncoder.js'
 export * from './utils/encoding.js'
 export * from './utils/EventHandler.js'
 export * from './utils/ID.js'
 export * from './utils/isParentOf.js'
+export * from './utils/logging.js'
 export * from './utils/PermanentUserData.js'
 export * from './utils/RelativePosition.js'
 export * from './utils/Snapshot.js'
@@ -27,6 +31,7 @@ export * from './structs/AbstractStruct.js'
 export * from './structs/GC.js'
 export * from './structs/ContentBinary.js'
 export * from './structs/ContentDeleted.js'
+export * from './structs/ContentDoc.js'
 export * from './structs/ContentEmbed.js'
 export * from './structs/ContentFormat.js'
 export * from './structs/ContentJSON.js'

src/structs/AbstractStruct.js

@@ -1,28 +1,25 @@
 
 import {
-  StructStore, ID, Transaction // eslint-disable-line
+  AbstractUpdateEncoder, ID, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js' // eslint-disable-line
 import * as error from 'lib0/error.js'
 
-/**
- * @private
- */
 export class AbstractStruct {
   /**
    * @param {ID} id
    * @param {number} length
    */
   constructor (id, length) {
-    /**
-     * The uniqe identifier of this struct.
-     * @type {ID}
-     * @readonly
-     */
     this.id = id
     this.length = length
-    this.deleted = false
+  }
+
+  /**
+   * @type {boolean}
+   */
+  get deleted () {
+    throw error.methodUnimplemented()
   }
 
   /**
@@ -37,10 +34,9 @@ export class AbstractStruct {
   }
 
   /**
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    * @param {number} offset
    * @param {number} encodingRef
-   * @private
    */
   write (encoder, offset, encodingRef) {
     throw error.methodUnimplemented()
@@ -48,46 +44,9 @@ export class AbstractStruct {
 
   /**
    * @param {Transaction} transaction
-   */
-  integrate (transaction) {
-    throw error.methodUnimplemented()
-  }
-}
-
-/**
- * @private
- */
-export class AbstractStructRef {
-  /**
-   * @param {ID} id
-   */
-  constructor (id) {
-    /**
-     * @type {Array<ID>}
-     */
-    this._missing = []
-    /**
-     * The uniqe identifier of this type.
-     * @type {ID}
-     */
-    this.id = id
-  }
-
-  /**
-   * @param {Transaction} transaction
-   * @return {Array<ID|null>}
-   */
-  getMissing (transaction) {
-    return this._missing
-  }
-
-  /**
-   * @param {Transaction} transaction
-   * @param {StructStore} store
    * @param {number} offset
-   * @return {AbstractStruct}
    */
-  toStruct (transaction, store, offset) {
+  integrate (transaction, offset) {
     throw error.methodUnimplemented()
   }
 }

src/structs/ContentAny.js

@@ -1,13 +1,7 @@
 import {
-  Transaction, Item, StructStore // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
-/**
- * @private
- */
 export class ContentAny {
   /**
    * @param {Array<any>} arr
@@ -80,15 +74,15 @@ export class ContentAny {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
     const len = this.arr.length
-    encoding.writeVarUint(encoder, len - offset)
+    encoder.writeLen(len - offset)
     for (let i = offset; i < len; i++) {
       const c = this.arr[i]
-      encoding.writeAny(encoder, c)
+      encoder.writeAny(c)
     }
   }
 
@@ -101,16 +95,14 @@ export class ContentAny {
 }
 
 /**
- * @private
- *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentAny}
  */
 export const readContentAny = decoder => {
-  const len = decoding.readVarUint(decoder)
+  const len = decoder.readLen()
   const cs = []
   for (let i = 0; i < len; i++) {
-    cs.push(decoding.readAny(decoder))
+    cs.push(decoder.readAny())
   }
   return new ContentAny(cs)
 }

src/structs/ContentBinary.js

@@ -1,15 +1,9 @@
 import {
-  StructStore, Item, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-import * as buffer from 'lib0/buffer.js'
 import * as error from 'lib0/error.js'
 
-/**
- * @private
- */
 export class ContentBinary {
   /**
    * @param {Uint8Array} content
@@ -76,11 +70,11 @@ export class ContentBinary {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarUint8Array(encoder, this.content)
+    encoder.writeBuf(this.content)
   }
 
   /**
@@ -92,9 +86,7 @@ export class ContentBinary {
 }
 
 /**
- * @private
- *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentBinary}
  */
-export const readContentBinary = decoder => new ContentBinary(buffer.copyUint8Array(decoding.readVarUint8Array(decoder)))
+export const readContentBinary = decoder => new ContentBinary(decoder.readBuf())

src/structs/ContentDeleted.js

@@ -1,15 +1,9 @@
 
 import {
   addToDeleteSet,
-  StructStore, Item, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
-/**
- * @private
- */
 export class ContentDeleted {
   /**
    * @param {number} len
@@ -70,8 +64,8 @@ export class ContentDeleted {
    * @param {Item} item
    */
   integrate (transaction, item) {
-    addToDeleteSet(transaction.deleteSet, item.id, this.len)
-    item.deleted = true
+    addToDeleteSet(transaction.deleteSet, item.id.client, item.id.clock, this.len)
+    item.markDeleted()
   }
 
   /**
@@ -83,11 +77,11 @@ export class ContentDeleted {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarUint(encoder, this.len - offset)
+    encoder.writeLen(this.len - offset)
   }
 
   /**
@@ -101,7 +95,7 @@ export class ContentDeleted {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentDeleted}
  */
-export const readContentDeleted = decoder => new ContentDeleted(decoding.readVarUint(decoder))
+export const readContentDeleted = decoder => new ContentDeleted(decoder.readLen())

src/structs/ContentDoc.js

@@ -0,0 +1,135 @@
+
+import {
+  Doc, AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Transaction, Item // eslint-disable-line
+} from '../internals.js'
+
+import * as error from 'lib0/error.js'
+
+/**
+ * @private
+ */
+export class ContentDoc {
+  /**
+   * @param {Doc} doc
+   */
+  constructor (doc) {
+    if (doc._item) {
+      console.error('This document was already integrated as a sub-document. You should create a second instance instead with the same guid.')
+    }
+    /**
+     * @type {Doc}
+     */
+    this.doc = doc
+    /**
+     * @type {any}
+     */
+    const opts = {}
+    this.opts = opts
+    if (!doc.gc) {
+      opts.gc = false
+    }
+    if (doc.autoLoad) {
+      opts.autoLoad = true
+    }
+    if (doc.meta !== null) {
+      opts.meta = doc.meta
+    }
+  }
+
+  /**
+   * @return {number}
+   */
+  getLength () {
+    return 1
+  }
+
+  /**
+   * @return {Array<any>}
+   */
+  getContent () {
+    return [this.doc]
+  }
+
+  /**
+   * @return {boolean}
+   */
+  isCountable () {
+    return true
+  }
+
+  /**
+   * @return {ContentDoc}
+   */
+  copy () {
+    return new ContentDoc(this.doc)
+  }
+
+  /**
+   * @param {number} offset
+   * @return {ContentDoc}
+   */
+  splice (offset) {
+    throw error.methodUnimplemented()
+  }
+
+  /**
+   * @param {ContentDoc} right
+   * @return {boolean}
+   */
+  mergeWith (right) {
+    return false
+  }
+
+  /**
+   * @param {Transaction} transaction
+   * @param {Item} item
+   */
+  integrate (transaction, item) {
+    // this needs to be reflected in doc.destroy as well
+    this.doc._item = item
+    transaction.subdocsAdded.add(this.doc)
+    if (this.doc.shouldLoad) {
+      transaction.subdocsLoaded.add(this.doc)
+    }
+  }
+
+  /**
+   * @param {Transaction} transaction
+   */
+  delete (transaction) {
+    if (transaction.subdocsAdded.has(this.doc)) {
+      transaction.subdocsAdded.delete(this.doc)
+    } else {
+      transaction.subdocsRemoved.add(this.doc)
+    }
+  }
+
+  /**
+   * @param {StructStore} store
+   */
+  gc (store) { }
+
+  /**
+   * @param {AbstractUpdateEncoder} encoder
+   * @param {number} offset
+   */
+  write (encoder, offset) {
+    encoder.writeString(this.doc.guid)
+    encoder.writeAny(this.opts)
+  }
+
+  /**
+   * @return {number}
+   */
+  getRef () {
+    return 9
+  }
+}
+
+/**
+ * @private
+ *
+ * @param {AbstractUpdateDecoder} decoder
+ * @return {ContentDoc}
+ */
+export const readContentDoc = decoder => new ContentDoc(new Doc({ guid: decoder.readString(), ...decoder.readAny() }))

src/structs/ContentEmbed.js

@@ -1,10 +1,8 @@
 
 import {
-  StructStore, Item, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 import * as error from 'lib0/error.js'
 
 /**
@@ -76,11 +74,11 @@ export class ContentEmbed {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarString(encoder, JSON.stringify(this.embed))
+    encoder.writeJSON(this.embed)
   }
 
   /**
@@ -94,7 +92,7 @@ export class ContentEmbed {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentEmbed}
  */
-export const readContentEmbed = decoder => new ContentEmbed(JSON.parse(decoding.readVarString(decoder)))
+export const readContentEmbed = decoder => new ContentEmbed(decoder.readJSON())

src/structs/ContentFormat.js

@@ -1,10 +1,8 @@
 
 import {
-  Item, StructStore, Transaction // eslint-disable-line
+  AbstractType, AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 import * as error from 'lib0/error.js'
 
 /**
@@ -68,7 +66,11 @@ export class ContentFormat {
    * @param {Transaction} transaction
    * @param {Item} item
    */
-  integrate (transaction, item) {}
+  integrate (transaction, item) {
+    // @todo searchmarker are currently unsupported for rich text documents
+    /** @type {AbstractType<any>} */ (item.parent)._searchMarker = null
+  }
+
   /**
    * @param {Transaction} transaction
    */
@@ -78,12 +80,12 @@ export class ContentFormat {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarString(encoder, this.key)
-    encoding.writeVarString(encoder, JSON.stringify(this.value))
+    encoder.writeKey(this.key)
+    encoder.writeJSON(this.value)
   }
 
   /**
@@ -95,9 +97,7 @@ export class ContentFormat {
 }
 
 /**
- * @private
- *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentFormat}
  */
-export const readContentFormat = decoder => new ContentFormat(decoding.readVarString(decoder), JSON.parse(decoding.readVarString(decoder)))
+export const readContentFormat = decoder => new ContentFormat(decoder.readString(), decoder.readJSON())

src/structs/ContentJSON.js

@@ -1,10 +1,7 @@
 import {
-  Transaction, Item, StructStore // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 /**
  * @private
  */
@@ -80,15 +77,15 @@ export class ContentJSON {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
     const len = this.arr.length
-    encoding.writeVarUint(encoder, len - offset)
+    encoder.writeLen(len - offset)
     for (let i = offset; i < len; i++) {
       const c = this.arr[i]
-      encoding.writeVarString(encoder, c === undefined ? 'undefined' : JSON.stringify(c))
+      encoder.writeString(c === undefined ? 'undefined' : JSON.stringify(c))
     }
   }
 
@@ -103,14 +100,14 @@ export class ContentJSON {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentJSON}
  */
 export const readContentJSON = decoder => {
-  const len = decoding.readVarUint(decoder)
+  const len = decoder.readLen()
   const cs = []
   for (let i = 0; i < len; i++) {
-    const c = decoding.readVarString(decoder)
+    const c = decoder.readString()
     if (c === 'undefined') {
       cs.push(undefined)
     } else {

src/structs/ContentString.js

@@ -1,10 +1,7 @@
 import {
-  Transaction, Item, StructStore // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 /**
  * @private
  */
@@ -80,11 +77,11 @@ export class ContentString {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarString(encoder, offset === 0 ? this.str : this.str.slice(offset))
+    encoder.writeString(offset === 0 ? this.str : this.str.slice(offset))
   }
 
   /**
@@ -98,7 +95,7 @@ export class ContentString {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentString}
  */
-export const readContentString = decoder => new ContentString(decoding.readVarString(decoder))
+export const readContentString = decoder => new ContentString(decoder.readString())

src/structs/ContentType.js

@@ -7,15 +7,13 @@ import {
   readYXmlFragment,
   readYXmlHook,
   readYXmlText,
-  StructStore, Transaction, Item, YEvent, AbstractType // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Transaction, Item, YEvent, AbstractType // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js' // eslint-disable-line
-import * as decoding from 'lib0/decoding.js'
 import * as error from 'lib0/error.js'
 
 /**
- * @type {Array<function(decoding.Decoder):AbstractType<any>>}
+ * @type {Array<function(AbstractUpdateDecoder):AbstractType<any>>}
  * @private
  */
 export const typeRefs = [
@@ -115,7 +113,7 @@ export class ContentType {
         // We try to merge all deleted items after each transaction,
         // but we have no knowledge about that this needs to be merged
         // since it is not in transaction.ds. Hence we add it to transaction._mergeStructs
-        transaction._mergeStructs.add(item.id)
+        transaction._mergeStructs.push(item)
       }
       item = item.right
     }
@@ -124,7 +122,7 @@ export class ContentType {
         item.delete(transaction)
       } else {
         // same as above
-        transaction._mergeStructs.add(item.id)
+        transaction._mergeStructs.push(item)
       }
     })
     transaction.changed.delete(this.type)
@@ -150,7 +148,7 @@ export class ContentType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
@@ -168,7 +166,7 @@ export class ContentType {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentType}
  */
-export const readContentType = decoder => new ContentType(typeRefs[decoding.readVarUint(decoder)](decoder))
+export const readContentType = decoder => new ContentType(typeRefs[decoder.readTypeRef()](decoder))

src/structs/GC.js

@@ -1,28 +1,18 @@
 
 import {
-  AbstractStructRef,
   AbstractStruct,
-  createID,
   addStruct,
-  StructStore, Transaction, ID // eslint-disable-line
+  AbstractUpdateEncoder, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
-import * as decoding from 'lib0/decoding.js'
-import * as encoding from 'lib0/encoding.js'
-
 export const structGCRefNumber = 0
 
 /**
  * @private
  */
 export class GC extends AbstractStruct {
-  /**
-   * @param {ID} id
-   * @param {number} length
-   */
-  constructor (id, length) {
-    super(id, length)
-    this.deleted = true
+  get deleted () {
+    return true
   }
 
   delete () {}
@@ -38,53 +28,31 @@ export class GC extends AbstractStruct {
 
   /**
    * @param {Transaction} transaction
+   * @param {number} offset
    */
-  integrate (transaction) {
+  integrate (transaction, offset) {
+    if (offset > 0) {
+      this.id.clock += offset
+      this.length -= offset
+    }
     addStruct(transaction.doc.store, this)
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeUint8(encoder, structGCRefNumber)
-    encoding.writeVarUint(encoder, this.length - offset)
-  }
-}
-
-/**
- * @private
- */
-export class GCRef extends AbstractStructRef {
-  /**
-   * @param {decoding.Decoder} decoder
-   * @param {ID} id
-   * @param {number} info
-   */
-  constructor (decoder, id, info) {
-    super(id)
-    /**
-     * @type {number}
-     */
-    this.length = decoding.readVarUint(decoder)
+    encoder.writeInfo(structGCRefNumber)
+    encoder.writeLen(this.length - offset)
   }
 
   /**
    * @param {Transaction} transaction
    * @param {StructStore} store
-   * @param {number} offset
-   * @return {GC}
+   * @return {null | number}
    */
-  toStruct (transaction, store, offset) {
-    if (offset > 0) {
-      // @ts-ignore
-      this.id = createID(this.id.client, this.id.clock + offset)
-      this.length -= offset
-    }
-    return new GC(
-      this.id,
-      this.length
-    )
+  getMissing (transaction, store) {
+    return null
   }
 }

src/structs/Item.js

@@ -1,11 +1,7 @@
 
 import {
-  readID,
-  createID,
-  writeID,
   GC,
-  nextID,
-  AbstractStructRef,
+  getState,
   AbstractStruct,
   replaceStruct,
   addStruct,
@@ -21,15 +17,15 @@ import {
   readContentAny,
   readContentString,
   readContentEmbed,
+  readContentDoc,
+  createID,
   readContentFormat,
   readContentType,
   addChangedTypeToTransaction,
-  ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as error from 'lib0/error.js'
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 import * as maplib from 'lib0/map.js'
 import * as set from 'lib0/set.js'
 import * as binary from 'lib0/binary.js'
@@ -68,11 +64,12 @@ export const followRedone = (store, id) => {
  * sending it to other peers
  *
  * @param {Item|null} item
+ * @param {boolean} keep
  */
-export const keepItem = item => {
-  while (item !== null && !item.keep) {
-    item.keep = true
-    item = item.parent._item
+export const keepItem = (item, keep) => {
+  while (item !== null && item.keep !== keep) {
+    item.keep = keep
+    item = /** @type {AbstractType<any>} */ (item.parent)._item
   }
 }
 
@@ -87,12 +84,12 @@ export const keepItem = item => {
  * @private
  */
 export const splitItem = (transaction, leftItem, diff) => {
-  const id = leftItem.id
   // create rightItem
+  const { client, clock } = leftItem.id
   const rightItem = new Item(
-    createID(id.client, id.clock + diff),
+    createID(client, clock + diff),
     leftItem,
-    createID(id.client, id.clock + diff - 1),
+    createID(client, clock + diff - 1),
     leftItem.right,
     leftItem.rightOrigin,
     leftItem.parent,
@@ -100,7 +97,7 @@ export const splitItem = (transaction, leftItem, diff) => {
     leftItem.content.splice(diff)
   )
   if (leftItem.deleted) {
-    rightItem.deleted = true
+    rightItem.markDeleted()
   }
   if (leftItem.keep) {
     rightItem.keep = true
@@ -115,10 +112,10 @@ export const splitItem = (transaction, leftItem, diff) => {
     rightItem.right.left = rightItem
   }
   // right is more specific.
-  transaction._mergeStructs.add(rightItem.id)
+  transaction._mergeStructs.push(rightItem)
   // update parent._map
   if (rightItem.parentSub !== null && rightItem.right === null) {
-    rightItem.parent._map.set(rightItem.parentSub, rightItem)
+    /** @type {AbstractType<any>} */ (rightItem.parent)._map.set(rightItem.parentSub, rightItem)
   }
   leftItem.length = diff
   return rightItem
@@ -136,10 +133,14 @@ export const splitItem = (transaction, leftItem, diff) => {
  * @private
  */
 export const redoItem = (transaction, item, redoitems) => {
-  if (item.redone !== null) {
-    return getItemCleanStart(transaction, item.redone)
+  const doc = transaction.doc
+  const store = doc.store
+  const ownClientID = doc.clientID
+  const redone = item.redone
+  if (redone !== null) {
+    return getItemCleanStart(transaction, redone)
   }
-  let parentItem = item.parent._item
+  let parentItem = /** @type {AbstractType<any>} */ (item.parent)._item
   /**
    * @type {Item|null}
    */
@@ -157,14 +158,14 @@ export const redoItem = (transaction, item, redoitems) => {
     left = item
     while (left.right !== null) {
       left = left.right
-      if (left.id.client !== transaction.doc.clientID) {
+      if (left.id.client !== ownClientID) {
         // It is not possible to redo this item because it conflicts with a
         // change from another client
         return null
       }
     }
     if (left.right !== null) {
-      left = /** @type {Item} */ (item.parent._map.get(item.parentSub))
+      left = /** @type {Item} */ (/** @type {AbstractType<any>} */ (item.parent)._map.get(item.parentSub))
     }
     right = null
   }
@@ -186,10 +187,10 @@ export const redoItem = (transaction, item, redoitems) => {
        */
       let leftTrace = left
       // trace redone until parent matches
-      while (leftTrace !== null && leftTrace.parent._item !== parentItem) {
+      while (leftTrace !== null && /** @type {AbstractType<any>} */ (leftTrace.parent)._item !== parentItem) {
         leftTrace = leftTrace.redone === null ? null : getItemCleanStart(transaction, leftTrace.redone)
       }
-      if (leftTrace !== null && leftTrace.parent._item === parentItem) {
+      if (leftTrace !== null && /** @type {AbstractType<any>} */ (leftTrace.parent)._item === parentItem) {
         left = leftTrace
         break
       }
@@ -201,27 +202,29 @@ export const redoItem = (transaction, item, redoitems) => {
        */
       let rightTrace = right
       // trace redone until parent matches
-      while (rightTrace !== null && rightTrace.parent._item !== parentItem) {
+      while (rightTrace !== null && /** @type {AbstractType<any>} */ (rightTrace.parent)._item !== parentItem) {
         rightTrace = rightTrace.redone === null ? null : getItemCleanStart(transaction, rightTrace.redone)
       }
-      if (rightTrace !== null && rightTrace.parent._item === parentItem) {
+      if (rightTrace !== null && /** @type {AbstractType<any>} */ (rightTrace.parent)._item === parentItem) {
         right = rightTrace
         break
       }
       right = right.right
     }
   }
+  const nextClock = getState(store, ownClientID)
+  const nextId = createID(ownClientID, nextClock)
   const redoneItem = new Item(
-    nextID(transaction),
-    left, left === null ? null : left.lastId,
-    right, right === null ? null : right.id,
+    nextId,
+    left, left && left.lastId,
+    right, right && right.id,
     parentItem === null ? item.parent : /** @type {ContentType} */ (parentItem.content).type,
     item.parentSub,
     item.content.copy()
   )
-  item.redone = redoneItem.id
-  keepItem(redoneItem)
-  redoneItem.integrate(transaction)
+  item.redone = nextId
+  keepItem(redoneItem, true)
+  redoneItem.integrate(transaction, 0)
   return redoneItem
 }
 
@@ -235,7 +238,7 @@ export class Item extends AbstractStruct {
    * @param {ID | null} origin
    * @param {Item | null} right
    * @param {ID | null} rightOrigin
-   * @param {AbstractType<any>} parent
+   * @param {AbstractType<any>|ID|null} parent Is a type if integrated, is null if it is possible to copy parent from left or right, is ID before integration to search for it.
    * @param {string | null} parentSub
    * @param {AbstractContent} content
    */
@@ -244,7 +247,6 @@ export class Item extends AbstractStruct {
     /**
      * The item that was originally to the left of this item.
      * @type {ID | null}
-     * @readonly
      */
     this.origin = origin
     /**
@@ -259,14 +261,11 @@ export class Item extends AbstractStruct {
     this.right = right
     /**
      * The item that was originally to the right of this item.
-     * @readonly
      * @type {ID | null}
      */
     this.rightOrigin = rightOrigin
     /**
-     * The parent type.
-     * @type {AbstractType<any>}
-     * @readonly
+     * @type {AbstractType<any>|ID|null}
      */
     this.parent = parent
     /**
@@ -275,14 +274,8 @@ export class Item extends AbstractStruct {
      * to insert this item. If `parentSub = null` type._start is the list in
      * which to insert to. Otherwise it is `parent._map`.
      * @type {String | null}
-     * @readonly
      */
     this.parentSub = parentSub
-    /**
-     * Whether this item was deleted or not.
-     * @type {Boolean}
-     */
-    this.deleted = false
     /**
      * If this type's effect is reundone this type refers to the type that undid
      * this operation.
@@ -293,116 +286,240 @@ export class Item extends AbstractStruct {
      * @type {AbstractContent}
      */
     this.content = content
-    this.length = content.getLength()
-    this.countable = content.isCountable()
     /**
-     * If true, do not garbage collect this Item.
+     * bit1: keep
+     * bit2: countable
+     * bit3: deleted
+     * bit4: mark - mark node as fast-search-marker
+     * @type {number} byte
      */
-    this.keep = false
+    this.info = this.content.isCountable() ? binary.BIT2 : 0
+  }
+
+  /**
+   * This is used to mark the item as an indexed fast-search marker
+   *
+   * @type {boolean}
+   */
+  set marker (isMarked) {
+    if (((this.info & binary.BIT4) > 0) !== isMarked) {
+      this.info ^= binary.BIT4
+    }
+  }
+
+  get marker () {
+    return (this.info & binary.BIT4) > 0
+  }
+
+  /**
+   * If true, do not garbage collect this Item.
+   */
+  get keep () {
+    return (this.info & binary.BIT1) > 0
+  }
+
+  set keep (doKeep) {
+    if (this.keep !== doKeep) {
+      this.info ^= binary.BIT1
+    }
+  }
+
+  get countable () {
+    return (this.info & binary.BIT2) > 0
+  }
+
+  /**
+   * Whether this item was deleted or not.
+   * @type {Boolean}
+   */
+  get deleted () {
+    return (this.info & binary.BIT3) > 0
+  }
+
+  set deleted (doDelete) {
+    if (this.deleted !== doDelete) {
+      this.info ^= binary.BIT3
+    }
+  }
+
+  markDeleted () {
+    this.info |= binary.BIT3
+  }
+
+  /**
+   * Return the creator clientID of the missing op or define missing items and return null.
+   *
+   * @param {Transaction} transaction
+   * @param {StructStore} store
+   * @return {null | number}
+   */
+  getMissing (transaction, store) {
+    if (this.origin && this.origin.client !== this.id.client && this.origin.clock >= getState(store, this.origin.client)) {
+      return this.origin.client
+    }
+    if (this.rightOrigin && this.rightOrigin.client !== this.id.client && this.rightOrigin.clock >= getState(store, this.rightOrigin.client)) {
+      return this.rightOrigin.client
+    }
+    if (this.parent && this.parent.constructor === ID && this.id.client !== this.parent.client && this.parent.clock >= getState(store, this.parent.client)) {
+      return this.parent.client
+    }
+
+    // We have all missing ids, now find the items
+
+    if (this.origin) {
+      this.left = getItemCleanEnd(transaction, store, this.origin)
+      this.origin = this.left.lastId
+    }
+    if (this.rightOrigin) {
+      this.right = getItemCleanStart(transaction, this.rightOrigin)
+      this.rightOrigin = this.right.id
+    }
+    if ((this.left && this.left.constructor === GC) || (this.right && this.right.constructor === GC)) {
+      this.parent = null
+    }
+    // only set parent if this shouldn't be garbage collected
+    if (!this.parent) {
+      if (this.left && this.left.constructor === Item) {
+        this.parent = this.left.parent
+        this.parentSub = this.left.parentSub
+      }
+      if (this.right && this.right.constructor === Item) {
+        this.parent = this.right.parent
+        this.parentSub = this.right.parentSub
+      }
+    } else if (this.parent.constructor === ID) {
+      const parentItem = getItem(store, this.parent)
+      if (parentItem.constructor === GC) {
+        this.parent = null
+      } else {
+        this.parent = /** @type {ContentType} */ (parentItem.content).type
+      }
+    }
+    return null
   }
 
   /**
    * @param {Transaction} transaction
-   * @private
+   * @param {number} offset
    */
-  integrate (transaction) {
-    const store = transaction.doc.store
-    const id = this.id
-    const parent = this.parent
-    const parentSub = this.parentSub
-    const length = this.length
-    /**
-     * @type {Item|null}
-     */
-    let o
-    // set o to the first conflicting item
-    if (this.left !== null) {
-      o = this.left.right
-    } else if (parentSub !== null) {
-      o = parent._map.get(parentSub) || null
-      while (o !== null && o.left !== null) {
-        o = o.left
-      }
-    } else {
-      o = parent._start
+  integrate (transaction, offset) {
+    if (offset > 0) {
+      this.id.clock += offset
+      this.left = getItemCleanEnd(transaction, transaction.doc.store, createID(this.id.client, this.id.clock - 1))
+      this.origin = this.left.lastId
+      this.content = this.content.splice(offset)
+      this.length -= offset
     }
-    // TODO: use something like DeleteSet here (a tree implementation would be best)
-    /**
-     * @type {Set<Item>}
-     */
-    const conflictingItems = new Set()
-    /**
-     * @type {Set<Item>}
-     */
-    const itemsBeforeOrigin = new Set()
-    // Let c in conflictingItems, b in itemsBeforeOrigin
-    // ***{origin}bbbb{this}{c,b}{c,b}{o}***
-    // Note that conflictingItems is a subset of itemsBeforeOrigin
-    while (o !== null && o !== this.right) {
-      itemsBeforeOrigin.add(o)
-      conflictingItems.add(o)
-      if (compareIDs(this.origin, o.origin)) {
-        // case 1
-        if (o.id.client < id.client) {
-          this.left = o
-          conflictingItems.clear()
+
+    if (this.parent) {
+      if ((!this.left && (!this.right || this.right.left !== null)) || (this.left && this.left.right !== this.right)) {
+        /**
+         * @type {Item|null}
+         */
+        let left = this.left
+
+        /**
+         * @type {Item|null}
+         */
+        let o
+        // set o to the first conflicting item
+        if (left !== null) {
+          o = left.right
+        } else if (this.parentSub !== null) {
+          o = /** @type {AbstractType<any>} */ (this.parent)._map.get(this.parentSub) || null
+          while (o !== null && o.left !== null) {
+            o = o.left
+          }
+        } else {
+          o = /** @type {AbstractType<any>} */ (this.parent)._start
         }
-      } else if (o.origin !== null && itemsBeforeOrigin.has(getItem(store, o.origin))) {
-        // case 2
-        if (o.origin === null || !conflictingItems.has(getItem(store, o.origin))) {
-          this.left = o
-          conflictingItems.clear()
+        // TODO: use something like DeleteSet here (a tree implementation would be best)
+        // @todo use global set definitions
+        /**
+         * @type {Set<Item>}
+         */
+        const conflictingItems = new Set()
+        /**
+         * @type {Set<Item>}
+         */
+        const itemsBeforeOrigin = new Set()
+        // Let c in conflictingItems, b in itemsBeforeOrigin
+        // ***{origin}bbbb{this}{c,b}{c,b}{o}***
+        // Note that conflictingItems is a subset of itemsBeforeOrigin
+        while (o !== null && o !== this.right) {
+          itemsBeforeOrigin.add(o)
+          conflictingItems.add(o)
+          if (compareIDs(this.origin, o.origin)) {
+            // case 1
+            if (o.id.client < this.id.client) {
+              left = o
+              conflictingItems.clear()
+            } else if (compareIDs(this.rightOrigin, o.rightOrigin)) {
+              // this and o are conflicting and point to the same integration points. The id decides which item comes first.
+              // Since this is to the left of o, we can break here
+              break
+            } // else, o might be integrated before an item that this conflicts with. If so, we will find it in the next iterations
+          } else if (o.origin !== null && itemsBeforeOrigin.has(getItem(transaction.doc.store, o.origin))) { // use getItem instead of getItemCleanEnd because we don't want / need to split items.
+            // case 2
+            if (!conflictingItems.has(getItem(transaction.doc.store, o.origin))) {
+              left = o
+              conflictingItems.clear()
+            }
+          } else {
+            break
+          }
+          o = o.right
         }
-      } else {
-        break
+        this.left = left
       }
-      o = o.right
-    }
-    // reconnect left/right + update parent map/start if necessary
-    if (this.left !== null) {
-      const right = this.left.right
-      this.right = right
-      this.left.right = this
-    } else {
-      let r
-      if (parentSub !== null) {
-        r = parent._map.get(parentSub) || null
-        while (r !== null && r.left !== null) {
-          r = r.left
-        }
-      } else {
-        r = parent._start
-        parent._start = this
-      }
-      this.right = r
-    }
-    if (this.right !== null) {
-      this.right.left = this
-    } else if (parentSub !== null) {
-      // set as current parent value if right === null and this is parentSub
-      parent._map.set(parentSub, this)
+      // reconnect left/right + update parent map/start if necessary
       if (this.left !== null) {
-        // this is the current attribute value of parent. delete right
-        this.left.delete(transaction)
+        const right = this.left.right
+        this.right = right
+        this.left.right = this
+      } else {
+        let r
+        if (this.parentSub !== null) {
+          r = /** @type {AbstractType<any>} */ (this.parent)._map.get(this.parentSub) || null
+          while (r !== null && r.left !== null) {
+            r = r.left
+          }
+        } else {
+          r = /** @type {AbstractType<any>} */ (this.parent)._start
+          ;/** @type {AbstractType<any>} */ (this.parent)._start = this
+        }
+        this.right = r
       }
-    }
-    // adjust length of parent
-    if (parentSub === null && this.countable && !this.deleted) {
-      parent._length += length
-    }
-    addStruct(store, this)
-    this.content.integrate(transaction, this)
-    // add parent to transaction.changed
-    addChangedTypeToTransaction(transaction, parent, parentSub)
-    if ((parent._item !== null && parent._item.deleted) || (this.right !== null && parentSub !== null)) {
-      // delete if parent is deleted or if this is not the current attribute value of parent
-      this.delete(transaction)
+      if (this.right !== null) {
+        this.right.left = this
+      } else if (this.parentSub !== null) {
+        // set as current parent value if right === null and this is parentSub
+        /** @type {AbstractType<any>} */ (this.parent)._map.set(this.parentSub, this)
+        if (this.left !== null) {
+          // this is the current attribute value of parent. delete right
+          this.left.delete(transaction)
+        }
+      }
+      // adjust length of parent
+      if (this.parentSub === null && this.countable && !this.deleted) {
+        /** @type {AbstractType<any>} */ (this.parent)._length += this.length
+      }
+      addStruct(transaction.doc.store, this)
+      this.content.integrate(transaction, this)
+      // add parent to transaction.changed
+      addChangedTypeToTransaction(transaction, /** @type {AbstractType<any>} */ (this.parent), this.parentSub)
+      if ((/** @type {AbstractType<any>} */ (this.parent)._item !== null && /** @type {AbstractType<any>} */ (this.parent)._item.deleted) || (this.parentSub !== null && this.right !== null)) {
+        // delete if parent is deleted or if this is not the current attribute value of parent
+        this.delete(transaction)
+      }
+    } else {
+      // parent is not defined. Integrate GC struct instead
+      new GC(this.id, this.length).integrate(transaction, 0)
     }
   }
 
   /**
    * Returns the next non-deleted item
-   * @private
    */
   get next () {
     let n = this.right
@@ -414,7 +531,6 @@ export class Item extends AbstractStruct {
 
   /**
    * Returns the previous non-deleted item
-   * @private
    */
   get prev () {
     let n = this.left
@@ -428,7 +544,8 @@ export class Item extends AbstractStruct {
    * Computes the last content address of this Item.
    */
   get lastId () {
-    return createID(this.id.client, this.id.clock + this.length - 1)
+    // allocating ids is pretty costly because of the amount of ids created, so we try to reuse whenever possible
+    return this.length === 1 ? this.id : createID(this.id.client, this.id.clock + this.length - 1)
   }
 
   /**
@@ -470,13 +587,13 @@ export class Item extends AbstractStruct {
    */
   delete (transaction) {
     if (!this.deleted) {
-      const parent = this.parent
+      const parent = /** @type {AbstractType<any>} */ (this.parent)
       // adjust the length of parent
       if (this.countable && this.parentSub === null) {
         parent._length -= this.length
       }
-      this.deleted = true
-      addToDeleteSet(transaction.deleteSet, this.id, this.length)
+      this.markDeleted()
+      addToDeleteSet(transaction.deleteSet, this.id.client, this.id.clock, this.length)
       maplib.setIfUndefined(transaction.changed, parent, set.create).add(this.parentSub)
       this.content.delete(transaction)
     }
@@ -485,8 +602,6 @@ export class Item extends AbstractStruct {
   /**
    * @param {StructStore} store
    * @param {boolean} parentGCd
-   *
-   * @private
    */
   gc (store, parentGCd) {
     if (!this.deleted) {
@@ -506,10 +621,8 @@ export class Item extends AbstractStruct {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    * @param {number} offset
-   *
-   * @private
    */
   write (encoder, offset) {
     const origin = offset > 0 ? createID(this.id.client, this.id.clock + offset - 1) : this.origin
@@ -519,27 +632,28 @@ export class Item extends AbstractStruct {
       (origin === null ? 0 : binary.BIT8) | // origin is defined
       (rightOrigin === null ? 0 : binary.BIT7) | // right origin is defined
       (parentSub === null ? 0 : binary.BIT6) // parentSub is non-null
-    encoding.writeUint8(encoder, info)
+    encoder.writeInfo(info)
     if (origin !== null) {
-      writeID(encoder, origin)
+      encoder.writeLeftID(origin)
     }
     if (rightOrigin !== null) {
-      writeID(encoder, rightOrigin)
+      encoder.writeRightID(rightOrigin)
     }
     if (origin === null && rightOrigin === null) {
-      const parent = this.parent
-      if (parent._item === null) {
+      const parent = /** @type {AbstractType<any>} */ (this.parent)
+      const parentItem = parent._item
+      if (parentItem === null) {
         // parent type on y._map
         // find the correct key
         const ykey = findRootTypeKey(parent)
-        encoding.writeVarUint(encoder, 1) // write parentYKey
-        encoding.writeVarString(encoder, ykey)
+        encoder.writeParentInfo(true) // write parentYKey
+        encoder.writeString(ykey)
       } else {
-        encoding.writeVarUint(encoder, 0) // write parent id
-        writeID(encoder, parent._item.id)
+        encoder.writeParentInfo(false) // write parent id
+        encoder.writeLeftID(parentItem.id)
       }
       if (parentSub !== null) {
-        encoding.writeVarString(encoder, parentSub)
+        encoder.writeString(parentSub)
       }
     }
     this.content.write(encoder, offset)
@@ -547,26 +661,27 @@ export class Item extends AbstractStruct {
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @param {number} info
  */
-const readItemContent = (decoder, info) => contentRefs[info & binary.BITS5](decoder)
+export const readItemContent = (decoder, info) => contentRefs[info & binary.BITS5](decoder)
 
 /**
  * A lookup map for reading Item content.
  *
- * @type {Array<function(decoding.Decoder):AbstractContent>}
+ * @type {Array<function(AbstractUpdateDecoder):AbstractContent>}
  */
 export const contentRefs = [
   () => { throw error.unexpectedCase() }, // GC is not ItemContent
-  readContentDeleted,
-  readContentJSON,
-  readContentBinary,
-  readContentString,
-  readContentEmbed,
-  readContentFormat,
-  readContentType,
-  readContentAny
+  readContentDeleted, // 1
+  readContentJSON, // 2
+  readContentBinary, // 3
+  readContentString, // 4
+  readContentEmbed, // 5
+  readContentFormat, // 6
+  readContentType, // 7
+  readContentAny, // 8
+  readContentDoc // 9
 ]
 
 /**
@@ -646,7 +761,7 @@ export class AbstractContent {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
@@ -660,125 +775,3 @@ export class AbstractContent {
     throw error.methodUnimplemented()
   }
 }
-
-/**
- * @private
- */
-export class ItemRef extends AbstractStructRef {
-  /**
-   * @param {decoding.Decoder} decoder
-   * @param {ID} id
-   * @param {number} info
-   */
-  constructor (decoder, id, info) {
-    super(id)
-    /**
-     * The item that was originally to the left of this item.
-     * @type {ID | null}
-     */
-    this.left = (info & binary.BIT8) === binary.BIT8 ? readID(decoder) : null
-    /**
-     * The item that was originally to the right of this item.
-     * @type {ID | null}
-     */
-    this.right = (info & binary.BIT7) === binary.BIT7 ? readID(decoder) : null
-    const canCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
-    const hasParentYKey = canCopyParentInfo ? decoding.readVarUint(decoder) === 1 : false
-    /**
-     * If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
-     * and we read the next string as parentYKey.
-     * It indicates how we store/retrieve parent from `y.share`
-     * @type {string|null}
-     */
-    this.parentYKey = canCopyParentInfo && hasParentYKey ? decoding.readVarString(decoder) : null
-    /**
-     * The parent type.
-     * @type {ID | null}
-     */
-    this.parent = canCopyParentInfo && !hasParentYKey ? readID(decoder) : null
-    /**
-     * If the parent refers to this item with some kind of key (e.g. YMap, the
-     * key is specified here. The key is then used to refer to the list in which
-     * to insert this item. If `parentSub = null` type._start is the list in
-     * which to insert to. Otherwise it is `parent._map`.
-     * @type {String | null}
-     */
-    this.parentSub = canCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoding.readVarString(decoder) : null
-    const missing = this._missing
-    if (this.left !== null) {
-      missing.push(this.left)
-    }
-    if (this.right !== null) {
-      missing.push(this.right)
-    }
-    if (this.parent !== null) {
-      missing.push(this.parent)
-    }
-    /**
-     * @type {AbstractContent}
-     */
-    this.content = readItemContent(decoder, info)
-    this.length = this.content.getLength()
-  }
-
-  /**
-   * @param {Transaction} transaction
-   * @param {StructStore} store
-   * @param {number} offset
-   * @return {Item|GC}
-   */
-  toStruct (transaction, store, offset) {
-    if (offset > 0) {
-      /**
-       * @type {ID}
-       */
-      const id = this.id
-      this.id = createID(id.client, id.clock + offset)
-      this.left = createID(this.id.client, this.id.clock - 1)
-      this.content = this.content.splice(offset)
-      this.length -= offset
-    }
-
-    const left = this.left === null ? null : getItemCleanEnd(transaction, store, this.left)
-    const right = this.right === null ? null : getItemCleanStart(transaction, this.right)
-    let parent = null
-    let parentSub = this.parentSub
-    if (this.parent !== null) {
-      const parentItem = getItem(store, this.parent)
-      // Edge case: toStruct is called with an offset > 0. In this case left is defined.
-      // Depending in which order structs arrive, left may be GC'd and the parent not
-      // deleted. This is why we check if left is GC'd. Strictly we don't have
-      // to check if right is GC'd, but we will in case we run into future issues
-      if (!parentItem.deleted && (left === null || left.constructor !== GC) && (right === null || right.constructor !== GC)) {
-        parent = /** @type {ContentType} */ (parentItem.content).type
-      }
-    } else if (this.parentYKey !== null) {
-      parent = transaction.doc.get(this.parentYKey)
-    } else if (left !== null) {
-      if (left.constructor !== GC) {
-        parent = left.parent
-        parentSub = left.parentSub
-      }
-    } else if (right !== null) {
-      if (right.constructor !== GC) {
-        parent = right.parent
-        parentSub = right.parentSub
-      }
-    } else {
-      throw error.unexpectedCase()
-    }
-
-    return parent === null
-      ? new GC(this.id, this.length)
-      : new Item(
-        this.id,
-        left,
-        this.left,
-        right,
-        this.right,
-        parent,
-        parentSub,
-        this.content
-      )
-  }
-}

src/types/AbstractType.js

@@ -4,25 +4,230 @@ import {
   callEventHandlerListeners,
   addEventHandlerListener,
   createEventHandler,
-  nextID,
+  getState,
   isVisible,
   ContentType,
+  createID,
   ContentAny,
   ContentBinary,
-  createID,
   getItemCleanStart,
-  Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
+  ContentDoc, YText, YArray, AbstractUpdateEncoder, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
 } from '../internals.js'
 
 import * as map from 'lib0/map.js'
 import * as iterator from 'lib0/iterator.js'
 import * as error from 'lib0/error.js'
-import * as encoding from 'lib0/encoding.js' // eslint-disable-line
+import * as math from 'lib0/math.js'
+
+const maxSearchMarker = 80
+
+/**
+ * A unique timestamp that identifies each marker.
+ *
+ * Time is relative,.. this is more like an ever-increasing clock.
+ *
+ * @type {number}
+ */
+let globalSearchMarkerTimestamp = 0
+
+export class ArraySearchMarker {
+  /**
+   * @param {Item} p
+   * @param {number} index
+   */
+  constructor (p, index) {
+    p.marker = true
+    this.p = p
+    this.index = index
+    this.timestamp = globalSearchMarkerTimestamp++
+  }
+}
+
+/**
+ * @param {ArraySearchMarker} marker
+ */
+const refreshMarkerTimestamp = marker => { marker.timestamp = globalSearchMarkerTimestamp++ }
+
+/**
+ * This is rather complex so this function is the only thing that should overwrite a marker
+ *
+ * @param {ArraySearchMarker} marker
+ * @param {Item} p
+ * @param {number} index
+ */
+const overwriteMarker = (marker, p, index) => {
+  marker.p.marker = false
+  marker.p = p
+  p.marker = true
+  marker.index = index
+  marker.timestamp = globalSearchMarkerTimestamp++
+}
+
+/**
+ * @param {Array<ArraySearchMarker>} searchMarker
+ * @param {Item} p
+ * @param {number} index
+ */
+const markPosition = (searchMarker, p, index) => {
+  if (searchMarker.length >= maxSearchMarker) {
+    // override oldest marker (we don't want to create more objects)
+    const marker = searchMarker.reduce((a, b) => a.timestamp < b.timestamp ? a : b)
+    overwriteMarker(marker, p, index)
+    return marker
+  } else {
+    // create new marker
+    const pm = new ArraySearchMarker(p, index)
+    searchMarker.push(pm)
+    return pm
+  }
+}
+
+/**
+ * Search marker help us to find positions in the associative array faster.
+ *
+ * They speed up the process of finding a position without much bookkeeping.
+ *
+ * A maximum of `maxSearchMarker` objects are created.
+ *
+ * This function always returns a refreshed marker (updated timestamp)
+ *
+ * @param {AbstractType<any>} yarray
+ * @param {number} index
+ */
+export const findMarker = (yarray, index) => {
+  if (yarray._start === null || index === 0 || yarray._searchMarker === null) {
+    return null
+  }
+  const marker = yarray._searchMarker.length === 0 ? null : yarray._searchMarker.reduce((a, b) => math.abs(index - a.index) < math.abs(index - b.index) ? a : b)
+  let p = yarray._start
+  let pindex = 0
+  if (marker !== null) {
+    p = marker.p
+    pindex = marker.index
+    refreshMarkerTimestamp(marker) // we used it, we might need to use it again
+  }
+  // iterate to right if possible
+  while (p.right !== null && pindex < index) {
+    if (!p.deleted && p.countable) {
+      if (index < pindex + p.length) {
+        break
+      }
+      pindex += p.length
+    }
+    p = p.right
+  }
+  // iterate to left if necessary (might be that pindex > index)
+  while (p.left !== null && pindex > index) {
+    p = p.left
+    if (!p.deleted && p.countable) {
+      pindex -= p.length
+    }
+  }
+  // we want to make sure that p can't be merged with left, because that would screw up everything
+  // in that cas just return what we have (it is most likely the best marker anyway)
+  // iterate to left until p can't be merged with left
+  while (p.left !== null && p.left.id.client === p.id.client && p.left.id.clock + p.left.length === p.id.clock) {
+    p = p.left
+    if (!p.deleted && p.countable) {
+      pindex -= p.length
+    }
+  }
+
+  // @todo remove!
+  // assure position
+  // {
+  //   let start = yarray._start
+  //   let pos = 0
+  //   while (start !== p) {
+  //     if (!start.deleted && start.countable) {
+  //       pos += start.length
+  //     }
+  //     start = /** @type {Item} */ (start.right)
+  //   }
+  //   if (pos !== pindex) {
+  //     debugger
+  //     throw new Error('Gotcha position fail!')
+  //   }
+  // }
+  // if (marker) {
+  //   if (window.lengthes == null) {
+  //     window.lengthes = []
+  //     window.getLengthes = () => window.lengthes.sort((a, b) => a - b)
+  //   }
+  //   window.lengthes.push(marker.index - pindex)
+  //   console.log('distance', marker.index - pindex, 'len', p && p.parent.length)
+  // }
+  if (marker !== null && math.abs(marker.index - pindex) < /** @type {YText|YArray<any>} */ (p.parent).length / maxSearchMarker) {
+    // adjust existing marker
+    overwriteMarker(marker, p, pindex)
+    return marker
+  } else {
+    // create new marker
+    return markPosition(yarray._searchMarker, p, pindex)
+  }
+}
+
+/**
+ * Update markers when a change happened.
+ *
+ * This should be called before doing a deletion!
+ *
+ * @param {Array<ArraySearchMarker>} searchMarker
+ * @param {number} index
+ * @param {number} len If insertion, len is positive. If deletion, len is negative.
+ */
+export const updateMarkerChanges = (searchMarker, index, len) => {
+  for (let i = searchMarker.length - 1; i >= 0; i--) {
+    const m = searchMarker[i]
+    if (len > 0) {
+      /**
+       * @type {Item|null}
+       */
+      let p = m.p
+      p.marker = false
+      // Ideally we just want to do a simple position comparison, but this will only work if
+      // search markers don't point to deleted items for formats.
+      // Iterate marker to prev undeleted countable position so we know what to do when updating a position
+      while (p && (p.deleted || !p.countable)) {
+        p = p.left
+        if (p && !p.deleted && p.countable) {
+          // adjust position. the loop should break now
+          m.index -= p.length
+        }
+      }
+      if (p === null || p.marker === true) {
+        // remove search marker if updated position is null or if position is already marked
+        searchMarker.splice(i, 1)
+        continue
+      }
+      m.p = p
+      p.marker = true
+    }
+    if (index < m.index || (len > 0 && index === m.index)) { // a simple index <= m.index check would actually suffice
+      m.index = math.max(index, m.index + len)
+    }
+  }
+}
+
+/**
+ * Accumulate all (list) children of a type and return them as an Array.
+ *
+ * @param {AbstractType<any>} t
+ * @return {Array<Item>}
+ */
+export const getTypeChildren = t => {
+  let s = t._start
+  const arr = []
+  while (s) {
+    arr.push(s)
+    s = s.right
+  }
+  return arr
+}
 
 /**
  * Call event listeners with an event. This will also add an event to all
  * parents (for `.observeDeep` handlers).
- * @private
  *
  * @template EventType
  * @param {AbstractType<EventType>} type
@@ -38,7 +243,7 @@ export const callTypeObservers = (type, transaction, event) => {
     if (type._item === null) {
       break
     }
-    type = type._item.parent
+    type = /** @type {AbstractType<any>} */ (type._item.parent)
   }
   callEventHandlerListeners(changedType._eH, event, transaction)
 }
@@ -54,17 +259,14 @@ export class AbstractType {
      */
     this._item = null
     /**
-     * @private
      * @type {Map<string,Item>}
      */
     this._map = new Map()
     /**
-     * @private
      * @type {Item|null}
      */
     this._start = null
     /**
-     * @private
      * @type {Doc|null}
      */
     this.doc = null
@@ -79,6 +281,10 @@ export class AbstractType {
      * @type {EventHandler<Array<YEvent>,Transaction>}
      */
     this._dEH = createEventHandler()
+    /**
+     * @type {null | Array<ArraySearchMarker>}
+     */
+    this._searchMarker = null
   }
 
   /**
@@ -90,7 +296,6 @@ export class AbstractType {
    *
    * @param {Doc} y The Yjs instance
    * @param {Item|null} item
-   * @private
    */
   _integrate (y, item) {
     this.doc = y
@@ -99,15 +304,13 @@ export class AbstractType {
 
   /**
    * @return {AbstractType<EventType>}
-   * @private
    */
   _copy () {
     throw error.methodUnimplemented()
   }
 
   /**
-   * @param {encoding.Encoder} encoder
-   * @private
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) { }
 
@@ -128,10 +331,12 @@ export class AbstractType {
    *
    * @param {Transaction} transaction
    * @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
-   *
-   * @private
    */
-  _callObserver (transaction, parentSubs) { /* skip if no type is specified */ }
+  _callObserver (transaction, parentSubs) {
+    if (!transaction.local && this._searchMarker) {
+      this._searchMarker.length = 0
+    }
+  }
 
   /**
    * Observe all events that are created on this type.
@@ -347,7 +552,13 @@ export const typeListForEachSnapshot = (type, f, snapshot) => {
  * @function
  */
 export const typeListGet = (type, index) => {
-  for (let n = type._start; n !== null; n = n.right) {
+  const marker = findMarker(type, index)
+  let n = type._start
+  if (marker !== null) {
+    n = marker.p
+    index -= marker.index
+  }
+  for (; n !== null; n = n.right) {
     if (!n.deleted && n.countable) {
       if (index < n.length) {
         return n.content.getContent()[index]
@@ -368,6 +579,9 @@ export const typeListGet = (type, index) => {
  */
 export const typeListInsertGenericsAfter = (transaction, parent, referenceItem, content) => {
   let left = referenceItem
+  const doc = transaction.doc
+  const ownClientId = doc.clientID
+  const store = doc.store
   const right = referenceItem === null ? parent._start : referenceItem.right
   /**
    * @type {Array<Object|Array<any>|number>}
@@ -375,8 +589,8 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem,
   let jsonContent = []
   const packJsonContent = () => {
     if (jsonContent.length > 0) {
-      left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentAny(jsonContent))
-      left.integrate(transaction)
+      left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentAny(jsonContent))
+      left.integrate(transaction, 0)
       jsonContent = []
     }
   }
@@ -394,13 +608,17 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem,
         switch (c.constructor) {
           case Uint8Array:
           case ArrayBuffer:
-            left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentBinary(new Uint8Array(/** @type {Uint8Array} */ (c))))
-            left.integrate(transaction)
+            left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentBinary(new Uint8Array(/** @type {Uint8Array} */ (c))))
+            left.integrate(transaction, 0)
+            break
+          case Doc:
+            left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentDoc(/** @type {Doc} */ (c)))
+            left.integrate(transaction, 0)
             break
           default:
             if (c instanceof AbstractType) {
-              left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentType(c))
-              left.integrate(transaction)
+              left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentType(c))
+              left.integrate(transaction, 0)
             } else {
               throw new Error('Unexpected content type in insert operation')
             }
@@ -421,9 +639,24 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem,
  */
 export const typeListInsertGenerics = (transaction, parent, index, content) => {
   if (index === 0) {
+    if (parent._searchMarker) {
+      updateMarkerChanges(parent._searchMarker, index, content.length)
+    }
     return typeListInsertGenericsAfter(transaction, parent, null, content)
   }
+  const startIndex = index
+  const marker = findMarker(parent, index)
   let n = parent._start
+  if (marker !== null) {
+    n = marker.p
+    index -= marker.index
+    // we need to iterate one to the left so that the algorithm works
+    if (index === 0) {
+      // @todo refactor this as it actually doesn't consider formats
+      n = n.prev // important! get the left undeleted item so that we can actually decrease index
+      index += (n && n.countable && !n.deleted) ? n.length : 0
+    }
+  }
   for (; n !== null; n = n.right) {
     if (!n.deleted && n.countable) {
       if (index <= n.length) {
@@ -436,6 +669,9 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => {
       index -= n.length
     }
   }
+  if (parent._searchMarker) {
+    updateMarkerChanges(parent._searchMarker, startIndex, content.length)
+  }
   return typeListInsertGenericsAfter(transaction, parent, n, content)
 }
 
@@ -450,7 +686,14 @@ export const typeListInsertGenerics = (transaction, parent, index, content) => {
  */
 export const typeListDelete = (transaction, parent, index, length) => {
   if (length === 0) { return }
+  const startIndex = index
+  const startLength = length
+  const marker = findMarker(parent, index)
   let n = parent._start
+  if (marker !== null) {
+    n = marker.p
+    index -= marker.index
+  }
   // compute the first item to be deleted
   for (; n !== null && index > 0; n = n.right) {
     if (!n.deleted && n.countable) {
@@ -474,6 +717,9 @@ export const typeListDelete = (transaction, parent, index, length) => {
   if (length > 0) {
     throw error.create('array length exceeded')
   }
+  if (parent._searchMarker) {
+    updateMarkerChanges(parent._searchMarker, startIndex, -startLength + length /* in case we remove the above exception */)
+  }
 }
 
 /**
@@ -502,6 +748,8 @@ export const typeMapDelete = (transaction, parent, key) => {
  */
 export const typeMapSet = (transaction, parent, key, value) => {
   const left = parent._map.get(key) || null
+  const doc = transaction.doc
+  const ownClientId = doc.clientID
   let content
   if (value == null) {
     content = new ContentAny([value])
@@ -517,6 +765,9 @@ export const typeMapSet = (transaction, parent, key, value) => {
       case Uint8Array:
         content = new ContentBinary(/** @type {Uint8Array} */ (value))
         break
+      case Doc:
+        content = new ContentDoc(/** @type {Doc} */ (value))
+        break
       default:
         if (value instanceof AbstractType) {
           content = new ContentType(value)
@@ -525,7 +776,7 @@ export const typeMapSet = (transaction, parent, key, value) => {
         }
     }
   }
-  new Item(nextID(transaction), left, left === null ? null : left.lastId, null, null, parent, key, content).integrate(transaction)
+  new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, null, null, parent, key, content).integrate(transaction, 0)
 }
 
 /**
@@ -553,11 +804,11 @@ export const typeMapGetAll = (parent) => {
    * @type {Object<string,any>}
    */
   const res = {}
-  for (const [key, value] of parent._map) {
+  parent._map.forEach((value, key) => {
     if (!value.deleted) {
       res[key] = value.content.getContent()[value.length - 1]
     }
-  }
+  })
   return res
 }
 

src/types/YArray.js

@@ -15,12 +15,9 @@ import {
   YArrayRefID,
   callTypeObservers,
   transact,
-  Doc, Transaction, Item // eslint-disable-line
+  ArraySearchMarker, AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line
 } from '../internals.js'
 
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
-import * as encoding from 'lib0/encoding.js'
-
 /**
  * Event that describes the changes on a YArray
  * @template T
@@ -40,7 +37,7 @@ export class YArrayEvent extends YEvent {
  * A shared Array implementation.
  * @template T
  * @extends AbstractType<YArrayEvent<T>>
- * @implements {IterableIterator<T>}
+ * @implements {Iterable<T>}
  */
 export class YArray extends AbstractType {
   constructor () {
@@ -50,6 +47,10 @@ export class YArray extends AbstractType {
      * @private
      */
     this._prelimContent = []
+    /**
+     * @type {Array<ArraySearchMarker>}
+     */
+    this._searchMarker = []
   }
 
   /**
@@ -61,8 +62,6 @@ export class YArray extends AbstractType {
    *
    * @param {Doc} y The Yjs instance
    * @param {Item} item
-   *
-   * @private
    */
   _integrate (y, item) {
     super._integrate(y, item)
@@ -83,10 +82,9 @@ export class YArray extends AbstractType {
    *
    * @param {Transaction} transaction
    * @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
-   *
-   * @private
    */
   _callObserver (transaction, parentSubs) {
+    super._callObserver(transaction, parentSubs)
     callTypeObservers(this, transaction, new YArrayEvent(this, transaction))
   }
 
@@ -125,6 +123,15 @@ export class YArray extends AbstractType {
     this.insert(this.length, content)
   }
 
+  /**
+   * Preppends content to this YArray.
+   *
+   * @param {Array<T>} content Array of content to preppend.
+   */
+  unshift (content) {
+    this.insert(0, content)
+  }
+
   /**
    * Deletes elements starting from an index.
    *
@@ -199,16 +206,15 @@ export class YArray extends AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
-   * @private
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YArrayRefID)
+    encoder.writeTypeRef(YArrayRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  *
  * @private
  * @function

src/types/YMap.js

@@ -14,11 +14,9 @@ import {
   YMapRefID,
   callTypeObservers,
   transact,
-  Doc, Transaction, Item // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
 import * as iterator from 'lib0/iterator.js'
 
 /**
@@ -42,16 +40,26 @@ export class YMapEvent extends YEvent {
  * A shared Map implementation.
  *
  * @extends AbstractType<YMapEvent<T>>
- * @implements {IterableIterator}
+ * @implements {Iterable<T>}
  */
 export class YMap extends AbstractType {
-  constructor () {
+  /**
+   *
+   * @param {Iterable<readonly [string, any]>=} entries - an optional iterable to initialize the YMap
+   */
+  constructor (entries) {
     super()
     /**
      * @type {Map<string,any>?}
      * @private
      */
-    this._prelimContent = new Map()
+    this._prelimContent = null
+
+    if (entries === undefined) {
+      this._prelimContent = new Map()
+    } else {
+      this._prelimContent = new Map(entries)
+    }
   }
 
   /**
@@ -63,14 +71,12 @@ export class YMap extends AbstractType {
    *
    * @param {Doc} y The Yjs instance
    * @param {Item} item
-   *
-   * @private
    */
   _integrate (y, item) {
     super._integrate(y, item)
-    for (const [key, value] of /** @type {Map<string, any>} */ (this._prelimContent)) {
+    ;/** @type {Map<string, any>} */ (this._prelimContent).forEach((value, key) => {
       this.set(key, value)
-    }
+    })
     this._prelimContent = null
   }
 
@@ -83,8 +89,6 @@ export class YMap extends AbstractType {
    *
    * @param {Transaction} transaction
    * @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
-   *
-   * @private
    */
   _callObserver (transaction, parentSubs) {
     callTypeObservers(this, transaction, new YMapEvent(this, transaction, parentSubs))
@@ -100,15 +104,24 @@ export class YMap extends AbstractType {
      * @type {Object<string,T>}
      */
     const map = {}
-    for (const [key, item] of this._map) {
+    this._map.forEach((item, key) => {
       if (!item.deleted) {
         const v = item.content.getContent()[item.length - 1]
         map[key] = v instanceof AbstractType ? v.toJSON() : v
       }
-    }
+    })
     return map
   }
 
+  /**
+   * Returns the size of the YMap (count of key/value pairs)
+   *
+   * @return {number}
+   */
+  get size () {
+    return [...createMapIterator(this._map)].length
+  }
+
   /**
    * Returns the keys for each element in the YMap Type.
    *
@@ -119,9 +132,9 @@ export class YMap extends AbstractType {
   }
 
   /**
-   * Returns the keys for each element in the YMap Type.
+   * Returns the values for each element in the YMap Type.
    *
-   * @return {IterableIterator<string>}
+   * @return {IterableIterator<any>}
    */
   values () {
     return iterator.iteratorMap(createMapIterator(this._map), /** @param {any} v */ v => v[1].content.getContent()[v[1].length - 1])
@@ -137,7 +150,7 @@ export class YMap extends AbstractType {
   }
 
   /**
-   * Executes a provided function on once on overy key-value pair.
+   * Executes a provided function on once on every key-value pair.
    *
    * @param {function(T,string,YMap<T>):void} f A function to execute on every element of this YArray.
    */
@@ -146,11 +159,11 @@ export class YMap extends AbstractType {
      * @type {Object<string,T>}
      */
     const map = {}
-    for (const [key, item] of this._map) {
+    this._map.forEach((item, key) => {
       if (!item.deleted) {
         f(item.content.getContent()[item.length - 1], key, this)
       }
-    }
+    })
     return map
   }
 
@@ -214,17 +227,15 @@ export class YMap extends AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
-   *
-   * @private
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YMapRefID)
+    encoder.writeTypeRef(YMapRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  *
  * @private
  * @function

src/types/YText.js

@@ -6,23 +6,28 @@
 import {
   YEvent,
   AbstractType,
-  nextID,
-  createID,
   getItemCleanStart,
+  getState,
   isVisible,
+  createID,
   YTextRefID,
   callTypeObservers,
   transact,
   ContentEmbed,
+  GC,
   ContentFormat,
   ContentString,
   splitSnapshotAffectedStructs,
-  ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
+  iterateDeletedStructs,
+  iterateStructs,
+  findMarker,
+  updateMarkerChanges,
+  ArraySearchMarker, AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
-import * as encoding from 'lib0/encoding.js'
 import * as object from 'lib0/object.js'
+import * as map from 'lib0/map.js'
+import * as error from 'lib0/error.js'
 
 /**
  * @param {any} a
@@ -31,75 +36,79 @@ import * as object from 'lib0/object.js'
  */
 const equalAttrs = (a, b) => a === b || (typeof a === 'object' && typeof b === 'object' && a && b && object.equalFlat(a, b))
 
-export class ItemListPosition {
-  /**
-   * @param {Item|null} left
-   * @param {Item|null} right
-   */
-  constructor (left, right) {
-    this.left = left
-    this.right = right
-  }
-}
-
-export class ItemTextListPosition extends ItemListPosition {
+export class ItemTextListPosition {
   /**
    * @param {Item|null} left
    * @param {Item|null} right
+   * @param {number} index
    * @param {Map<string,any>} currentAttributes
    */
-  constructor (left, right, currentAttributes) {
-    super(left, right)
+  constructor (left, right, index, currentAttributes) {
+    this.left = left
+    this.right = right
+    this.index = index
     this.currentAttributes = currentAttributes
   }
-}
 
-export class ItemInsertionResult extends ItemListPosition {
   /**
-   * @param {Item|null} left
-   * @param {Item|null} right
-   * @param {Map<string,any>} negatedAttributes
+   * Only call this if you know that this.right is defined
    */
-  constructor (left, right, negatedAttributes) {
-    super(left, right)
-    this.negatedAttributes = negatedAttributes
+  forward () {
+    if (this.right === null) {
+      error.unexpectedCase()
+    }
+    switch (this.right.content.constructor) {
+      case ContentEmbed:
+      case ContentString:
+        if (!this.right.deleted) {
+          this.index += this.right.length
+        }
+        break
+      case ContentFormat:
+        if (!this.right.deleted) {
+          updateCurrentAttributes(this.currentAttributes, /** @type {ContentFormat} */ (this.right.content))
+        }
+        break
+    }
+    this.left = this.right
+    this.right = this.right.right
   }
 }
 
 /**
  * @param {Transaction} transaction
- * @param {Map<string,any>} currentAttributes
- * @param {Item|null} left
- * @param {Item|null} right
- * @param {number} count
+ * @param {ItemTextListPosition} pos
+ * @param {number} count steps to move forward
  * @return {ItemTextListPosition}
  *
  * @private
  * @function
  */
-const findNextPosition = (transaction, currentAttributes, left, right, count) => {
-  while (right !== null && count > 0) {
-    switch (right.content.constructor) {
+const findNextPosition = (transaction, pos, count) => {
+  while (pos.right !== null && count > 0) {
+    switch (pos.right.content.constructor) {
       case ContentEmbed:
       case ContentString:
-        if (!right.deleted) {
-          if (count < right.length) {
+        if (!pos.right.deleted) {
+          if (count < pos.right.length) {
             // split right
-            getItemCleanStart(transaction, createID(right.id.client, right.id.clock + count))
+            getItemCleanStart(transaction, createID(pos.right.id.client, pos.right.id.clock + count))
           }
-          count -= right.length
+          pos.index += pos.right.length
+          count -= pos.right.length
         }
         break
       case ContentFormat:
-        if (!right.deleted) {
-          updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
+        if (!pos.right.deleted) {
+          updateCurrentAttributes(pos.currentAttributes, /** @type {ContentFormat} */ (pos.right.content))
         }
         break
     }
-    left = right
-    right = right.right
+    pos.left = pos.right
+    pos.right = pos.right.right
+    // pos.forward() - we don't forward because that would halve the performance because we already do the checks above
   }
-  return new ItemTextListPosition(left, right, currentAttributes)
+  return pos
 }
 
 /**
@@ -113,8 +122,14 @@ const findNextPosition = (transaction, currentAttributes, left, right, count) =>
  */
 const findPosition = (transaction, parent, index) => {
   const currentAttributes = new Map()
-  const right = parent._start
-  return findNextPosition(transaction, currentAttributes, null, right, index)
+  const marker = findMarker(parent, index)
+  if (marker) {
+    const pos = new ItemTextListPosition(marker.p.left, marker.p, marker.index, currentAttributes)
+    return findNextPosition(transaction, pos, index - marker.index)
+  } else {
+    const pos = new ItemTextListPosition(null, parent._start, 0, currentAttributes)
+    return findNextPosition(transaction, pos, index)
+  }
 }
 
 /**
@@ -122,35 +137,35 @@ const findPosition = (transaction, parent, index) => {
  *
  * @param {Transaction} transaction
  * @param {AbstractType<any>} parent
- * @param {Item|null} left
- * @param {Item|null} right
+ * @param {ItemTextListPosition} currPos
  * @param {Map<string,any>} negatedAttributes
- * @return {ItemListPosition}
  *
  * @private
  * @function
  */
-const insertNegatedAttributes = (transaction, parent, left, right, negatedAttributes) => {
+const insertNegatedAttributes = (transaction, parent, currPos, negatedAttributes) => {
   // check if we really need to remove attributes
   while (
-    right !== null && (
-      right.deleted === true || (
-        right.content.constructor === ContentFormat &&
-        equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (right.content).key), /** @type {ContentFormat} */ (right.content).value)
+    currPos.right !== null && (
+      currPos.right.deleted === true || (
+        currPos.right.content.constructor === ContentFormat &&
+        equalAttrs(negatedAttributes.get(/** @type {ContentFormat} */ (currPos.right.content).key), /** @type {ContentFormat} */ (currPos.right.content).value)
       )
     )
   ) {
-    if (!right.deleted) {
-      negatedAttributes.delete(/** @type {ContentFormat} */ (right.content).key)
+    if (!currPos.right.deleted) {
+      negatedAttributes.delete(/** @type {ContentFormat} */ (currPos.right.content).key)
     }
-    left = right
-    right = right.right
+    currPos.forward()
   }
-  for (const [key, val] of negatedAttributes) {
-    left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentFormat(key, val))
-    left.integrate(transaction)
-  }
-  return { left, right }
+  const doc = transaction.doc
+  const ownClientId = doc.clientID
+  let left = currPos.left
+  const right = currPos.right
+  negatedAttributes.forEach((val, key) => {
+    left = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val))
+    left.integrate(transaction, 0)
+  })
 }
 
 /**
@@ -170,118 +185,112 @@ const updateCurrentAttributes = (currentAttributes, format) => {
 }
 
 /**
- * @param {Item|null} left
- * @param {Item|null} right
- * @param {Map<string,any>} currentAttributes
+ * @param {ItemTextListPosition} currPos
  * @param {Object<string,any>} attributes
- * @return {ItemListPosition}
  *
  * @private
  * @function
  */
-const minimizeAttributeChanges = (left, right, currentAttributes, attributes) => {
+const minimizeAttributeChanges = (currPos, attributes) => {
   // go right while attributes[right.key] === right.value (or right is deleted)
   while (true) {
-    if (right === null) {
+    if (currPos.right === null) {
       break
-    } else if (right.deleted) {
-      // continue
-    } else if (right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (right.content)).key] || null, /** @type {ContentFormat} */ (right.content).value)) {
-      // found a format, update currentAttributes and continue
-      updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
+    } else if (currPos.right.deleted || (currPos.right.content.constructor === ContentFormat && equalAttrs(attributes[(/** @type {ContentFormat} */ (currPos.right.content)).key] || null, /** @type {ContentFormat} */ (currPos.right.content).value))) {
+      //
     } else {
       break
     }
-    left = right
-    right = right.right
+    currPos.forward()
   }
-  return new ItemListPosition(left, right)
 }
 
 /**
  * @param {Transaction} transaction
  * @param {AbstractType<any>} parent
- * @param {Item|null} left
- * @param {Item|null} right
- * @param {Map<string,any>} currentAttributes
+ * @param {ItemTextListPosition} currPos
  * @param {Object<string,any>} attributes
- * @return {ItemInsertionResult}
+ * @return {Map<string,any>}
  *
  * @private
  * @function
  **/
-const insertAttributes = (transaction, parent, left, right, currentAttributes, attributes) => {
+const insertAttributes = (transaction, parent, currPos, attributes) => {
+  const doc = transaction.doc
+  const ownClientId = doc.clientID
   const negatedAttributes = new Map()
   // insert format-start items
   for (const key in attributes) {
     const val = attributes[key]
-    const currentVal = currentAttributes.get(key) || null
+    const currentVal = currPos.currentAttributes.get(key) || null
     if (!equalAttrs(currentVal, val)) {
       // save negated attribute (set null if currentVal undefined)
       negatedAttributes.set(key, currentVal)
-      left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentFormat(key, val))
-      left.integrate(transaction)
+      const { left, right } = currPos
+      currPos.right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentFormat(key, val))
+      currPos.right.integrate(transaction, 0)
+      currPos.forward()
     }
   }
-  return new ItemInsertionResult(left, right, negatedAttributes)
+  return negatedAttributes
 }
 
 /**
  * @param {Transaction} transaction
  * @param {AbstractType<any>} parent
- * @param {Item|null} left
- * @param {Item|null} right
- * @param {Map<string,any>} currentAttributes
+ * @param {ItemTextListPosition} currPos
  * @param {string|object} text
  * @param {Object<string,any>} attributes
- * @return {ItemListPosition}
  *
  * @private
  * @function
  **/
-const insertText = (transaction, parent, left, right, currentAttributes, text, attributes) => {
-  for (const [key] of currentAttributes) {
+const insertText = (transaction, parent, currPos, text, attributes) => {
+  currPos.currentAttributes.forEach((val, key) => {
     if (attributes[key] === undefined) {
       attributes[key] = null
     }
-  }
-  const minPos = minimizeAttributeChanges(left, right, currentAttributes, attributes)
-  const insertPos = insertAttributes(transaction, parent, minPos.left, minPos.right, currentAttributes, attributes)
-  left = insertPos.left
-  right = insertPos.right
+  })
+  const doc = transaction.doc
+  const ownClientId = doc.clientID
+  minimizeAttributeChanges(currPos, attributes)
+  const negatedAttributes = insertAttributes(transaction, parent, currPos, attributes)
   // insert content
-  const content = text.constructor === String ? new ContentString(text) : new ContentEmbed(text)
-  left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, content)
-  left.integrate(transaction)
-  return insertNegatedAttributes(transaction, parent, left, insertPos.right, insertPos.negatedAttributes)
+  const content = text.constructor === String ? new ContentString(/** @type {string} */ (text)) : new ContentEmbed(text)
+  let { left, right, index } = currPos
+  if (parent._searchMarker) {
+    updateMarkerChanges(parent._searchMarker, currPos.index, content.getLength())
+  }
+  right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, content)
+  right.integrate(transaction, 0)
+  currPos.right = right
+  currPos.index = index
+  currPos.forward()
+  insertNegatedAttributes(transaction, parent, currPos, negatedAttributes)
 }
 
 /**
  * @param {Transaction} transaction
  * @param {AbstractType<any>} parent
- * @param {Item|null} left
- * @param {Item|null} right
- * @param {Map<string,any>} currentAttributes
+ * @param {ItemTextListPosition} currPos
  * @param {number} length
  * @param {Object<string,any>} attributes
- * @return {ItemListPosition}
  *
  * @private
  * @function
  */
-const formatText = (transaction, parent, left, right, currentAttributes, length, attributes) => {
-  const minPos = minimizeAttributeChanges(left, right, currentAttributes, attributes)
-  const insertPos = insertAttributes(transaction, parent, minPos.left, minPos.right, currentAttributes, attributes)
-  const negatedAttributes = insertPos.negatedAttributes
-  left = insertPos.left
-  right = insertPos.right
+const formatText = (transaction, parent, currPos, length, attributes) => {
+  const doc = transaction.doc
+  const ownClientId = doc.clientID
+  minimizeAttributeChanges(currPos, attributes)
+  const negatedAttributes = insertAttributes(transaction, parent, currPos, attributes)
   // iterate until first non-format or null is found
   // delete all formats with attributes[format.key] != null
-  while (length > 0 && right !== null) {
-    if (!right.deleted) {
-      switch (right.content.constructor) {
+  while (length > 0 && currPos.right !== null) {
+    if (!currPos.right.deleted) {
+      switch (currPos.right.content.constructor) {
         case ContentFormat: {
-          const { key, value } = /** @type {ContentFormat} */ (right.content)
+          const { key, value } = /** @type {ContentFormat} */ (currPos.right.content)
           const attr = attributes[key]
           if (attr !== undefined) {
             if (equalAttrs(attr, value)) {
@@ -289,22 +298,20 @@ const formatText = (transaction, parent, left, right, currentAttributes, length,
             } else {
               negatedAttributes.set(key, value)
             }
-            right.delete(transaction)
+            currPos.right.delete(transaction)
           }
-          updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
           break
         }
         case ContentEmbed:
         case ContentString:
-          if (length < right.length) {
-            getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length))
+          if (length < currPos.right.length) {
+            getItemCleanStart(transaction, createID(currPos.right.id.client, currPos.right.id.clock + length))
           }
-          length -= right.length
+          length -= currPos.right.length
           break
       }
     }
-    left = right
-    right = right.right
+    currPos.forward()
   }
   // Quill just assumes that the editor starts with a newline and that it always
   // ends with a newline. We only insert that newline when a new newline is
@@ -314,44 +321,153 @@ const formatText = (transaction, parent, left, right, currentAttributes, length,
     for (; length > 0; length--) {
       newlines += '\n'
     }
-    left = new Item(nextID(transaction), left, left === null ? null : left.lastId, right, right === null ? null : right.id, parent, null, new ContentString(newlines))
-    left.integrate(transaction)
+    currPos.right = new Item(createID(ownClientId, getState(doc.store, ownClientId)), currPos.left, currPos.left && currPos.left.lastId, currPos.right, currPos.right && currPos.right.id, parent, null, new ContentString(newlines))
+    currPos.right.integrate(transaction, 0)
+    currPos.forward()
   }
-  return insertNegatedAttributes(transaction, parent, left, right, negatedAttributes)
+  insertNegatedAttributes(transaction, parent, currPos, negatedAttributes)
+}
+
+/**
+ * Call this function after string content has been deleted in order to
+ * clean up formatting Items.
+ *
+ * @param {Transaction} transaction
+ * @param {Item} start
+ * @param {Item|null} end exclusive end, automatically iterates to the next Content Item
+ * @param {Map<string,any>} startAttributes
+ * @param {Map<string,any>} endAttributes This attribute is modified!
+ * @return {number} The amount of formatting Items deleted.
+ *
+ * @function
+ */
+const cleanupFormattingGap = (transaction, start, end, startAttributes, endAttributes) => {
+  while (end && end.content.constructor !== ContentString && end.content.constructor !== ContentEmbed) {
+    if (!end.deleted && end.content.constructor === ContentFormat) {
+      updateCurrentAttributes(endAttributes, /** @type {ContentFormat} */ (end.content))
+    }
+    end = end.right
+  }
+  let cleanups = 0
+  while (start !== end) {
+    if (!start.deleted) {
+      const content = start.content
+      switch (content.constructor) {
+        case ContentFormat: {
+          const { key, value } = /** @type {ContentFormat} */ (content)
+          if ((endAttributes.get(key) || null) !== value || (startAttributes.get(key) || null) === value) {
+            // Either this format is overwritten or it is not necessary because the attribute already existed.
+            start.delete(transaction)
+            cleanups++
+          }
+          break
+        }
+      }
+    }
+    start = /** @type {Item} */ (start.right)
+  }
+  return cleanups
 }
 
 /**
  * @param {Transaction} transaction
- * @param {Item|null} left
- * @param {Item|null} right
- * @param {Map<string,any>} currentAttributes
+ * @param {Item | null} item
+ */
+const cleanupContextlessFormattingGap = (transaction, item) => {
+  // iterate until item.right is null or content
+  while (item && item.right && (item.right.deleted || (item.right.content.constructor !== ContentString && item.right.content.constructor !== ContentEmbed))) {
+    item = item.right
+  }
+  const attrs = new Set()
+  // iterate back until a content item is found
+  while (item && (item.deleted || (item.content.constructor !== ContentString && item.content.constructor !== ContentEmbed))) {
+    if (!item.deleted && item.content.constructor === ContentFormat) {
+      const key = /** @type {ContentFormat} */ (item.content).key
+      if (attrs.has(key)) {
+        item.delete(transaction)
+      } else {
+        attrs.add(key)
+      }
+    }
+    item = item.left
+  }
+}
+
+/**
+ * This function is experimental and subject to change / be removed.
+ *
+ * Ideally, we don't need this function at all. Formatting attributes should be cleaned up
+ * automatically after each change. This function iterates twice over the complete YText type
+ * and removes unnecessary formatting attributes. This is also helpful for testing.
+ *
+ * This function won't be exported anymore as soon as there is confidence that the YText type works as intended.
+ *
+ * @param {YText} type
+ * @return {number} How many formatting attributes have been cleaned up.
+ */
+export const cleanupYTextFormatting = type => {
+  let res = 0
+  transact(/** @type {Doc} */ (type.doc), transaction => {
+    let start = /** @type {Item} */ (type._start)
+    let end = type._start
+    let startAttributes = map.create()
+    const currentAttributes = map.copy(startAttributes)
+    while (end) {
+      if (end.deleted === false) {
+        switch (end.content.constructor) {
+          case ContentFormat:
+            updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (end.content))
+            break
+          case ContentEmbed:
+          case ContentString:
+            res += cleanupFormattingGap(transaction, start, end, startAttributes, currentAttributes)
+            startAttributes = map.copy(currentAttributes)
+            start = end
+            break
+        }
+      }
+      end = end.right
+    }
+  })
+  return res
+}
+
+/**
+ * @param {Transaction} transaction
+ * @param {ItemTextListPosition} currPos
  * @param {number} length
- * @return {ItemListPosition}
+ * @return {ItemTextListPosition}
  *
  * @private
  * @function
  */
-const deleteText = (transaction, left, right, currentAttributes, length) => {
-  while (length > 0 && right !== null) {
-    if (right.deleted === false) {
-      switch (right.content.constructor) {
-        case ContentFormat:
-          updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (right.content))
-          break
+const deleteText = (transaction, currPos, length) => {
+  const startLength = length
+  const startAttrs = map.copy(currPos.currentAttributes)
+  const start = currPos.right
+  while (length > 0 && currPos.right !== null) {
+    if (currPos.right.deleted === false) {
+      switch (currPos.right.content.constructor) {
         case ContentEmbed:
         case ContentString:
-          if (length < right.length) {
-            getItemCleanStart(transaction, createID(right.id.client, right.id.clock + length))
+          if (length < currPos.right.length) {
+            getItemCleanStart(transaction, createID(currPos.right.id.client, currPos.right.id.clock + length))
           }
-          length -= right.length
-          right.delete(transaction)
+          length -= currPos.right.length
+          currPos.right.delete(transaction)
           break
       }
     }
-    left = right
-    right = right.right
+    currPos.forward()
   }
-  return { left, right }
+  if (start) {
+    cleanupFormattingGap(transaction, start, currPos.right, startAttrs, map.copy(currPos.currentAttributes))
+  }
+  const parent = /** @type {AbstractType<any>} */ (/** @type {Item} */ (currPos.left || currPos.right).parent)
+  if (parent._searchMarker) {
+    updateMarkerChanges(parent._searchMarker, currPos.index, -startLength + length)
+  }
+  return currPos
 }
 
 /**
@@ -385,7 +501,7 @@ const deleteText = (transaction, left, right, currentAttributes, length) => {
  * @typedef {Object} DeltaItem
  * @property {number|undefined} DeltaItem.delete
  * @property {number|undefined} DeltaItem.retain
- * @property {string|undefined} DeltaItem.string
+ * @property {string|undefined} DeltaItem.insert
  * @property {Object<string,any>} DeltaItem.attributes
  */
 
@@ -400,7 +516,6 @@ export class YTextEvent extends YEvent {
   constructor (ytext, transaction) {
     super(ytext, transaction)
     /**
-     * @private
      * @type {Array<DeltaItem>|null}
      */
     this._delta = null
@@ -452,11 +567,11 @@ export class YTextEvent extends YEvent {
                 op = { insert }
                 if (currentAttributes.size > 0) {
                   op.attributes = {}
-                  for (const [key, value] of currentAttributes) {
+                  currentAttributes.forEach((value, key) => {
                     if (value !== null) {
                       op.attributes[key] = value
                     }
-                  }
+                  })
                 }
                 insert = ''
                 break
@@ -612,11 +727,19 @@ export class YText extends AbstractType {
     /**
      * Array of pending operations on this type
      * @type {Array<function():void>?}
-     * @private
      */
     this._pending = string !== undefined ? [() => this.insert(0, string)] : []
+    /**
+     * @type {Array<ArraySearchMarker>}
+     */
+    this._searchMarker = []
   }
 
+  /**
+   * Number of characters of this text type.
+   *
+   * @type {number}
+   */
   get length () {
     return this._length
   }
@@ -624,8 +747,6 @@ export class YText extends AbstractType {
   /**
    * @param {Doc} y
    * @param {Item} item
-   *
-   * @private
    */
   _integrate (y, item) {
     super._integrate(y, item)
@@ -646,11 +767,60 @@ export class YText extends AbstractType {
    *
    * @param {Transaction} transaction
    * @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
-   *
-   * @private
    */
   _callObserver (transaction, parentSubs) {
-    callTypeObservers(this, transaction, new YTextEvent(this, transaction))
+    super._callObserver(transaction, parentSubs)
+    const event = new YTextEvent(this, transaction)
+    const doc = transaction.doc
+    // If a remote change happened, we try to cleanup potential formatting duplicates.
+    if (!transaction.local) {
+      // check if another formatting item was inserted
+      let foundFormattingItem = false
+      for (const [client, afterClock] of transaction.afterState.entries()) {
+        const clock = transaction.beforeState.get(client) || 0
+        if (afterClock === clock) {
+          continue
+        }
+        iterateStructs(transaction, /** @type {Array<Item|GC>} */ (doc.store.clients.get(client)), clock, afterClock, item => {
+          if (!item.deleted && /** @type {Item} */ (item).content.constructor === ContentFormat) {
+            foundFormattingItem = true
+          }
+        })
+        if (foundFormattingItem) {
+          break
+        }
+      }
+      if (!foundFormattingItem) {
+        iterateDeletedStructs(transaction, transaction.deleteSet, item => {
+          if (item instanceof GC || foundFormattingItem) {
+            return
+          }
+          if (item.parent === this && item.content.constructor === ContentFormat) {
+            foundFormattingItem = true
+          }
+        })
+      }
+      transact(doc, (t) => {
+        if (foundFormattingItem) {
+          // If a formatting item was inserted, we simply clean the whole type.
+          // We need to compute currentAttributes for the current position anyway.
+          cleanupYTextFormatting(this)
+        } else {
+          // If no formatting attribute was inserted, we can make due with contextless
+          // formatting cleanups.
+          // Contextless: it is not necessary to compute currentAttributes for the affected position.
+          iterateDeletedStructs(t, t.deleteSet, item => {
+            if (item instanceof GC) {
+              return
+            }
+            if (item.parent === this) {
+              cleanupContextlessFormattingGap(t, item)
+            }
+          })
+        }
+      })
+    }
+    callTypeObservers(this, transaction, event)
   }
 
   /**
@@ -687,17 +857,16 @@ export class YText extends AbstractType {
    * Apply a {@link Delta} on this shared YText type.
    *
    * @param {any} delta The changes to apply on this element.
+   * @param {object}  [opts]
+   * @param {boolean} [opts.sanitize] Sanitize input delta. Removes ending newlines if set to true.
+   *
    *
    * @public
    */
-  applyDelta (delta) {
+  applyDelta (delta, { sanitize = true } = {}) {
     if (this.doc !== null) {
       transact(this.doc, transaction => {
-        /**
-         * @type {ItemListPosition}
-         */
-        let pos = new ItemListPosition(null, this._start)
-        const currentAttributes = new Map()
+        const currPos = new ItemTextListPosition(null, this._start, 0, new Map())
         for (let i = 0; i < delta.length; i++) {
           const op = delta[i]
           if (op.insert !== undefined) {
@@ -706,14 +875,14 @@ export class YText extends AbstractType {
             // there is a newline at the end of the content.
             // If we omit this step, clients will see a different number of
             // paragraphs, but nothing bad will happen.
-            const ins = (typeof op.insert === 'string' && i === delta.length - 1 && pos.right === null && op.insert.slice(-1) === '\n') ? op.insert.slice(0, -1) : op.insert
+            const ins = (!sanitize && typeof op.insert === 'string' && i === delta.length - 1 && currPos.right === null && op.insert.slice(-1) === '\n') ? op.insert.slice(0, -1) : op.insert
             if (typeof ins !== 'string' || ins.length > 0) {
-              pos = insertText(transaction, this, pos.left, pos.right, currentAttributes, ins, op.attributes || {})
+              insertText(transaction, this, currPos, ins, op.attributes || {})
             }
           } else if (op.retain !== undefined) {
-            pos = formatText(transaction, this, pos.left, pos.right, currentAttributes, op.retain, op.attributes || {})
+            formatText(transaction, this, currPos, op.retain, op.attributes || {})
           } else if (op.delete !== undefined) {
-            pos = deleteText(transaction, pos.left, pos.right, currentAttributes, op.delete)
+            deleteText(transaction, currPos, op.delete)
           }
         }
       })
@@ -749,10 +918,10 @@ export class YText extends AbstractType {
          */
         const attributes = {}
         let addAttributes = false
-        for (const [key, value] of currentAttributes) {
+        currentAttributes.forEach((value, key) => {
           addAttributes = true
           attributes[key] = value
-        }
+        })
         /**
          * @type {Object<string,any>}
          */
@@ -795,12 +964,24 @@ export class YText extends AbstractType {
               str += /** @type {ContentString} */ (n.content).str
               break
             }
-            case ContentEmbed:
+            case ContentEmbed: {
               packStr()
-              ops.push({
+              /**
+               * @type {Object<string,any>}
+               */
+              const op = {
                 insert: /** @type {ContentEmbed} */ (n.content).embed
-              })
+              }
+              if (currentAttributes.size > 0) {
+                const attrs = /** @type {Object<string,any>} */ ({})
+                op.attributes = attrs
+                currentAttributes.forEach((value, key) => {
+                  attrs[key] = value
+                })
+              }
+              ops.push(op)
               break
+            }
             case ContentFormat:
               if (isVisible(n, snapshot)) {
                 packStr()
@@ -833,13 +1014,13 @@ export class YText extends AbstractType {
     const y = this.doc
     if (y !== null) {
       transact(y, transaction => {
-        const { left, right, currentAttributes } = findPosition(transaction, this, index)
+        const pos = findPosition(transaction, this, index)
         if (!attributes) {
           attributes = {}
           // @ts-ignore
-          currentAttributes.forEach((v, k) => { attributes[k] = v })
+          pos.currentAttributes.forEach((v, k) => { attributes[k] = v })
         }
-        insertText(transaction, this, left, right, currentAttributes, text, attributes)
+        insertText(transaction, this, pos, text, attributes)
       })
     } else {
       /** @type {Array<function>} */ (this._pending).push(() => this.insert(index, text, attributes))
@@ -863,8 +1044,8 @@ export class YText extends AbstractType {
     const y = this.doc
     if (y !== null) {
       transact(y, transaction => {
-        const { left, right, currentAttributes } = findPosition(transaction, this, index)
-        insertText(transaction, this, left, right, currentAttributes, embed, attributes)
+        const pos = findPosition(transaction, this, index)
+        insertText(transaction, this, pos, embed, attributes)
       })
     } else {
       /** @type {Array<function>} */ (this._pending).push(() => this.insertEmbed(index, embed, attributes))
@@ -886,8 +1067,7 @@ export class YText extends AbstractType {
     const y = this.doc
     if (y !== null) {
       transact(y, transaction => {
-        const { left, right, currentAttributes } = findPosition(transaction, this, index)
-        deleteText(transaction, left, right, currentAttributes, length)
+        deleteText(transaction, findPosition(transaction, this, index), length)
       })
     } else {
       /** @type {Array<function>} */ (this._pending).push(() => this.delete(index, length))
@@ -905,14 +1085,17 @@ export class YText extends AbstractType {
    * @public
    */
   format (index, length, attributes) {
+    if (length === 0) {
+      return
+    }
     const y = this.doc
     if (y !== null) {
       transact(y, transaction => {
-        const { left, right, currentAttributes } = findPosition(transaction, this, index)
-        if (right === null) {
+        const pos = findPosition(transaction, this, index)
+        if (pos.right === null) {
           return
         }
-        formatText(transaction, this, left, right, currentAttributes, length, attributes)
+        formatText(transaction, this, pos, length, attributes)
       })
     } else {
       /** @type {Array<function>} */ (this._pending).push(() => this.format(index, length, attributes))
@@ -920,17 +1103,15 @@ export class YText extends AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
-   *
-   * @private
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YTextRefID)
+    encoder.writeTypeRef(YTextRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YText}
  *
  * @private

src/types/YXmlElement.js

@@ -8,12 +8,9 @@ import {
   typeMapGetAll,
   typeListForEach,
   YXmlElementRefID,
-  Snapshot, Doc, Item // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Snapshot, Doc, Item // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 /**
  * An YXmlElement imitates the behavior of a
  * {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}.
@@ -27,7 +24,6 @@ export class YXmlElement extends YXmlFragment {
     this.nodeName = nodeName
     /**
      * @type {Map<string, any>|null}
-     * @private
      */
     this._prelimAttrs = new Map()
   }
@@ -41,7 +37,6 @@ export class YXmlElement extends YXmlFragment {
    *
    * @param {Doc} y The Yjs instance
    * @param {Item} item
-   * @private
    */
   _integrate (y, item) {
     super._integrate(y, item)
@@ -55,7 +50,6 @@ export class YXmlElement extends YXmlFragment {
    * Creates an Item with the same effect as this Item (without position effect)
    *
    * @return {YXmlElement}
-   * @private
    */
   _copy () {
     return new YXmlElement(this.nodeName)
@@ -184,20 +178,18 @@ export class YXmlElement extends YXmlFragment {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @private
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YXmlElementRefID)
-    encoding.writeVarString(encoder, this.nodeName)
+    encoder.writeTypeRef(YXmlElementRefID)
+    encoder.writeKey(this.nodeName)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlElement}
  *
- * @private
  * @function
  */
-export const readYXmlElement = decoder => new YXmlElement(decoding.readVarString(decoder))
+export const readYXmlElement = decoder => new YXmlElement(decoder.readKey())

src/types/YXmlFragment.js

@@ -14,12 +14,9 @@ import {
   YXmlFragmentRefID,
   callTypeObservers,
   transact,
-  Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
-
 /**
  * Define the elements to which a set of CSS queries apply.
  * {@link https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors|CSS_Selectors}
@@ -48,7 +45,7 @@ import * as decoding from 'lib0/decoding.js' // eslint-disable-line
  * Can be created with {@link YXmlFragment#createTreeWalker}
  *
  * @public
- * @implements {IterableIterator}
+ * @implements {Iterable<YXmlElement|YXmlText|YXmlElement|YXmlHook>}
  */
 export class YXmlTreeWalker {
   /**
@@ -81,10 +78,10 @@ export class YXmlTreeWalker {
      * @type {Item|null}
      */
     let n = this._currentNode
-    let type = /** @type {ContentType} */ (n.content).type
+    let type = /** @type {any} */ (n.content).type
     if (n !== null && (!this._firstCall || n.deleted || !this._filter(type))) { // if first call, we check if we can use the first item
       do {
-        type = /** @type {ContentType} */ (n.content).type
+        type = /** @type {any} */ (n.content).type
         if (!n.deleted && (type.constructor === YXmlElement || type.constructor === YXmlFragment) && type._start !== null) {
           // walk down in the tree
           n = type._start
@@ -97,7 +94,7 @@ export class YXmlTreeWalker {
             } else if (n.parent === this._root) {
               n = null
             } else {
-              n = n.parent._item
+              n = /** @type {AbstractType<any>} */ (n.parent)._item
             }
           }
         }
@@ -127,7 +124,6 @@ export class YXmlFragment extends AbstractType {
     super()
     /**
      * @type {Array<any>|null}
-     * @private
      */
     this._prelimContent = []
   }
@@ -141,7 +137,6 @@ export class YXmlFragment extends AbstractType {
    *
    * @param {Doc} y The Yjs instance
    * @param {Item} item
-   * @private
    */
   _integrate (y, item) {
     super._integrate(y, item)
@@ -224,7 +219,6 @@ export class YXmlFragment extends AbstractType {
 
   /**
    * Creates YXmlEvent and calls observers.
-   * @private
    *
    * @param {Transaction} transaction
    * @param {Set<null|string>} parentSubs Keys changed on this type. `null` if list was modified.
@@ -328,16 +322,15 @@ export class YXmlFragment extends AbstractType {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @private
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YXmlFragmentRefID)
+    encoder.writeTypeRef(YXmlFragmentRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlFragment}
  *
  * @private

src/types/YXmlHook.js

@@ -1,10 +1,9 @@
 
 import {
   YMap,
-  YXmlHookRefID
+  YXmlHookRefID,
+  AbstractUpdateDecoder, AbstractUpdateEncoder // eslint-disable-line
 } from '../internals.js'
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 
 /**
  * You can manage binding to a custom type with YXmlHook.
@@ -25,8 +24,6 @@ export class YXmlHook extends YMap {
 
   /**
    * Creates an Item with the same effect as this Item (without position effect)
-   *
-   * @private
    */
   _copy () {
     return new YXmlHook(this.hookName)
@@ -68,23 +65,20 @@ export class YXmlHook extends YMap {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @param {encoding.Encoder} encoder The encoder to write data to.
-   *
-   * @private
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    */
   _write (encoder) {
-    super._write(encoder)
-    encoding.writeVarUint(encoder, YXmlHookRefID)
-    encoding.writeVarString(encoder, this.hookName)
+    encoder.writeTypeRef(YXmlHookRefID)
+    encoder.writeKey(this.hookName)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlHook}
  *
  * @private
  * @function
  */
 export const readYXmlHook = decoder =>
-  new YXmlHook(decoding.readVarString(decoder))
+  new YXmlHook(decoder.readKey())

src/types/YXmlText.js

@@ -1,8 +1,9 @@
 
-import { YText, YXmlTextRefID } from '../internals.js'
-
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
+import {
+  YText,
+  YXmlTextRefID,
+  AbstractUpdateDecoder, AbstractUpdateEncoder // eslint-disable-line
+} from '../internals.js'
 
 /**
  * Represents text in a Dom Element. In the future this type will also handle
@@ -78,17 +79,15 @@ export class YXmlText extends YText {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
-   *
-   * @private
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YXmlTextRefID)
+    encoder.writeTypeRef(YXmlTextRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlText}
  *
  * @private

src/utils/AbstractConnector.js

@@ -0,0 +1,26 @@
+
+import { Observable } from 'lib0/observable.js'
+
+import {
+  Doc // eslint-disable-line
+} from '../internals.js'
+
+/**
+ * This is an abstract interface that all Connectors should implement to keep them interchangeable.
+ *
+ * @note This interface is experimental and it is not advised to actually inherit this class.
+ *       It just serves as typing information.
+ *
+ * @extends {Observable<any>}
+ */
+export class AbstractConnector extends Observable {
+  /**
+   * @param {Doc} ydoc
+   * @param {any} awareness
+   */
+  constructor (ydoc, awareness) {
+    super()
+    this.doc = ydoc
+    this.awareness = awareness
+  }
+}

src/utils/DeleteSet.js

@@ -1,11 +1,10 @@
 
 import {
   findIndexSS,
-  createID,
   getState,
   splitItem,
   iterateStructs,
-  Item, GC, StructStore, Transaction, ID // eslint-disable-line
+  AbstractUpdateDecoder, AbstractDSDecoder, AbstractDSEncoder, DSDecoderV2, DSEncoderV2, Item, GC, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as array from 'lib0/array.js'
@@ -42,7 +41,6 @@ export class DeleteSet {
   constructor () {
     /**
      * @type {Map<number,Array<DeleteItem>>}
-     * @private
      */
     this.clients = new Map()
   }
@@ -164,14 +162,15 @@ export const mergeDeleteSets = dss => {
 
 /**
  * @param {DeleteSet} ds
- * @param {ID} id
+ * @param {number} client
+ * @param {number} clock
  * @param {number} length
  *
  * @private
  * @function
  */
-export const addToDeleteSet = (ds, id, length) => {
-  map.setIfUndefined(ds.clients, id.client, () => []).push(new DeleteItem(id.clock, length))
+export const addToDeleteSet = (ds, client, clock, length) => {
+  map.setIfUndefined(ds.clients, client, () => []).push(new DeleteItem(clock, length))
 }
 
 export const createDeleteSet = () => new DeleteSet()
@@ -211,28 +210,29 @@ export const createDeleteSetFromStructStore = ss => {
 }
 
 /**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractDSEncoder} encoder
  * @param {DeleteSet} ds
  *
  * @private
  * @function
  */
 export const writeDeleteSet = (encoder, ds) => {
-  encoding.writeVarUint(encoder, ds.clients.size)
+  encoding.writeVarUint(encoder.restEncoder, ds.clients.size)
   ds.clients.forEach((dsitems, client) => {
-    encoding.writeVarUint(encoder, client)
+    encoder.resetDsCurVal()
+    encoding.writeVarUint(encoder.restEncoder, client)
     const len = dsitems.length
-    encoding.writeVarUint(encoder, len)
+    encoding.writeVarUint(encoder.restEncoder, len)
     for (let i = 0; i < len; i++) {
       const item = dsitems[i]
-      encoding.writeVarUint(encoder, item.clock)
-      encoding.writeVarUint(encoder, item.len)
+      encoder.writeDsClock(item.clock)
+      encoder.writeDsLen(item.len)
     }
   })
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractDSDecoder} decoder
  * @return {DeleteSet}
  *
  * @private
@@ -240,19 +240,27 @@ export const writeDeleteSet = (encoder, ds) => {
  */
 export const readDeleteSet = decoder => {
   const ds = new DeleteSet()
-  const numClients = decoding.readVarUint(decoder)
+  const numClients = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < numClients; i++) {
-    const client = decoding.readVarUint(decoder)
-    const numberOfDeletes = decoding.readVarUint(decoder)
-    for (let i = 0; i < numberOfDeletes; i++) {
-      addToDeleteSet(ds, createID(client, decoding.readVarUint(decoder)), decoding.readVarUint(decoder))
+    decoder.resetDsCurVal()
+    const client = decoding.readVarUint(decoder.restDecoder)
+    const numberOfDeletes = decoding.readVarUint(decoder.restDecoder)
+    if (numberOfDeletes > 0) {
+      const dsField = map.setIfUndefined(ds.clients, client, () => [])
+      for (let i = 0; i < numberOfDeletes; i++) {
+        dsField.push(new DeleteItem(decoder.readDsClock(), decoder.readDsLen()))
+      }
     }
   }
   return ds
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @todo YDecoder also contains references to String and other Decoders. Would make sense to exchange YDecoder.toUint8Array for YDecoder.DsToUint8Array()..
+ */
+
+/**
+ * @param {AbstractDSDecoder} decoder
  * @param {Transaction} transaction
  * @param {StructStore} store
  *
@@ -261,18 +269,19 @@ export const readDeleteSet = decoder => {
  */
 export const readAndApplyDeleteSet = (decoder, transaction, store) => {
   const unappliedDS = new DeleteSet()
-  const numClients = decoding.readVarUint(decoder)
+  const numClients = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < numClients; i++) {
-    const client = decoding.readVarUint(decoder)
-    const numberOfDeletes = decoding.readVarUint(decoder)
+    decoder.resetDsCurVal()
+    const client = decoding.readVarUint(decoder.restDecoder)
+    const numberOfDeletes = decoding.readVarUint(decoder.restDecoder)
     const structs = store.clients.get(client) || []
     const state = getState(store, client)
     for (let i = 0; i < numberOfDeletes; i++) {
-      const clock = decoding.readVarUint(decoder)
-      const len = decoding.readVarUint(decoder)
+      const clock = decoder.readDsClock()
+      const clockEnd = clock + decoder.readDsLen()
       if (clock < state) {
-        if (state < clock + len) {
-          addToDeleteSet(unappliedDS, createID(client, state), clock + len - state)
+        if (state < clockEnd) {
+          addToDeleteSet(unappliedDS, client, state, clockEnd - state)
         }
         let index = findIndexSS(structs, clock)
         /**
@@ -289,10 +298,10 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => {
         while (index < structs.length) {
           // @ts-ignore
           struct = structs[index++]
-          if (struct.id.clock < clock + len) {
+          if (struct.id.clock < clockEnd) {
             if (!struct.deleted) {
-              if (clock + len < struct.id.clock + struct.length) {
-                structs.splice(index, 0, splitItem(transaction, struct, clock + len - struct.id.clock))
+              if (clockEnd < struct.id.clock + struct.length) {
+                structs.splice(index, 0, splitItem(transaction, struct, clockEnd - struct.id.clock))
               }
               struct.delete(transaction)
             }
@@ -301,14 +310,14 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => {
           }
         }
       } else {
-        addToDeleteSet(unappliedDS, createID(client, clock), len)
+        addToDeleteSet(unappliedDS, client, clock, clockEnd - clock)
       }
     }
   }
   if (unappliedDS.clients.size > 0) {
     // TODO: no need for encoding+decoding ds anymore
-    const unappliedDSEncoder = encoding.createEncoder()
+    const unappliedDSEncoder = new DSEncoderV2()
     writeDeleteSet(unappliedDSEncoder, unappliedDS)
-    store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toUint8Array(unappliedDSEncoder)))
+    store.pendingDeleteReaders.push(new DSDecoderV2(decoding.createDecoder((unappliedDSEncoder.toUint8Array()))))
   }
 }

src/utils/Doc.js

@@ -10,12 +10,24 @@ import {
   YMap,
   YXmlFragment,
   transact,
-  Item, Transaction, YEvent // eslint-disable-line
+  ContentDoc, Item, Transaction, YEvent // eslint-disable-line
 } from '../internals.js'
 
 import { Observable } from 'lib0/observable.js'
 import * as random from 'lib0/random.js'
 import * as map from 'lib0/map.js'
+import * as array from 'lib0/array.js'
+
+export const generateNewClientId = random.uint32
+
+/**
+ * @typedef {Object} DocOpts
+ * @property {boolean} [DocOpts.gc=true] Disable garbage collection (default: gc=true)
+ * @property {function(Item):boolean} [DocOpts.gcFilter] Will be called before an Item is garbage collected. Return false to keep the Item.
+ * @property {string} [DocOpts.guid] Define a globally unique identifier for this document
+ * @property {any} [DocOpts.meta] Any kind of meta information you want to associate with this document. If this is a subdocument, remote peers will store the meta information as well.
+ * @property {boolean} [DocOpts.autoLoad] If a subdocument, automatically load document. If this is a subdocument, remote peers will load the document as well automatically.
+ */
 
 /**
  * A Yjs instance handles the state of shared data.
@@ -23,13 +35,14 @@ import * as map from 'lib0/map.js'
  */
 export class Doc extends Observable {
   /**
-   * @param {Object} conf configuration
-   * @param {boolean} [conf.gc] Disable garbage collection (default: gc=true)
+   * @param {DocOpts} [opts] configuration
    */
-  constructor ({ gc = true } = {}) {
+  constructor ({ guid = random.uuidv4(), gc = true, gcFilter = () => true, meta = null, autoLoad = false } = {}) {
     super()
     this.gc = gc
-    this.clientID = random.uint32()
+    this.gcFilter = gcFilter
+    this.clientID = generateNewClientId()
+    this.guid = guid
     /**
      * @type {Map<string, AbstractType<YEvent>>}
      */
@@ -37,14 +50,49 @@ export class Doc extends Observable {
     this.store = new StructStore()
     /**
      * @type {Transaction | null}
-     * @private
      */
     this._transaction = null
     /**
      * @type {Array<Transaction>}
-     * @private
      */
     this._transactionCleanups = []
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocs = new Set()
+    /**
+     * If this document is a subdocument - a document integrated into another document - then _item is defined.
+     * @type {Item?}
+     */
+    this._item = null
+    this.shouldLoad = autoLoad
+    this.autoLoad = autoLoad
+    this.meta = meta
+  }
+
+  /**
+   * Notify the parent document that you request to load data into this subdocument (if it is a subdocument).
+   *
+   * `load()` might be used in the future to request any provider to load the most current data.
+   *
+   * It is safe to call `load()` multiple times.
+   */
+  load () {
+    const item = this._item
+    if (item !== null && !this.shouldLoad) {
+      transact(/** @type {any} */ (item.parent).doc, transaction => {
+        transaction.subdocsLoaded.add(this)
+      }, null, true)
+    }
+    this.shouldLoad = true
+  }
+
+  getSubdocs () {
+    return this.subdocs
+  }
+
+  getSubdocGuids () {
+    return new Set(Array.from(this.subdocs).map(doc => doc.guid))
   }
 
   /**
@@ -103,6 +151,7 @@ export class Doc extends Observable {
         t._map = type._map
         type._map.forEach(/** @param {Item?} n */ n => {
           for (; n !== null; n = n.left) {
+            // @ts-ignore
             n.parent = t
           }
         })
@@ -123,62 +172,97 @@ export class Doc extends Observable {
 
   /**
    * @template T
-   * @param {string} name
+   * @param {string} [name]
    * @return {YArray<T>}
    *
    * @public
    */
-  getArray (name) {
+  getArray (name = '') {
     // @ts-ignore
     return this.get(name, YArray)
   }
 
   /**
-   * @param {string} name
+   * @param {string} [name]
    * @return {YText}
    *
    * @public
    */
-  getText (name) {
+  getText (name = '') {
     // @ts-ignore
     return this.get(name, YText)
   }
 
   /**
-   * @param {string} name
+   * @param {string} [name]
    * @return {YMap<any>}
    *
    * @public
    */
-  getMap (name) {
+  getMap (name = '') {
     // @ts-ignore
     return this.get(name, YMap)
   }
 
   /**
-   * @param {string} name
+   * @param {string} [name]
    * @return {YXmlFragment}
    *
    * @public
    */
-  getXmlFragment (name) {
+  getXmlFragment (name = '') {
     // @ts-ignore
     return this.get(name, YXmlFragment)
   }
 
   /**
-   * Emit `destroy` event and unregister all event handlers.
+   * Converts the entire document into a js object, recursively traversing each yjs type
    *
-   * @protected
+   * @return {Object<string, any>}
+   */
+  toJSON () {
+    /**
+     * @type {Object<string, any>}
+     */
+    const doc = {}
+
+    this.share.forEach((value, key) => {
+      doc[key] = value.toJSON()
+    })
+
+    return doc
+  }
+
+  /**
+   * Emit `destroy` event and unregister all event handlers.
    */
   destroy () {
+    array.from(this.subdocs).forEach(subdoc => subdoc.destroy())
+    const item = this._item
+    if (item !== null) {
+      this._item = null
+      const content = /** @type {ContentDoc} */ (item.content)
+      if (item.deleted) {
+        // @ts-ignore
+        content.doc = null
+      } else {
+        content.doc = new Doc({ guid: this.guid, ...content.opts })
+        content.doc._item = item
+      }
+      transact(/** @type {any} */ (item).parent.doc, transaction => {
+        if (!item.deleted) {
+          transaction.subdocsAdded.add(content.doc)
+        }
+        transaction.subdocsRemoved.add(this)
+      }, null, true)
+    }
     this.emit('destroyed', [true])
     super.destroy()
   }
 
   /**
    * @param {string} eventName
-   * @param {function} f
+   * @param {function(...any):any} f
    */
   on (eventName, f) {
     super.on(eventName, f)

src/utils/ID.js

@@ -81,7 +81,7 @@ export const readID = decoder =>
  */
 export const findRootTypeKey = type => {
   // @ts-ignore _y must be defined, otherwise unexpected case
-  for (const [key, value] of type.doc.share) {
+  for (const [key, value] of type.doc.share.entries()) {
     if (value === type) {
       return key
     }

src/utils/PermanentUserData.js

@@ -5,11 +5,11 @@ import {
   readDeleteSet,
   writeDeleteSet,
   createDeleteSet,
-  ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line
+  DSEncoderV1, DSDecoderV1, ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line
 } from '../internals.js'
 
 import * as decoding from 'lib0/decoding.js'
-import * as encoding from 'lib0/encoding.js'
+
 import { mergeDeleteSets, isDeleted } from './DeleteSet.js'
 
 export class PermanentUserData {
@@ -46,12 +46,12 @@ export class PermanentUserData {
         event.changes.added.forEach(item => {
           item.content.getContent().forEach(encodedDs => {
             if (encodedDs instanceof Uint8Array) {
-              this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(decoding.createDecoder(encodedDs))]))
+              this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(new DSDecoderV1(decoding.createDecoder(encodedDs)))]))
             }
           })
         })
       })
-      this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(decoding.createDecoder(encodedDs)))))
+      this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(new DSDecoderV1(encodedDs)))))
       ids.observe(/** @param {YArrayEvent<any>} event */ event =>
         event.changes.added.forEach(item => item.content.getContent().forEach(addClientId))
       )
@@ -97,11 +97,11 @@ export class PermanentUserData {
               user.get('ids').push([clientid])
             }
           })
-          const encoder = encoding.createEncoder()
+          const encoder = new DSEncoderV1()
           const ds = this.dss.get(userDescription)
           if (ds) {
             writeDeleteSet(encoder, ds)
-            user.get('ds').push([encoding.toUint8Array(encoder)])
+            user.get('ds').push([encoder.toUint8Array()])
           }
         }
       }, 0)
@@ -111,9 +111,9 @@ export class PermanentUserData {
         const yds = user.get('ds')
         const ds = transaction.deleteSet
         if (transaction.local && ds.clients.size > 0 && filter(transaction, ds)) {
-          const encoder = encoding.createEncoder()
+          const encoder = new DSEncoderV1()
           writeDeleteSet(encoder, ds)
-          yds.push([encoding.toUint8Array(encoder)])
+          yds.push([encoder.toUint8Array()])
         }
       })
     })
@@ -132,7 +132,7 @@ export class PermanentUserData {
    * @return {string | null}
    */
   getUserByDeletedId (id) {
-    for (const [userDescription, ds] of this.dss) {
+    for (const [userDescription, ds] of this.dss.entries()) {
       if (isDeleted(ds, id)) {
         return userDescription
       }

src/utils/RelativePosition.js

@@ -1,12 +1,12 @@
 
 import {
-  createID,
   writeID,
   readID,
   compareIDs,
   getState,
   findRootTypeKey,
   Item,
+  createID,
   ContentType,
   followRedone,
   ID, Doc, AbstractType // eslint-disable-line
@@ -107,7 +107,7 @@ export const createRelativePosition = (type, item) => {
   if (type._item === null) {
     tname = findRootTypeKey(type)
   } else {
-    typeid = type._item.id
+    typeid = createID(type._item.id.client, type._item.id.clock)
   }
   return new RelativePosition(typeid, tname, item)
 }
@@ -227,7 +227,7 @@ export const createAbsolutePositionFromRelativePosition = (rpos, doc) => {
     if (!(right instanceof Item)) {
       return null
     }
-    type = right.parent
+    type = /** @type {AbstractType<any>} */ (right.parent)
     if (type._item === null || !type._item.deleted) {
       index = right.deleted || !right.countable ? 0 : res.diff
       let n = right.left

src/utils/Snapshot.js

@@ -4,21 +4,25 @@ import {
   createDeleteSetFromStructStore,
   getStateVector,
   getItemCleanStart,
-  createID,
   iterateDeletedStructs,
   writeDeleteSet,
   writeStateVector,
   readDeleteSet,
   readStateVector,
   createDeleteSet,
+  createID,
   getState,
-  Transaction, Doc, DeleteSet, Item // eslint-disable-line
+  findIndexSS,
+  UpdateEncoderV2,
+  DefaultDSEncoder,
+  applyUpdateV2,
+  AbstractDSDecoder, AbstractDSEncoder, DSEncoderV2, DSDecoderV1, DSDecoderV2, Transaction, Doc, DeleteSet, Item // eslint-disable-line
 } from '../internals.js'
 
 import * as map from 'lib0/map.js'
 import * as set from 'lib0/set.js'
-import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
+import * as encoding from 'lib0/encoding.js'
 
 export class Snapshot {
   /**
@@ -28,13 +32,11 @@ export class Snapshot {
   constructor (ds, sv) {
     /**
      * @type {DeleteSet}
-     * @private
      */
     this.ds = ds
     /**
      * State Map
      * @type {Map<number,number>}
-     * @private
      */
     this.sv = sv
   }
@@ -53,12 +55,12 @@ export const equalSnapshots = (snap1, snap2) => {
   if (sv1.size !== sv2.size || ds1.size !== ds2.size) {
     return false
   }
-  for (const [key, value] of sv1) {
+  for (const [key, value] of sv1.entries()) {
     if (sv2.get(key) !== value) {
       return false
     }
   }
-  for (const [client, dsitems1] of ds1) {
+  for (const [client, dsitems1] of ds1.entries()) {
     const dsitems2 = ds2.get(client) || []
     if (dsitems1.length !== dsitems2.length) {
       return false
@@ -76,23 +78,35 @@ export const equalSnapshots = (snap1, snap2) => {
 
 /**
  * @param {Snapshot} snapshot
+ * @param {AbstractDSEncoder} [encoder]
  * @return {Uint8Array}
  */
-export const encodeSnapshot = snapshot => {
-  const encoder = encoding.createEncoder()
+export const encodeSnapshotV2 = (snapshot, encoder = new DSEncoderV2()) => {
   writeDeleteSet(encoder, snapshot.ds)
   writeStateVector(encoder, snapshot.sv)
-  return encoding.toUint8Array(encoder)
+  return encoder.toUint8Array()
+}
+
+/**
+ * @param {Snapshot} snapshot
+ * @return {Uint8Array}
+ */
+export const encodeSnapshot = snapshot => encodeSnapshotV2(snapshot, new DefaultDSEncoder())
+
+/**
+ * @param {Uint8Array} buf
+ * @param {AbstractDSDecoder} [decoder]
+ * @return {Snapshot}
+ */
+export const decodeSnapshotV2 = (buf, decoder = new DSDecoderV2(decoding.createDecoder(buf))) => {
+  return new Snapshot(readDeleteSet(decoder), readStateVector(decoder))
 }
 
 /**
  * @param {Uint8Array} buf
  * @return {Snapshot}
  */
-export const decodeSnapshot = buf => {
-  const decoder = decoding.createDecoder(buf)
-  return new Snapshot(readDeleteSet(decoder), readStateVector(decoder))
-}
+export const decodeSnapshot = buf => decodeSnapshotV2(buf, new DSDecoderV1(decoding.createDecoder(buf)))
 
 /**
  * @param {DeleteSet} ds
@@ -138,3 +152,51 @@ export const splitSnapshotAffectedStructs = (transaction, snapshot) => {
     meta.add(snapshot)
   }
 }
+
+/**
+ * @param {Doc} originDoc
+ * @param {Snapshot} snapshot
+ * @param {Doc} [newDoc] Optionally, you may define the Yjs document that receives the data from originDoc
+ * @return {Doc}
+ */
+export const createDocFromSnapshot = (originDoc, snapshot, newDoc = new Doc()) => {
+  if (originDoc.gc) {
+    // we should not try to restore a GC-ed document, because some of the restored items might have their content deleted
+    throw new Error('originDoc must not be garbage collected')
+  }
+  const { sv, ds } = snapshot
+
+  const encoder = new UpdateEncoderV2()
+  originDoc.transact(transaction => {
+    let size = 0
+    sv.forEach(clock => {
+      if (clock > 0) {
+        size++
+      }
+    })
+    encoding.writeVarUint(encoder.restEncoder, size)
+    // splitting the structs before writing them to the encoder
+    for (const [client, clock] of sv) {
+      if (clock === 0) {
+        continue
+      }
+      if (clock < getState(originDoc.store, client)) {
+        getItemCleanStart(transaction, createID(client, clock))
+      }
+      const structs = originDoc.store.clients.get(client) || []
+      const lastStructIndex = findIndexSS(structs, clock - 1)
+      // write # encoded structs
+      encoding.writeVarUint(encoder.restEncoder, lastStructIndex + 1)
+      encoder.writeClient(client)
+      // first clock written is 0
+      encoding.writeVarUint(encoder.restEncoder, 0)
+      for (let i = 0; i <= lastStructIndex; i++) {
+        structs[i].write(encoder, 0)
+      }
+    }
+    writeDeleteSet(encoder, ds)
+  })
+
+  applyUpdateV2(newDoc, encoder.toUint8Array(), 'snapshot')
+  return newDoc
+}

src/utils/StructStore.js

@@ -2,18 +2,16 @@
 import {
   GC,
   splitItem,
-  GCRef, ItemRef, Transaction, ID, Item // eslint-disable-line
+  Transaction, ID, Item, DSDecoderV2 // eslint-disable-line
 } from '../internals.js'
 
 import * as math from 'lib0/math.js'
 import * as error from 'lib0/error.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
 
 export class StructStore {
   constructor () {
     /**
      * @type {Map<number,Array<GC|Item>>}
-     * @private
      */
     this.clients = new Map()
     /**
@@ -22,20 +20,17 @@ export class StructStore {
      * We could shift the array of refs instead, but shift is incredible
      * slow in Chrome for arrays with more than 100k elements
      * @see tryResumePendingStructRefs
-     * @type {Map<number,{i:number,refs:Array<GCRef|ItemRef>}>}
-     * @private
+     * @type {Map<number,{i:number,refs:Array<GC|Item>}>}
      */
     this.pendingClientsStructRefs = new Map()
     /**
      * Stack of pending structs waiting for struct dependencies
      * Maximum length of stack is structReaders.size
-     * @type {Array<GCRef|ItemRef>}
-     * @private
+     * @type {Array<GC|Item>}
      */
     this.pendingStack = []
     /**
-     * @type {Array<decoding.Decoder>}
-     * @private
+     * @type {Array<DSDecoderV2>}
      */
     this.pendingDeleteReaders = []
   }
@@ -118,7 +113,7 @@ export const addStruct = (store, struct) => {
 
 /**
  * Perform a binary search on a sorted array
- * @param {Array<any>} structs
+ * @param {Array<Item|GC>} structs
  * @param {number} clock
  * @return {number}
  *
@@ -128,10 +123,18 @@ export const addStruct = (store, struct) => {
 export const findIndexSS = (structs, clock) => {
   let left = 0
   let right = structs.length - 1
+  let mid = structs[right]
+  let midclock = mid.id.clock
+  if (midclock === clock) {
+    return right
+  }
+  // @todo does it even make sense to pivot the search?
+  // If a good split misses, it might actually increase the time to find the correct item.
+  // Currently, the only advantage is that search with pivoting might find the item on the first try.
+  let midindex = math.floor((clock / (midclock + mid.length - 1)) * right) // pivoting the search
   while (left <= right) {
-    const midindex = math.floor((left + right) / 2)
-    const mid = structs[midindex]
-    const midclock = mid.id.clock
+    mid = structs[midindex]
+    midclock = mid.id.clock
     if (midclock <= clock) {
       if (clock < midclock + mid.length) {
         return midindex
@@ -140,6 +143,7 @@ export const findIndexSS = (structs, clock) => {
     } else {
       right = midindex - 1
     }
+    midindex = math.floor((left + right) / 2)
   }
   // Always check state before looking for a struct in StructStore
   // Therefore the case of not finding a struct is unexpected
@@ -167,16 +171,10 @@ export const find = (store, id) => {
 
 /**
  * Expects that id is actually in store. This function throws or is an infinite loop otherwise.
- *
- * @param {StructStore} store
- * @param {ID} id
- * @return {Item}
- *
  * @private
  * @function
  */
-// @ts-ignore
-export const getItem = (store, id) => find(store, id)
+export const getItem = /** @type {function(StructStore,ID):Item} */ (find)
 
 /**
  * @param {Transaction} transaction

src/utils/Transaction.js

@@ -1,7 +1,6 @@
 
 import {
   getState,
-  createID,
   writeStructsFromTransaction,
   writeDeleteSet,
   DeleteSet,
@@ -10,13 +9,15 @@ import {
   findIndexSS,
   callEventHandlerListeners,
   Item,
-  ID, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line
+  generateNewClientId,
+  createID,
+  AbstractUpdateEncoder, GC, StructStore, UpdateEncoderV2, DefaultUpdateEncoder, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
 import * as map from 'lib0/map.js'
 import * as math from 'lib0/math.js'
 import * as set from 'lib0/set.js'
+import * as logging from 'lib0/logging.js'
 import { callAll } from 'lib0/function.js'
 
 /**
@@ -84,10 +85,9 @@ export class Transaction {
      */
     this.changedParentTypes = new Map()
     /**
-     * @type {Set<ID>}
-     * @private
+     * @type {Array<AbstractStruct>}
      */
-    this._mergeStructs = new Set()
+    this._mergeStructs = []
     /**
      * @type {any}
      */
@@ -102,21 +102,34 @@ export class Transaction {
      * @type {boolean}
      */
     this.local = local
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocsAdded = new Set()
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocsRemoved = new Set()
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocsLoaded = new Set()
   }
 }
 
 /**
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Transaction} transaction
+ * @return {boolean} Whether data was written.
  */
-export const computeUpdateMessageFromTransaction = transaction => {
+export const writeUpdateMessageFromTransaction = (encoder, transaction) => {
   if (transaction.deleteSet.clients.size === 0 && !map.any(transaction.afterState, (clock, client) => transaction.beforeState.get(client) !== clock)) {
-    return null
+    return false
   }
-  const encoder = encoding.createEncoder()
   sortAndMergeDeleteSet(transaction.deleteSet)
   writeStructsFromTransaction(encoder, transaction)
   writeDeleteSet(encoder, transaction.deleteSet)
-  return encoder
+  return true
 }
 
 /**
@@ -145,6 +158,85 @@ export const addChangedTypeToTransaction = (transaction, type, parentSub) => {
   }
 }
 
+/**
+ * @param {Array<AbstractStruct>} structs
+ * @param {number} pos
+ */
+const tryToMergeWithLeft = (structs, pos) => {
+  const left = structs[pos - 1]
+  const right = structs[pos]
+  if (left.deleted === right.deleted && left.constructor === right.constructor) {
+    if (left.mergeWith(right)) {
+      structs.splice(pos, 1)
+      if (right instanceof Item && right.parentSub !== null && /** @type {AbstractType<any>} */ (right.parent)._map.get(right.parentSub) === right) {
+        /** @type {AbstractType<any>} */ (right.parent)._map.set(right.parentSub, /** @type {Item} */ (left))
+      }
+    }
+  }
+}
+
+/**
+ * @param {DeleteSet} ds
+ * @param {StructStore} store
+ * @param {function(Item):boolean} gcFilter
+ */
+const tryGcDeleteSet = (ds, store, gcFilter) => {
+  for (const [client, deleteItems] of ds.clients.entries()) {
+    const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
+    for (let di = deleteItems.length - 1; di >= 0; di--) {
+      const deleteItem = deleteItems[di]
+      const endDeleteItemClock = deleteItem.clock + deleteItem.len
+      for (
+        let si = findIndexSS(structs, deleteItem.clock), struct = structs[si];
+        si < structs.length && struct.id.clock < endDeleteItemClock;
+        struct = structs[++si]
+      ) {
+        const struct = structs[si]
+        if (deleteItem.clock + deleteItem.len <= struct.id.clock) {
+          break
+        }
+        if (struct instanceof Item && struct.deleted && !struct.keep && gcFilter(struct)) {
+          struct.gc(store, false)
+        }
+      }
+    }
+  }
+}
+
+/**
+ * @param {DeleteSet} ds
+ * @param {StructStore} store
+ */
+const tryMergeDeleteSet = (ds, store) => {
+  // try to merge deleted / gc'd items
+  // merge from right to left for better efficiecy and so we don't miss any merge targets
+  ds.clients.forEach((deleteItems, client) => {
+    const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
+    for (let di = deleteItems.length - 1; di >= 0; di--) {
+      const deleteItem = deleteItems[di]
+      // start with merging the item next to the last deleted item
+      const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1))
+      for (
+        let si = mostRightIndexToCheck, struct = structs[si];
+        si > 0 && struct.id.clock >= deleteItem.clock;
+        struct = structs[--si]
+      ) {
+        tryToMergeWithLeft(structs, si)
+      }
+    }
+  })
+}
+
+/**
+ * @param {DeleteSet} ds
+ * @param {StructStore} store
+ * @param {function(Item):boolean} gcFilter
+ */
+export const tryGc = (ds, store, gcFilter) => {
+  tryGcDeleteSet(ds, store, gcFilter)
+  tryMergeDeleteSet(ds, store)
+}
+
 /**
  * @param {Array<Transaction>} transactionCleanups
  * @param {number} i
@@ -155,6 +247,7 @@ const cleanupTransactions = (transactionCleanups, i) => {
     const doc = transaction.doc
     const store = doc.store
     const ds = transaction.deleteSet
+    const mergeStructs = transaction._mergeStructs
     try {
       sortAndMergeDeleteSet(ds)
       transaction.afterState = getStateVector(transaction.doc.store)
@@ -201,83 +294,31 @@ const cleanupTransactions = (transactionCleanups, i) => {
       })
       callAll(fs, [])
     } finally {
-      /**
-       * @param {Array<AbstractStruct>} structs
-       * @param {number} pos
-       */
-      const tryToMergeWithLeft = (structs, pos) => {
-        const left = structs[pos - 1]
-        const right = structs[pos]
-        if (left.deleted === right.deleted && left.constructor === right.constructor) {
-          if (left.mergeWith(right)) {
-            structs.splice(pos, 1)
-            if (right instanceof Item && right.parentSub !== null && right.parent._map.get(right.parentSub) === right) {
-              right.parent._map.set(right.parentSub, /** @type {Item} */ (left))
-            }
-          }
-        }
-      }
       // Replace deleted items with ItemDeleted / GC.
       // This is where content is actually remove from the Yjs Doc.
       if (doc.gc) {
-        for (const [client, deleteItems] of ds.clients) {
-          const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client))
-          for (let di = deleteItems.length - 1; di >= 0; di--) {
-            const deleteItem = deleteItems[di]
-            const endDeleteItemClock = deleteItem.clock + deleteItem.len
-            for (
-              let si = findIndexSS(structs, deleteItem.clock), struct = structs[si];
-              si < structs.length && struct.id.clock < endDeleteItemClock;
-              struct = structs[++si]
-            ) {
-              const struct = structs[si]
-              if (deleteItem.clock + deleteItem.len <= struct.id.clock) {
-                break
-              }
-              if (struct instanceof Item && struct.deleted && !struct.keep) {
-                struct.gc(store, false)
-              }
-            }
-          }
-        }
-      }
-      // try to merge deleted / gc'd items
-      // merge from right to left for better efficiecy and so we don't miss any merge targets
-      for (const [client, deleteItems] of ds.clients) {
-        const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client))
-        for (let di = deleteItems.length - 1; di >= 0; di--) {
-          const deleteItem = deleteItems[di]
-          // start with merging the item next to the last deleted item
-          const mostRightIndexToCheck = math.min(structs.length - 1, 1 + findIndexSS(structs, deleteItem.clock + deleteItem.len - 1))
-          for (
-            let si = mostRightIndexToCheck, struct = structs[si];
-            si > 0 && struct.id.clock >= deleteItem.clock;
-            struct = structs[--si]
-          ) {
-            tryToMergeWithLeft(structs, si)
-          }
-        }
+        tryGcDeleteSet(ds, store, doc.gcFilter)
       }
+      tryMergeDeleteSet(ds, store)
 
       // on all affected store.clients props, try to merge
-      for (const [client, clock] of transaction.afterState) {
+      transaction.afterState.forEach((clock, client) => {
         const beforeClock = transaction.beforeState.get(client) || 0
         if (beforeClock !== clock) {
-          const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client))
+          const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
           // we iterate from right to left so we can safely remove entries
           const firstChangePos = math.max(findIndexSS(structs, beforeClock), 1)
           for (let i = structs.length - 1; i >= firstChangePos; i--) {
             tryToMergeWithLeft(structs, i)
           }
         }
-      }
+      })
       // try to merge mergeStructs
       // @todo: it makes more sense to transform mergeStructs to a DS, sort it, and merge from right to left
       //        but at the moment DS does not handle duplicates
-      for (const mid of transaction._mergeStructs) {
-        const client = mid.client
-        const clock = mid.clock
-        const structs = /** @type {Array<AbstractStruct>} */ (store.clients.get(client))
+      for (let i = 0; i < mergeStructs.length; i++) {
+        const { client, clock } = mergeStructs[i].id
+        const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
         const replacedStructPos = findIndexSS(structs, clock)
         if (replacedStructPos + 1 < structs.length) {
           tryToMergeWithLeft(structs, replacedStructPos + 1)
@@ -286,16 +327,35 @@ const cleanupTransactions = (transactionCleanups, i) => {
           tryToMergeWithLeft(structs, replacedStructPos)
         }
       }
+      if (!transaction.local && transaction.afterState.get(doc.clientID) !== transaction.beforeState.get(doc.clientID)) {
+        doc.clientID = generateNewClientId()
+        logging.print(logging.ORANGE, logging.BOLD, '[yjs] ', logging.UNBOLD, logging.RED, 'Changed the client-id because another client seems to be using it.')
+      }
       // @todo Merge all the transactions into one and provide send the data as a single update message
       doc.emit('afterTransactionCleanup', [transaction, doc])
       if (doc._observers.has('update')) {
-        const updateMessage = computeUpdateMessageFromTransaction(transaction)
-        if (updateMessage !== null) {
-          doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc])
+        const encoder = new DefaultUpdateEncoder()
+        const hasContent = writeUpdateMessageFromTransaction(encoder, transaction)
+        if (hasContent) {
+          doc.emit('update', [encoder.toUint8Array(), transaction.origin, doc])
         }
       }
+      if (doc._observers.has('updateV2')) {
+        const encoder = new UpdateEncoderV2()
+        const hasContent = writeUpdateMessageFromTransaction(encoder, transaction)
+        if (hasContent) {
+          doc.emit('updateV2', [encoder.toUint8Array(), transaction.origin, doc])
+        }
+      }
+      transaction.subdocsAdded.forEach(subdoc => doc.subdocs.add(subdoc))
+      transaction.subdocsRemoved.forEach(subdoc => doc.subdocs.delete(subdoc))
+
+      doc.emit('subdocs', [{ loaded: transaction.subdocsLoaded, added: transaction.subdocsAdded, removed: transaction.subdocsRemoved }])
+      transaction.subdocsRemoved.forEach(subdoc => subdoc.destroy())
+
       if (transactionCleanups.length <= i + 1) {
         doc._transactionCleanups = []
+        doc.emit('afterAllTransactions', [doc, transactionCleanups])
       } else {
         cleanupTransactions(transactionCleanups, i + 1)
       }
@@ -310,7 +370,6 @@ const cleanupTransactions = (transactionCleanups, i) => {
  * @param {function(Transaction):void} f
  * @param {any} [origin=true]
  *
- * @private
  * @function
  */
 export const transact = (doc, f, origin = null, local = true) => {
@@ -320,6 +379,9 @@ export const transact = (doc, f, origin = null, local = true) => {
     initialCall = true
     doc._transaction = new Transaction(doc, origin, local)
     transactionCleanups.push(doc._transaction)
+    if (transactionCleanups.length === 1) {
+      doc.emit('beforeAllTransactions', [doc])
+    }
     doc.emit('beforeTransaction', [doc._transaction, doc])
   }
   try {

src/utils/UndoManager.js

@@ -3,14 +3,14 @@ import {
   iterateDeletedStructs,
   keepItem,
   transact,
+  createID,
   redoItem,
   iterateStructs,
   isParentOf,
-  createID,
   followRedone,
   getItemCleanStart,
   getState,
-  Transaction, Doc, Item, GC, DeleteSet, AbstractType // eslint-disable-line
+  ID, Transaction, Doc, Item, GC, DeleteSet, AbstractType // eslint-disable-line
 } from '../internals.js'
 
 import * as time from 'lib0/time.js'
@@ -19,13 +19,13 @@ import { Observable } from 'lib0/observable.js'
 class StackItem {
   /**
    * @param {DeleteSet} ds
-   * @param {number} start clock start of the local client
-   * @param {number} len
+   * @param {Map<number,number>} beforeState
+   * @param {Map<number,number>} afterState
    */
-  constructor (ds, start, len) {
+  constructor (ds, beforeState, afterState) {
     this.ds = ds
-    this.start = start
-    this.len = len
+    this.beforeState = beforeState
+    this.afterState = afterState
     /**
      * Use this to save and restore metadata like selection range
      */
@@ -50,27 +50,58 @@ const popStackItem = (undoManager, stack, eventType) => {
   transact(doc, transaction => {
     while (stack.length > 0 && result === null) {
       const store = doc.store
-      const clientID = doc.clientID
       const stackItem = /** @type {StackItem} */ (stack.pop())
-      const stackStartClock = stackItem.start
-      const stackEndClock = stackItem.start + stackItem.len
+      /**
+       * @type {Set<Item>}
+       */
       const itemsToRedo = new Set()
-      // @todo iterateStructs should not need the structs parameter
-      const structs = /** @type {Array<GC|Item>} */ (store.clients.get(clientID))
+      /**
+       * @type {Array<Item>}
+       */
+      const itemsToDelete = []
       let performedChange = false
-      if (stackStartClock !== stackEndClock) {
-        // make sure structs don't overlap with the range of created operations [stackItem.start, stackItem.start + stackItem.end)
-        getItemCleanStart(transaction, createID(clientID, stackStartClock))
-        if (stackEndClock < getState(doc.store, clientID)) {
-          getItemCleanStart(transaction, createID(clientID, stackEndClock))
+      stackItem.afterState.forEach((endClock, client) => {
+        const startClock = stackItem.beforeState.get(client) || 0
+        const len = endClock - startClock
+        // @todo iterateStructs should not need the structs parameter
+        const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
+        if (startClock !== endClock) {
+          // make sure structs don't overlap with the range of created operations [stackItem.start, stackItem.start + stackItem.end)
+          // this must be executed before deleted structs are iterated.
+          getItemCleanStart(transaction, createID(client, startClock))
+          if (endClock < getState(doc.store, client)) {
+            getItemCleanStart(transaction, createID(client, endClock))
+          }
+          iterateStructs(transaction, structs, startClock, len, struct => {
+            if (struct instanceof Item) {
+              if (struct.redone !== null) {
+                let { item, diff } = followRedone(store, struct.id)
+                if (diff > 0) {
+                  item = getItemCleanStart(transaction, createID(item.id.client, item.id.clock + diff))
+                }
+                if (item.length > len) {
+                  getItemCleanStart(transaction, createID(item.id.client, endClock))
+                }
+                struct = item
+              }
+              if (!struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) {
+                itemsToDelete.push(struct)
+              }
+            }
+          })
         }
-      }
+      })
       iterateDeletedStructs(transaction, stackItem.ds, struct => {
+        const id = struct.id
+        const clock = id.clock
+        const client = id.client
+        const startClock = stackItem.beforeState.get(client) || 0
+        const endClock = stackItem.afterState.get(client) || 0
         if (
           struct instanceof Item &&
           scope.some(type => isParentOf(type, struct)) &&
           // Never redo structs in [stackItem.start, stackItem.start + stackItem.end) because they were created and deleted in the same capture interval.
-          !(struct.id.client === clientID && struct.id.clock >= stackStartClock && struct.id.clock < stackEndClock)
+          !(clock >= startClock && clock < endClock)
         ) {
           itemsToRedo.add(struct)
         }
@@ -78,27 +109,6 @@ const popStackItem = (undoManager, stack, eventType) => {
       itemsToRedo.forEach(struct => {
         performedChange = redoItem(transaction, struct, itemsToRedo) !== null || performedChange
       })
-      /**
-       * @type {Array<Item>}
-       */
-      const itemsToDelete = []
-      iterateStructs(transaction, structs, stackStartClock, stackItem.len, struct => {
-        if (struct instanceof Item) {
-          if (struct.redone !== null) {
-            let { item, diff } = followRedone(store, struct.id)
-            if (diff > 0) {
-              item = getItemCleanStart(transaction, createID(item.id.client, item.id.clock + diff))
-            }
-            if (item.length > stackItem.len) {
-              getItemCleanStart(transaction, createID(item.id.client, stackEndClock))
-            }
-            struct = item
-          }
-          if (!struct.deleted && scope.some(type => isParentOf(type, /** @type {Item} */ (struct)))) {
-            itemsToDelete.push(struct)
-          }
-        }
-      })
       // We want to delete in reverse order so that children are deleted before
       // parents, so we have more information available when items are filtered.
       for (let i = itemsToDelete.length - 1; i >= 0; i--) {
@@ -113,6 +123,12 @@ const popStackItem = (undoManager, stack, eventType) => {
         undoManager.emit('stack-item-popped', [{ stackItem: result, type: eventType }, undoManager])
       }
     }
+    transaction.changed.forEach((subProps, type) => {
+      // destroy search marker if necessary
+      if (subProps.has(null) && type._searchMarker) {
+        type._searchMarker.length = 0
+      }
+    })
   }, undoManager)
   return result
 }
@@ -141,10 +157,7 @@ export class UndoManager extends Observable {
    * @param {AbstractType<any>|Array<AbstractType<any>>} typeScope Accepts either a single type, or an array of types
    * @param {UndoManagerOptions} options
    */
-  constructor (typeScope, { captureTimeout, deleteFilter = () => true, trackedOrigins = new Set([null]) } = {}) {
-    if (captureTimeout == null) {
-      captureTimeout = 500
-    }
+  constructor (typeScope, { captureTimeout = 500, deleteFilter = () => true, trackedOrigins = new Set([null]) } = {}) {
     super()
     this.scope = typeScope instanceof Array ? typeScope : [typeScope]
     this.deleteFilter = deleteFilter
@@ -181,17 +194,17 @@ export class UndoManager extends Observable {
         // neither undoing nor redoing: delete redoStack
         this.redoStack = []
       }
-      const beforeState = transaction.beforeState.get(this.doc.clientID) || 0
-      const afterState = transaction.afterState.get(this.doc.clientID) || 0
+      const beforeState = transaction.beforeState
+      const afterState = transaction.afterState
       const now = time.getUnixTime()
       if (now - this.lastChange < captureTimeout && stack.length > 0 && !undoing && !redoing) {
         // append change to last stack op
         const lastOp = stack[stack.length - 1]
         lastOp.ds = mergeDeleteSets([lastOp.ds, transaction.deleteSet])
-        lastOp.len = afterState - lastOp.start
+        lastOp.afterState = afterState
       } else {
         // create a new stack op
-        stack.push(new StackItem(transaction.deleteSet, beforeState, afterState - beforeState))
+        stack.push(new StackItem(transaction.deleteSet, beforeState, afterState))
       }
       if (!undoing && !redoing) {
         this.lastChange = now
@@ -199,13 +212,32 @@ export class UndoManager extends Observable {
       // make sure that deleted structs are not gc'd
       iterateDeletedStructs(transaction, transaction.deleteSet, /** @param {Item|GC} item */ item => {
         if (item instanceof Item && this.scope.some(type => isParentOf(type, item))) {
-          keepItem(item)
+          keepItem(item, true)
         }
       })
       this.emit('stack-item-added', [{ stackItem: stack[stack.length - 1], origin: transaction.origin, type: undoing ? 'redo' : 'undo' }, this])
     })
   }
 
+  clear () {
+    this.doc.transact(transaction => {
+      /**
+       * @param {StackItem} stackItem
+       */
+      const clearItem = stackItem => {
+        iterateDeletedStructs(transaction, stackItem.ds, item => {
+          if (item instanceof Item && this.scope.some(type => isParentOf(type, item))) {
+            keepItem(item, false)
+          }
+        })
+      }
+      this.undoStack.forEach(clearItem)
+      this.redoStack.forEach(clearItem)
+    })
+    this.undoStack = []
+    this.redoStack = []
+  }
+
   /**
    * UndoManager merges Undo-StackItem if they are created within time-gap
    * smaller than `options.captureTimeout`. Call `um.stopCapturing()` so that the next

src/utils/UpdateDecoder.js

@@ -0,0 +1,392 @@
+import * as buffer from 'lib0/buffer.js'
+import * as error from 'lib0/error.js'
+import * as decoding from 'lib0/decoding.js'
+import {
+  ID, createID
+} from '../internals.js'
+
+export class AbstractDSDecoder {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    this.restDecoder = decoder
+    error.methodUnimplemented()
+  }
+
+  resetDsCurVal () { }
+
+  /**
+   * @return {number}
+   */
+  readDsClock () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {number}
+   */
+  readDsLen () {
+    error.methodUnimplemented()
+  }
+}
+
+export class AbstractUpdateDecoder extends AbstractDSDecoder {
+  /**
+   * @return {ID}
+   */
+  readLeftID () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {ID}
+   */
+  readRightID () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Read the next client id.
+   * Use this in favor of readID whenever possible to reduce the number of objects created.
+   *
+   * @return {number}
+   */
+  readClient () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readInfo () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {string}
+   */
+  readString () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {boolean} isKey
+   */
+  readParentInfo () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readTypeRef () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @return {number} len
+   */
+  readLen () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {any}
+   */
+  readAny () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  readBuf () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Legacy implementation uses JSON parse. We use any-decoding in v2.
+   *
+   * @return {any}
+   */
+  readJSON () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {string}
+   */
+  readKey () {
+    error.methodUnimplemented()
+  }
+}
+
+export class DSDecoderV1 {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    this.restDecoder = decoder
+  }
+
+  resetDsCurVal () {
+    // nop
+  }
+
+  /**
+   * @return {number}
+   */
+  readDsClock () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * @return {number}
+   */
+  readDsLen () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+}
+
+export class UpdateDecoderV1 extends DSDecoderV1 {
+  /**
+   * @return {ID}
+   */
+  readLeftID () {
+    return createID(decoding.readVarUint(this.restDecoder), decoding.readVarUint(this.restDecoder))
+  }
+
+  /**
+   * @return {ID}
+   */
+  readRightID () {
+    return createID(decoding.readVarUint(this.restDecoder), decoding.readVarUint(this.restDecoder))
+  }
+
+  /**
+   * Read the next client id.
+   * Use this in favor of readID whenever possible to reduce the number of objects created.
+   */
+  readClient () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readInfo () {
+    return decoding.readUint8(this.restDecoder)
+  }
+
+  /**
+   * @return {string}
+   */
+  readString () {
+    return decoding.readVarString(this.restDecoder)
+  }
+
+  /**
+   * @return {boolean} isKey
+   */
+  readParentInfo () {
+    return decoding.readVarUint(this.restDecoder) === 1
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readTypeRef () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @return {number} len
+   */
+  readLen () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * @return {any}
+   */
+  readAny () {
+    return decoding.readAny(this.restDecoder)
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  readBuf () {
+    return buffer.copyUint8Array(decoding.readVarUint8Array(this.restDecoder))
+  }
+
+  /**
+   * Legacy implementation uses JSON parse. We use any-decoding in v2.
+   *
+   * @return {any}
+   */
+  readJSON () {
+    return JSON.parse(decoding.readVarString(this.restDecoder))
+  }
+
+  /**
+   * @return {string}
+   */
+  readKey () {
+    return decoding.readVarString(this.restDecoder)
+  }
+}
+
+export class DSDecoderV2 {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    this.dsCurrVal = 0
+    this.restDecoder = decoder
+  }
+
+  resetDsCurVal () {
+    this.dsCurrVal = 0
+  }
+
+  readDsClock () {
+    this.dsCurrVal += decoding.readVarUint(this.restDecoder)
+    return this.dsCurrVal
+  }
+
+  readDsLen () {
+    const diff = decoding.readVarUint(this.restDecoder) + 1
+    this.dsCurrVal += diff
+    return diff
+  }
+}
+
+export class UpdateDecoderV2 extends DSDecoderV2 {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    super(decoder)
+    /**
+     * List of cached keys. If the keys[id] does not exist, we read a new key
+     * from stringEncoder and push it to keys.
+     *
+     * @type {Array<string>}
+     */
+    this.keys = []
+    decoding.readUint8(decoder) // read feature flag - currently unused
+    this.keyClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.clientDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.leftClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.rightClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.infoDecoder = new decoding.RleDecoder(decoding.readVarUint8Array(decoder), decoding.readUint8)
+    this.stringDecoder = new decoding.StringDecoder(decoding.readVarUint8Array(decoder))
+    this.parentInfoDecoder = new decoding.RleDecoder(decoding.readVarUint8Array(decoder), decoding.readUint8)
+    this.typeRefDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.lenDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder))
+  }
+
+  /**
+   * @return {ID}
+   */
+  readLeftID () {
+    return new ID(this.clientDecoder.read(), this.leftClockDecoder.read())
+  }
+
+  /**
+   * @return {ID}
+   */
+  readRightID () {
+    return new ID(this.clientDecoder.read(), this.rightClockDecoder.read())
+  }
+
+  /**
+   * Read the next client id.
+   * Use this in favor of readID whenever possible to reduce the number of objects created.
+   */
+  readClient () {
+    return this.clientDecoder.read()
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readInfo () {
+    return /** @type {number} */ (this.infoDecoder.read())
+  }
+
+  /**
+   * @return {string}
+   */
+  readString () {
+    return this.stringDecoder.read()
+  }
+
+  /**
+   * @return {boolean}
+   */
+  readParentInfo () {
+    return this.parentInfoDecoder.read() === 1
+  }
+
+  /**
+   * @return {number} An unsigned 8-bit integer
+   */
+  readTypeRef () {
+    return this.typeRefDecoder.read()
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @return {number}
+   */
+  readLen () {
+    return this.lenDecoder.read()
+  }
+
+  /**
+   * @return {any}
+   */
+  readAny () {
+    return decoding.readAny(this.restDecoder)
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  readBuf () {
+    return decoding.readVarUint8Array(this.restDecoder)
+  }
+
+  /**
+   * This is mainly here for legacy purposes.
+   *
+   * Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder.
+   *
+   * @return {any}
+   */
+  readJSON () {
+    return decoding.readAny(this.restDecoder)
+  }
+
+  /**
+   * @return {string}
+   */
+  readKey () {
+    const keyClock = this.keyClockDecoder.read()
+    if (keyClock < this.keys.length) {
+      return this.keys[keyClock]
+    } else {
+      const key = this.stringDecoder.read()
+      this.keys.push(key)
+      return key
+    }
+  }
+}

src/utils/UpdateEncoder.js

@@ -0,0 +1,408 @@
+
+import * as error from 'lib0/error.js'
+import * as encoding from 'lib0/encoding.js'
+
+import {
+  ID // eslint-disable-line
+} from '../internals.js'
+
+export class AbstractDSEncoder {
+  constructor () {
+    this.restEncoder = encoding.createEncoder()
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  toUint8Array () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Resets the ds value to 0.
+   * The v2 encoder uses this information to reset the initial diff value.
+   */
+  resetDsCurVal () { }
+
+  /**
+   * @param {number} clock
+   */
+  writeDsClock (clock) { }
+
+  /**
+   * @param {number} len
+   */
+  writeDsLen (len) { }
+}
+
+export class AbstractUpdateEncoder extends AbstractDSEncoder {
+  /**
+   * @return {Uint8Array}
+   */
+  toUint8Array () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeLeftID (id) { }
+
+  /**
+   * @param {ID} id
+   */
+  writeRightID (id) { }
+
+  /**
+   * Use writeClient and writeClock instead of writeID if possible.
+   * @param {number} client
+   */
+  writeClient (client) { }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeInfo (info) { }
+
+  /**
+   * @param {string} s
+   */
+  writeString (s) { }
+
+  /**
+   * @param {boolean} isYKey
+   */
+  writeParentInfo (isYKey) { }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeTypeRef (info) { }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @param {number} len
+   */
+  writeLen (len) { }
+
+  /**
+   * @param {any} any
+   */
+  writeAny (any) { }
+
+  /**
+   * @param {Uint8Array} buf
+   */
+  writeBuf (buf) { }
+
+  /**
+   * @param {any} embed
+   */
+  writeJSON (embed) { }
+
+  /**
+   * @param {string} key
+   */
+  writeKey (key) { }
+}
+
+export class DSEncoderV1 {
+  constructor () {
+    this.restEncoder = new encoding.Encoder()
+  }
+
+  toUint8Array () {
+    return encoding.toUint8Array(this.restEncoder)
+  }
+
+  resetDsCurVal () {
+    // nop
+  }
+
+  /**
+   * @param {number} clock
+   */
+  writeDsClock (clock) {
+    encoding.writeVarUint(this.restEncoder, clock)
+  }
+
+  /**
+   * @param {number} len
+   */
+  writeDsLen (len) {
+    encoding.writeVarUint(this.restEncoder, len)
+  }
+}
+
+export class UpdateEncoderV1 extends DSEncoderV1 {
+  /**
+   * @param {ID} id
+   */
+  writeLeftID (id) {
+    encoding.writeVarUint(this.restEncoder, id.client)
+    encoding.writeVarUint(this.restEncoder, id.clock)
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeRightID (id) {
+    encoding.writeVarUint(this.restEncoder, id.client)
+    encoding.writeVarUint(this.restEncoder, id.clock)
+  }
+
+  /**
+   * Use writeClient and writeClock instead of writeID if possible.
+   * @param {number} client
+   */
+  writeClient (client) {
+    encoding.writeVarUint(this.restEncoder, client)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeInfo (info) {
+    encoding.writeUint8(this.restEncoder, info)
+  }
+
+  /**
+   * @param {string} s
+   */
+  writeString (s) {
+    encoding.writeVarString(this.restEncoder, s)
+  }
+
+  /**
+   * @param {boolean} isYKey
+   */
+  writeParentInfo (isYKey) {
+    encoding.writeVarUint(this.restEncoder, isYKey ? 1 : 0)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeTypeRef (info) {
+    encoding.writeVarUint(this.restEncoder, info)
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @param {number} len
+   */
+  writeLen (len) {
+    encoding.writeVarUint(this.restEncoder, len)
+  }
+
+  /**
+   * @param {any} any
+   */
+  writeAny (any) {
+    encoding.writeAny(this.restEncoder, any)
+  }
+
+  /**
+   * @param {Uint8Array} buf
+   */
+  writeBuf (buf) {
+    encoding.writeVarUint8Array(this.restEncoder, buf)
+  }
+
+  /**
+   * @param {any} embed
+   */
+  writeJSON (embed) {
+    encoding.writeVarString(this.restEncoder, JSON.stringify(embed))
+  }
+
+  /**
+   * @param {string} key
+   */
+  writeKey (key) {
+    encoding.writeVarString(this.restEncoder, key)
+  }
+}
+
+export class DSEncoderV2 {
+  constructor () {
+    this.restEncoder = new encoding.Encoder() // encodes all the rest / non-optimized
+    this.dsCurrVal = 0
+  }
+
+  toUint8Array () {
+    return encoding.toUint8Array(this.restEncoder)
+  }
+
+  resetDsCurVal () {
+    this.dsCurrVal = 0
+  }
+
+  /**
+   * @param {number} clock
+   */
+  writeDsClock (clock) {
+    const diff = clock - this.dsCurrVal
+    this.dsCurrVal = clock
+    encoding.writeVarUint(this.restEncoder, diff)
+  }
+
+  /**
+   * @param {number} len
+   */
+  writeDsLen (len) {
+    if (len === 0) {
+      error.unexpectedCase()
+    }
+    encoding.writeVarUint(this.restEncoder, len - 1)
+    this.dsCurrVal += len
+  }
+}
+
+export class UpdateEncoderV2 extends DSEncoderV2 {
+  constructor () {
+    super()
+    /**
+     * @type {Map<string,number>}
+     */
+    this.keyMap = new Map()
+    /**
+     * Refers to the next uniqe key-identifier to me used.
+     * See writeKey method for more information.
+     *
+     * @type {number}
+     */
+    this.keyClock = 0
+    this.keyClockEncoder = new encoding.IntDiffOptRleEncoder()
+    this.clientEncoder = new encoding.UintOptRleEncoder()
+    this.leftClockEncoder = new encoding.IntDiffOptRleEncoder()
+    this.rightClockEncoder = new encoding.IntDiffOptRleEncoder()
+    this.infoEncoder = new encoding.RleEncoder(encoding.writeUint8)
+    this.stringEncoder = new encoding.StringEncoder()
+    this.parentInfoEncoder = new encoding.RleEncoder(encoding.writeUint8)
+    this.typeRefEncoder = new encoding.UintOptRleEncoder()
+    this.lenEncoder = new encoding.UintOptRleEncoder()
+  }
+
+  toUint8Array () {
+    const encoder = encoding.createEncoder()
+    encoding.writeUint8(encoder, 0) // this is a feature flag that we might use in the future
+    encoding.writeVarUint8Array(encoder, this.keyClockEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.clientEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.leftClockEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.rightClockEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, encoding.toUint8Array(this.infoEncoder))
+    encoding.writeVarUint8Array(encoder, this.stringEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, encoding.toUint8Array(this.parentInfoEncoder))
+    encoding.writeVarUint8Array(encoder, this.typeRefEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.lenEncoder.toUint8Array())
+    // @note The rest encoder is appended! (note the missing var)
+    encoding.writeUint8Array(encoder, encoding.toUint8Array(this.restEncoder))
+    return encoding.toUint8Array(encoder)
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeLeftID (id) {
+    this.clientEncoder.write(id.client)
+    this.leftClockEncoder.write(id.clock)
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeRightID (id) {
+    this.clientEncoder.write(id.client)
+    this.rightClockEncoder.write(id.clock)
+  }
+
+  /**
+   * @param {number} client
+   */
+  writeClient (client) {
+    this.clientEncoder.write(client)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeInfo (info) {
+    this.infoEncoder.write(info)
+  }
+
+  /**
+   * @param {string} s
+   */
+  writeString (s) {
+    this.stringEncoder.write(s)
+  }
+
+  /**
+   * @param {boolean} isYKey
+   */
+  writeParentInfo (isYKey) {
+    this.parentInfoEncoder.write(isYKey ? 1 : 0)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeTypeRef (info) {
+    this.typeRefEncoder.write(info)
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @param {number} len
+   */
+  writeLen (len) {
+    this.lenEncoder.write(len)
+  }
+
+  /**
+   * @param {any} any
+   */
+  writeAny (any) {
+    encoding.writeAny(this.restEncoder, any)
+  }
+
+  /**
+   * @param {Uint8Array} buf
+   */
+  writeBuf (buf) {
+    encoding.writeVarUint8Array(this.restEncoder, buf)
+  }
+
+  /**
+   * This is mainly here for legacy purposes.
+   *
+   * Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder.
+   *
+   * @param {any} embed
+   */
+  writeJSON (embed) {
+    encoding.writeAny(this.restEncoder, embed)
+  }
+
+  /**
+   * Property keys are often reused. For example, in y-prosemirror the key `bold` might
+   * occur very often. For a 3d application, the key `position` might occur very often.
+   *
+   * We cache these keys in a Map and refer to them via a unique number.
+   *
+   * @param {string} key
+   */
+  writeKey (key) {
+    const clock = this.keyMap.get(key)
+    if (clock === undefined) {
+      this.keyClockEncoder.write(this.keyClock++)
+      this.stringEncoder.write(key)
+    } else {
+      this.keyClockEncoder.write(this.keyClock++)
+    }
+  }
+}

src/utils/YEvent.js

@@ -78,7 +78,7 @@ export class YEvent {
   }
 
   /**
-   * @return {{added:Set<Item>,deleted:Set<Item>,delta:Array<{insert:Array<any>}|{delete:number}|{retain:number}>}}
+   * @return {{added:Set<Item>,deleted:Set<Item>,keys:Map<string,{action:'add'|'update'|'delete',oldValue:any}>,delta:Array<{insert:Array<any>}|{delete:number}|{retain:number}>}}
    */
   get changes () {
     let changes = this._changes
@@ -211,7 +211,7 @@ const getPathTo = (parent, child) => {
     } else {
       // parent is array-ish
       let i = 0
-      let c = child._item.parent._start
+      let c = /** @type {AbstractType<any>} */ (child._item.parent)._start
       while (c !== child._item && c !== null) {
         if (!c.deleted) {
           i++
@@ -220,7 +220,7 @@ const getPathTo = (parent, child) => {
       }
       path.unshift(i)
     }
-    child = child._item.parent
+    child = /** @type {AbstractType<any>} */ (child._item.parent)
   }
   return path
 }

src/utils/encoding.js

@@ -1,7 +1,8 @@
 
 /**
  * @module encoding
- *
+ */
+/*
  * We use the first five bits in the info flag for determining the type of the struct.
  *
  * 0: GC
@@ -16,27 +17,52 @@
 
 import {
   findIndexSS,
-  GCRef,
-  ItemRef,
-  writeID,
-  createID,
-  readID,
   getState,
+  createID,
   getStateVector,
   readAndApplyDeleteSet,
   writeDeleteSet,
   createDeleteSetFromStructStore,
   transact,
-  Doc, Transaction, AbstractStruct, StructStore, ID // eslint-disable-line
+  readItemContent,
+  UpdateDecoderV1,
+  UpdateDecoderV2,
+  UpdateEncoderV1,
+  UpdateEncoderV2,
+  DSDecoderV2,
+  DSEncoderV2,
+  DSDecoderV1,
+  DSEncoderV1,
+  AbstractDSEncoder, AbstractDSDecoder, AbstractUpdateEncoder, AbstractUpdateDecoder, AbstractContent, Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
 import * as binary from 'lib0/binary.js'
+import * as map from 'lib0/map.js'
+
+export let DefaultDSEncoder = DSEncoderV1
+export let DefaultDSDecoder = DSDecoderV1
+export let DefaultUpdateEncoder = UpdateEncoderV1
+export let DefaultUpdateDecoder = UpdateDecoderV1
+
+export const useV1Encoding = () => {
+  DefaultDSEncoder = DSEncoderV1
+  DefaultDSDecoder = DSDecoderV1
+  DefaultUpdateEncoder = UpdateEncoderV1
+  DefaultUpdateDecoder = UpdateDecoderV1
+}
+
+export const useV2Encoding = () => {
+  DefaultDSEncoder = DSEncoderV2
+  DefaultDSDecoder = DSDecoderV2
+  DefaultUpdateEncoder = UpdateEncoderV2
+  DefaultUpdateDecoder = UpdateDecoderV2
+}
 
 /**
- * @param {encoding.Encoder} encoder
- * @param {Array<AbstractStruct>} structs All structs by `client`
+ * @param {AbstractUpdateEncoder} encoder
+ * @param {Array<GC|Item>} structs All structs by `client`
  * @param {number} client
  * @param {number} clock write structs starting with `ID(client,clock)`
  *
@@ -46,41 +72,19 @@ const writeStructs = (encoder, structs, client, clock) => {
   // write first id
   const startNewStructs = findIndexSS(structs, clock)
   // write # encoded structs
-  encoding.writeVarUint(encoder, structs.length - startNewStructs)
-  writeID(encoder, createID(client, clock))
+  encoding.writeVarUint(encoder.restEncoder, structs.length - startNewStructs)
+  encoder.writeClient(client)
+  encoding.writeVarUint(encoder.restEncoder, clock)
   const firstStruct = structs[startNewStructs]
   // write first struct with an offset
-  firstStruct.write(encoder, clock - firstStruct.id.clock, 0)
+  firstStruct.write(encoder, clock - firstStruct.id.clock)
   for (let i = startNewStructs + 1; i < structs.length; i++) {
-    structs[i].write(encoder, 0, 0)
+    structs[i].write(encoder, 0)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
- * @param {number} numOfStructs
- * @param {ID} nextID
- * @return {Array<GCRef|ItemRef>}
- *
- * @private
- * @function
- */
-const readStructRefs = (decoder, numOfStructs, nextID) => {
-  /**
-   * @type {Array<GCRef|ItemRef>}
-   */
-  const refs = []
-  for (let i = 0; i < numOfStructs; i++) {
-    const info = decoding.readUint8(decoder)
-    const ref = (binary.BITS5 & info) === 0 ? new GCRef(decoder, nextID, info) : new ItemRef(decoder, nextID, info)
-    nextID = createID(nextID.client, nextID.clock + ref.length)
-    refs.push(ref)
-  }
-  return refs
-}
-
-/**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {StructStore} store
  * @param {Map<number,number>} _sm
  *
@@ -102,31 +106,93 @@ export const writeClientsStructs = (encoder, store, _sm) => {
     }
   })
   // write # states that were updated
-  encoding.writeVarUint(encoder, sm.size)
-  sm.forEach((clock, client) => {
+  encoding.writeVarUint(encoder.restEncoder, sm.size)
+  // Write items with higher client ids first
+  // This heavily improves the conflict algorithm.
+  Array.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => {
     // @ts-ignore
     writeStructs(encoder, store.clients.get(client), client, clock)
   })
 }
 
 /**
- * @param {decoding.Decoder} decoder The decoder object to read data from.
- * @return {Map<number,Array<GCRef|ItemRef>>}
+ * @param {AbstractUpdateDecoder} decoder The decoder object to read data from.
+ * @param {Map<number,Array<GC|Item>>} clientRefs
+ * @param {Doc} doc
+ * @return {Map<number,Array<GC|Item>>}
  *
  * @private
  * @function
  */
-export const readClientsStructRefs = decoder => {
-  /**
-   * @type {Map<number,Array<GCRef|ItemRef>>}
-   */
-  const clientRefs = new Map()
-  const numOfStateUpdates = decoding.readVarUint(decoder)
+export const readClientsStructRefs = (decoder, clientRefs, doc) => {
+  const numOfStateUpdates = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < numOfStateUpdates; i++) {
-    const numberOfStructs = decoding.readVarUint(decoder)
-    const nextID = readID(decoder)
-    const refs = readStructRefs(decoder, numberOfStructs, nextID)
-    clientRefs.set(nextID.client, refs)
+    const numberOfStructs = decoding.readVarUint(decoder.restDecoder)
+    /**
+     * @type {Array<GC|Item>}
+     */
+    const refs = new Array(numberOfStructs)
+    const client = decoder.readClient()
+    let clock = decoding.readVarUint(decoder.restDecoder)
+    // const start = performance.now()
+    clientRefs.set(client, refs)
+    for (let i = 0; i < numberOfStructs; i++) {
+      const info = decoder.readInfo()
+      if ((binary.BITS5 & info) !== 0) {
+        /**
+         * The optimized implementation doesn't use any variables because inlining variables is faster.
+         * Below a non-optimized version is shown that implements the basic algorithm with
+         * a few comments
+         */
+        const cantCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
+        // If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
+        // and we read the next string as parentYKey.
+        // It indicates how we store/retrieve parent from `y.share`
+        // @type {string|null}
+        const struct = new Item(
+          createID(client, clock),
+          null, // leftd
+          (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null, // origin
+          null, // right
+          (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null, // right origin
+          cantCopyParentInfo ? (decoder.readParentInfo() ? doc.get(decoder.readString()) : decoder.readLeftID()) : null, // parent
+          cantCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub
+          readItemContent(decoder, info) // item content
+        )
+        /* A non-optimized implementation of the above algorithm:
+
+        // The item that was originally to the left of this item.
+        const origin = (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null
+        // The item that was originally to the right of this item.
+        const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null
+        const cantCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
+        const hasParentYKey = cantCopyParentInfo ? decoder.readParentInfo() : false
+        // If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
+        // and we read the next string as parentYKey.
+        // It indicates how we store/retrieve parent from `y.share`
+        // @type {string|null}
+        const parentYKey = cantCopyParentInfo && hasParentYKey ? decoder.readString() : null
+
+        const struct = new Item(
+          createID(client, clock),
+          null, // leftd
+          origin, // origin
+          null, // right
+          rightOrigin, // right origin
+          cantCopyParentInfo && !hasParentYKey ? decoder.readLeftID() : (parentYKey !== null ? doc.get(parentYKey) : null), // parent
+          cantCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub
+          readItemContent(decoder, info) // item content
+        )
+        */
+        refs[i] = struct
+        clock += struct.length
+      } else {
+        const len = decoder.readLen()
+        refs[i] = new GC(createID(client, clock), len)
+        clock += len
+      }
+    }
+    // console.log('time to read: ', performance.now() - start) // @todo remove
   }
   return clientRefs
 }
@@ -157,33 +223,55 @@ export const readClientsStructRefs = decoder => {
  * @function
  */
 const resumeStructIntegration = (transaction, store) => {
-  const stack = store.pendingStack
+  const stack = store.pendingStack // @todo don't forget to append stackhead at the end
   const clientsStructRefs = store.pendingClientsStructRefs
-  // iterate over all struct readers until we are done
-  while (stack.length !== 0 || clientsStructRefs.size !== 0) {
-    if (stack.length === 0) {
-      // take any first struct from clientsStructRefs and put it on the stack
-      const [client, structRefs] = clientsStructRefs.entries().next().value
-      stack.push(structRefs.refs[structRefs.i++])
-      if (structRefs.refs.length === structRefs.i) {
-        clientsStructRefs.delete(client)
+  // sort them so that we take the higher id first, in case of conflicts the lower id will probably not conflict with the id from the higher user.
+  const clientsStructRefsIds = Array.from(clientsStructRefs.keys()).sort((a, b) => a - b)
+  if (clientsStructRefsIds.length === 0) {
+    return
+  }
+  const getNextStructTarget = () => {
+    let nextStructsTarget = /** @type {{i:number,refs:Array<GC|Item>}} */ (clientsStructRefs.get(clientsStructRefsIds[clientsStructRefsIds.length - 1]))
+    while (nextStructsTarget.refs.length === nextStructsTarget.i) {
+      clientsStructRefsIds.pop()
+      if (clientsStructRefsIds.length > 0) {
+        nextStructsTarget = /** @type {{i:number,refs:Array<GC|Item>}} */ (clientsStructRefs.get(clientsStructRefsIds[clientsStructRefsIds.length - 1]))
+      } else {
+        store.pendingClientsStructRefs.clear()
+        return null
       }
     }
-    const ref = stack[stack.length - 1]
-    const m = ref._missing
-    const client = ref.id.client
-    const localClock = getState(store, client)
-    const offset = ref.id.clock < localClock ? localClock - ref.id.clock : 0
-    if (ref.id.clock + offset !== localClock) {
+    return nextStructsTarget
+  }
+  let curStructsTarget = getNextStructTarget()
+  if (curStructsTarget === null && stack.length === 0) {
+    return
+  }
+  /**
+   * @type {GC|Item}
+   */
+  let stackHead = stack.length > 0
+    ? /** @type {GC|Item} */ (stack.pop())
+    : /** @type {any} */ (curStructsTarget).refs[/** @type {any} */ (curStructsTarget).i++]
+  // caching the state because it is used very often
+  const state = new Map()
+  // iterate over all struct readers until we are done
+  while (true) {
+    const localClock = map.setIfUndefined(state, stackHead.id.client, () => getState(store, stackHead.id.client))
+    const offset = stackHead.id.clock < localClock ? localClock - stackHead.id.clock : 0
+    if (stackHead.id.clock + offset !== localClock) {
       // A previous message from this client is missing
       // check if there is a pending structRef with a smaller clock and switch them
-      const structRefs = clientsStructRefs.get(client)
-      if (structRefs !== undefined) {
+      /**
+       * @type {{ refs: Array<GC|Item>, i: number }}
+       */
+      const structRefs = clientsStructRefs.get(stackHead.id.client) || { refs: [], i: 0 }
+      if (structRefs.refs.length !== structRefs.i) {
         const r = structRefs.refs[structRefs.i]
-        if (r.id.clock < ref.id.clock) {
+        if (r.id.clock < stackHead.id.clock) {
           // put ref with smaller clock on stack instead and continue
-          structRefs.refs[structRefs.i] = ref
-          stack[stack.length - 1] = r
+          structRefs.refs[structRefs.i] = stackHead
+          stackHead = r
           // sort the set because this approach might bring the list out of order
           structRefs.refs = structRefs.refs.slice(structRefs.i).sort((r1, r2) => r1.id.clock - r2.id.clock)
           structRefs.i = 0
@@ -191,33 +279,45 @@ const resumeStructIntegration = (transaction, store) => {
         }
       }
       // wait until missing struct is available
+      stack.push(stackHead)
       return
     }
-    while (m.length > 0) {
-      const missing = m[m.length - 1]
-      if (getState(store, missing.client) <= missing.clock) {
-        const client = missing.client
-        // get the struct reader that has the missing struct
-        const structRefs = clientsStructRefs.get(client)
-        if (structRefs === undefined) {
-          // This update message causally depends on another update message.
-          return
-        }
-        stack.push(structRefs.refs[structRefs.i++])
-        if (structRefs.i === structRefs.refs.length) {
-          clientsStructRefs.delete(client)
-        }
-        break
+    const missing = stackHead.getMissing(transaction, store)
+    if (missing === null) {
+      if (offset === 0 || offset < stackHead.length) {
+        stackHead.integrate(transaction, offset)
+        state.set(stackHead.id.client, stackHead.id.clock + stackHead.length)
       }
-      ref._missing.pop()
-    }
-    if (m.length === 0) {
-      if (offset < ref.length) {
-        ref.toStruct(transaction, store, offset).integrate(transaction)
+      // iterate to next stackHead
+      if (stack.length > 0) {
+        stackHead = /** @type {GC|Item} */ (stack.pop())
+      } else if (curStructsTarget !== null && curStructsTarget.i < curStructsTarget.refs.length) {
+        stackHead = /** @type {GC|Item} */ (curStructsTarget.refs[curStructsTarget.i++])
+      } else {
+        curStructsTarget = getNextStructTarget()
+        if (curStructsTarget === null) {
+          // we are done!
+          break
+        } else {
+          stackHead = /** @type {GC|Item} */ (curStructsTarget.refs[curStructsTarget.i++])
+        }
       }
-      stack.pop()
+    } else {
+      // get the struct reader that has the missing struct
+      /**
+       * @type {{ refs: Array<GC|Item>, i: number }}
+       */
+      const structRefs = clientsStructRefs.get(missing) || { refs: [], i: 0 }
+      if (structRefs.refs.length === structRefs.i) {
+        // This update message causally depends on another update message.
+        stack.push(stackHead)
+        return
+      }
+      stack.push(stackHead)
+      stackHead = structRefs.refs[structRefs.i++]
     }
   }
+  store.pendingClientsStructRefs.clear()
 }
 
 /**
@@ -236,7 +336,7 @@ export const tryResumePendingDeleteReaders = (transaction, store) => {
 }
 
 /**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Transaction} transaction
  *
  * @private
@@ -246,14 +346,14 @@ export const writeStructsFromTransaction = (encoder, transaction) => writeClient
 
 /**
  * @param {StructStore} store
- * @param {Map<number, Array<GCRef|ItemRef>>} clientsStructsRefs
+ * @param {Map<number, Array<GC|Item>>} clientsStructsRefs
  *
  * @private
  * @function
  */
 const mergeReadStructsIntoPendingReads = (store, clientsStructsRefs) => {
   const pendingClientsStructRefs = store.pendingClientsStructRefs
-  for (const [client, structRefs] of clientsStructsRefs) {
+  clientsStructsRefs.forEach((structRefs, client) => {
     const pendingStructRefs = pendingClientsStructRefs.get(client)
     if (pendingStructRefs === undefined) {
       pendingClientsStructRefs.set(client, { refs: structRefs, i: 0 })
@@ -266,7 +366,22 @@ const mergeReadStructsIntoPendingReads = (store, clientsStructsRefs) => {
       pendingStructRefs.i = 0
       pendingStructRefs.refs = merged.sort((r1, r2) => r1.id.clock - r2.id.clock)
     }
-  }
+  })
+}
+
+/**
+ * @param {Map<number,{refs:Array<GC|Item>,i:number}>} pendingClientsStructRefs
+ */
+const cleanupPendingStructs = pendingClientsStructRefs => {
+  // cleanup pendingClientsStructs if not fully finished
+  pendingClientsStructRefs.forEach((refs, client) => {
+    if (refs.i === refs.refs.length) {
+      pendingClientsStructRefs.delete(client)
+    } else {
+      refs.refs.splice(0, refs.i)
+      refs.i = 0
+    }
+  })
 }
 
 /**
@@ -274,7 +389,7 @@ const mergeReadStructsIntoPendingReads = (store, clientsStructsRefs) => {
  *
  * This is called when data is received from a remote peer.
  *
- * @param {decoding.Decoder} decoder The decoder object to read data from.
+ * @param {AbstractUpdateDecoder} decoder The decoder object to read data from.
  * @param {Transaction} transaction
  * @param {StructStore} store
  *
@@ -282,12 +397,43 @@ const mergeReadStructsIntoPendingReads = (store, clientsStructsRefs) => {
  * @function
  */
 export const readStructs = (decoder, transaction, store) => {
-  const clientsStructRefs = readClientsStructRefs(decoder)
+  const clientsStructRefs = new Map()
+  // let start = performance.now()
+  readClientsStructRefs(decoder, clientsStructRefs, transaction.doc)
+  // console.log('time to read structs: ', performance.now() - start) // @todo remove
+  // start = performance.now()
   mergeReadStructsIntoPendingReads(store, clientsStructRefs)
+  // console.log('time to merge: ', performance.now() - start) // @todo remove
+  // start = performance.now()
   resumeStructIntegration(transaction, store)
+  // console.log('time to integrate: ', performance.now() - start) // @todo remove
+  // start = performance.now()
+  cleanupPendingStructs(store.pendingClientsStructRefs)
+  // console.log('time to cleanup: ', performance.now() - start) // @todo remove
+  // start = performance.now()
   tryResumePendingDeleteReaders(transaction, store)
+  // console.log('time to resume delete readers: ', performance.now() - start) // @todo remove
+  // start = performance.now()
 }
 
+/**
+ * Read and apply a document update.
+ *
+ * This function has the same effect as `applyUpdate` but accepts an decoder.
+ *
+ * @param {decoding.Decoder} decoder
+ * @param {Doc} ydoc
+ * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
+ * @param {AbstractUpdateDecoder} [structDecoder]
+ *
+ * @function
+ */
+export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = new UpdateDecoderV2(decoder)) =>
+  transact(ydoc, transaction => {
+    readStructs(structDecoder, transaction, ydoc.store)
+    readAndApplyDeleteSet(structDecoder, transaction, ydoc.store)
+  }, transactionOrigin, false)
+
 /**
  * Read and apply a document update.
  *
@@ -299,11 +445,24 @@ export const readStructs = (decoder, transaction, store) => {
  *
  * @function
  */
-export const readUpdate = (decoder, ydoc, transactionOrigin) =>
-  transact(ydoc, transaction => {
-    readStructs(decoder, transaction, ydoc.store)
-    readAndApplyDeleteSet(decoder, transaction, ydoc.store)
-  }, transactionOrigin, false)
+export const readUpdate = (decoder, ydoc, transactionOrigin) => readUpdateV2(decoder, ydoc, transactionOrigin, new DefaultUpdateDecoder(decoder))
+
+/**
+ * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`.
+ *
+ * This function has the same effect as `readUpdate` but accepts an Uint8Array instead of a Decoder.
+ *
+ * @param {Doc} ydoc
+ * @param {Uint8Array} update
+ * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
+ * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder]
+ *
+ * @function
+ */
+export const applyUpdateV2 = (ydoc, update, transactionOrigin, YDecoder = UpdateDecoderV2) => {
+  const decoder = decoding.createDecoder(update)
+  readUpdateV2(decoder, ydoc, transactionOrigin, new YDecoder(decoder))
+}
 
 /**
  * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`.
@@ -316,14 +475,13 @@ export const readUpdate = (decoder, ydoc, transactionOrigin) =>
  *
  * @function
  */
-export const applyUpdate = (ydoc, update, transactionOrigin) =>
-  readUpdate(decoding.createDecoder(update), ydoc, transactionOrigin)
+export const applyUpdate = (ydoc, update, transactionOrigin) => applyUpdateV2(ydoc, update, transactionOrigin, DefaultUpdateDecoder)
 
 /**
  * Write all the document as a single update message. If you specify the state of the remote client (`targetStateVector`) it will
  * only write the operations that are missing.
  *
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Doc} doc
  * @param {Map<number,number>} [targetStateVector] The state of the target that receives the update. Leave empty to write all known structs
  *
@@ -342,31 +500,45 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map())
  *
  * @param {Doc} doc
  * @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs
+ * @param {AbstractUpdateEncoder} [encoder]
  * @return {Uint8Array}
  *
  * @function
  */
-export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => {
-  const encoder = encoding.createEncoder()
+export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => {
   const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector)
   writeStateAsUpdate(encoder, doc, targetStateVector)
-  return encoding.toUint8Array(encoder)
+  return encoder.toUint8Array()
 }
 
+/**
+ * Write all the document as a single update message that can be applied on the remote document. If you specify the state of the remote client (`targetState`) it will
+ * only write the operations that are missing.
+ *
+ * Use `writeStateAsUpdate` instead if you are working with lib0/encoding.js#Encoder
+ *
+ * @param {Doc} doc
+ * @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs
+ * @return {Uint8Array}
+ *
+ * @function
+ */
+export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => encodeStateAsUpdateV2(doc, encodedTargetStateVector, new DefaultUpdateEncoder())
+
 /**
  * Read state vector from Decoder and return as Map
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractDSDecoder} decoder
  * @return {Map<number,number>} Maps `client` to the number next expected `clock` from that client.
  *
  * @function
  */
 export const readStateVector = decoder => {
   const ss = new Map()
-  const ssLength = decoding.readVarUint(decoder)
+  const ssLength = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < ssLength; i++) {
-    const client = decoding.readVarUint(decoder)
-    const clock = decoding.readVarUint(decoder)
+    const client = decoding.readVarUint(decoder.restDecoder)
+    const clock = decoding.readVarUint(decoder.restDecoder)
     ss.set(client, clock)
   }
   return ss
@@ -380,28 +552,34 @@ export const readStateVector = decoder => {
  *
  * @function
  */
-export const decodeStateVector = decodedState => readStateVector(decoding.createDecoder(decodedState))
+export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
 
 /**
- * Write State Vector to `lib0/encoding.js#Encoder`.
+ * Read decodedState and return State as Map.
  *
- * @param {encoding.Encoder} encoder
+ * @param {Uint8Array} decodedState
+ * @return {Map<number,number>} Maps `client` to the number next expected `clock` from that client.
+ *
+ * @function
+ */
+export const decodeStateVector = decodedState => readStateVector(new DefaultDSDecoder(decoding.createDecoder(decodedState)))
+
+/**
+ * @param {AbstractDSEncoder} encoder
  * @param {Map<number,number>} sv
  * @function
  */
 export const writeStateVector = (encoder, sv) => {
-  encoding.writeVarUint(encoder, sv.size)
+  encoding.writeVarUint(encoder.restEncoder, sv.size)
   sv.forEach((clock, client) => {
-    encoding.writeVarUint(encoder, client)
-    encoding.writeVarUint(encoder, clock)
+    encoding.writeVarUint(encoder.restEncoder, client) // @todo use a special client decoder that is based on mapping
+    encoding.writeVarUint(encoder.restEncoder, clock)
   })
   return encoder
 }
 
 /**
- * Write State Vector to `lib0/encoding.js#Encoder`.
- *
- * @param {encoding.Encoder} encoder
+ * @param {AbstractDSEncoder} encoder
  * @param {Doc} doc
  *
  * @function
@@ -412,12 +590,22 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod
  * Encode State as Uint8Array.
  *
  * @param {Doc} doc
+ * @param {AbstractDSEncoder} [encoder]
  * @return {Uint8Array}
  *
  * @function
  */
-export const encodeStateVector = doc => {
-  const encoder = encoding.createEncoder()
+export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => {
   writeDocumentStateVector(encoder, doc)
-  return encoding.toUint8Array(encoder)
+  return encoder.toUint8Array()
 }
+
+/**
+ * Encode State as Uint8Array.
+ *
+ * @param {Doc} doc
+ * @return {Uint8Array}
+ *
+ * @function
+ */
+export const encodeStateVector = doc => encodeStateVectorV2(doc, new DefaultDSEncoder())

src/utils/isParentOf.js

@@ -16,7 +16,7 @@ export const isParentOf = (parent, child) => {
     if (child.parent === parent) {
       return true
     }
-    child = child.parent._item
+    child = /** @type {AbstractType<any>} */ (child.parent)._item
   }
   return false
 }

src/utils/logging.js

@@ -0,0 +1,22 @@
+
+import {
+  AbstractType // eslint-disable-line
+} from '../internals.js'
+
+/**
+ * Convenient helper to log type information.
+ *
+ * Do not use in productive systems as the output can be immense!
+ *
+ * @param {AbstractType<any>} type
+ */
+export const logType = type => {
+  const res = []
+  let n = type._start
+  while (n) {
+    res.push(n)
+    n = n.right
+  }
+  console.log('Children: ', res)
+  console.log('Children content: ', res.filter(m => !m.deleted).map(m => m.content))
+}

tests/doc.tests.js

@@ -0,0 +1,126 @@
+
+import * as Y from '../src/index.js'
+import * as t from 'lib0/testing.js'
+
+/**
+ * Client id should be changed when an instance receives updates from another client using the same client id.
+ *
+ * @param {t.TestCase} tc
+ */
+export const testClientIdDuplicateChange = tc => {
+  const doc1 = new Y.Doc()
+  doc1.clientID = 0
+  const doc2 = new Y.Doc()
+  doc2.clientID = 0
+  t.assert(doc2.clientID === doc1.clientID)
+  doc1.getArray('a').insert(0, [1, 2])
+  Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
+  t.assert(doc2.clientID !== doc1.clientID)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testGetTypeEmptyId = tc => {
+  const doc1 = new Y.Doc()
+  doc1.getText('').insert(0, 'h')
+  doc1.getText().insert(1, 'i')
+  const doc2 = new Y.Doc()
+  Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
+  t.assert(doc2.getText().toString() === 'hi')
+  t.assert(doc2.getText('').toString() === 'hi')
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testToJSON = tc => {
+  const doc = new Y.Doc()
+  t.compare(doc.toJSON(), {}, 'doc.toJSON yields empty object')
+
+  const arr = doc.getArray('array')
+  arr.push(['test1'])
+
+  const map = doc.getMap('map')
+  map.set('k1', 'v1')
+  const map2 = new Y.Map()
+  map.set('k2', map2)
+  map2.set('m2k1', 'm2v1')
+
+  t.compare(doc.toJSON(), {
+    array: ['test1'],
+    map: {
+      k1: 'v1',
+      k2: {
+        m2k1: 'm2v1'
+      }
+    }
+  }, 'doc.toJSON has array and recursive map')
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testSubdoc = tc => {
+  const doc = new Y.Doc()
+  doc.load() // doesn't do anything
+  {
+    /**
+     * @type {Array<any>|null}
+     */
+    let event = /** @type {any} */ (null)
+    doc.on('subdocs', subdocs => {
+      event = [Array.from(subdocs.added).map(x => x.guid), Array.from(subdocs.removed).map(x => x.guid), Array.from(subdocs.loaded).map(x => x.guid)]
+    })
+    const subdocs = doc.getMap('mysubdocs')
+    const docA = new Y.Doc({ guid: 'a' })
+    docA.load()
+    subdocs.set('a', docA)
+    t.compare(event, [['a'], [], ['a']])
+
+    event = null
+    subdocs.get('a').load()
+    t.assert(event === null)
+
+    event = null
+    subdocs.get('a').destroy()
+    t.compare(event, [['a'], ['a'], []])
+    subdocs.get('a').load()
+    t.compare(event, [[], [], ['a']])
+
+    subdocs.set('b', new Y.Doc({ guid: 'a' }))
+    t.compare(event, [['a'], [], []])
+    subdocs.get('b').load()
+    t.compare(event, [[], [], ['a']])
+
+    const docC = new Y.Doc({ guid: 'c' })
+    docC.load()
+    subdocs.set('c', docC)
+    t.compare(event, [['c'], [], ['c']])
+
+    t.compare(Array.from(doc.getSubdocGuids()), ['a', 'c'])
+  }
+
+  const doc2 = new Y.Doc()
+  {
+    t.compare(Array.from(doc2.getSubdocs()), [])
+    /**
+     * @type {Array<any>|null}
+     */
+    let event = /** @type {any} */ (null)
+    doc2.on('subdocs', subdocs => {
+      event = [Array.from(subdocs.added).map(d => d.guid), Array.from(subdocs.removed).map(d => d.guid), Array.from(subdocs.loaded).map(d => d.guid)]
+    })
+    Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc))
+    t.compare(event, [['a', 'a', 'c'], [], []])
+
+    doc2.getMap('mysubdocs').get('a').load()
+    t.compare(event, [[], [], ['a']])
+
+    t.compare(Array.from(doc2.getSubdocGuids()), ['a', 'c'])
+
+    doc2.getMap('mysubdocs').delete('a')
+    t.compare(event, [[], ['a'], []])
+    t.compare(Array.from(doc2.getSubdocGuids()), ['a', 'c'])
+  }
+}

tests/encoding.tests.js

@@ -9,14 +9,15 @@ import {
   readContentEmbed,
   readContentType,
   readContentFormat,
-  readContentAny
+  readContentAny,
+  readContentDoc
 } from '../src/internals.js'
 
 /**
  * @param {t.TestCase} tc
  */
 export const testStructReferences = tc => {
-  t.assert(contentRefs.length === 9)
+  t.assert(contentRefs.length === 10)
   t.assert(contentRefs[1] === readContentDeleted)
   t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json?
   t.assert(contentRefs[3] === readContentBinary)
@@ -25,4 +26,5 @@ export const testStructReferences = tc => {
   t.assert(contentRefs[6] === readContentFormat)
   t.assert(contentRefs[7] === readContentType)
   t.assert(contentRefs[8] === readContentAny)
+  t.assert(contentRefs[9] === readContentDoc)
 }

tests/index.js

@@ -1,10 +1,13 @@
 
-import * as array from './y-array.tests.js'
 import * as map from './y-map.tests.js'
+import * as array from './y-array.tests.js'
 import * as text from './y-text.tests.js'
 import * as xml from './y-xml.tests.js'
 import * as encoding from './encoding.tests.js'
 import * as undoredo from './undo-redo.tests.js'
+import * as compatibility from './compatibility.tests.js'
+import * as doc from './doc.tests.js'
+import * as snapshot from './snapshot.tests.js'
 
 import { runTests } from 'lib0/testing.js'
 import { isBrowser, isNode } from 'lib0/environment.js'
@@ -14,7 +17,7 @@ if (isBrowser) {
   log.createVConsole(document.body)
 }
 runTests({
-  map, array, text, xml, encoding, undoredo
+  doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot
 }).then(success => {
   /* istanbul ignore next */
   if (isNode) {

tests/snapshot.tests.js

@@ -0,0 +1,171 @@
+import { createDocFromSnapshot, Doc, snapshot, YMap } from '../src/internals'
+import * as t from 'lib0/testing.js'
+import { init } from './testHelper'
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testBasicRestoreSnapshot = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['hello'])
+  const snap = snapshot(doc)
+  doc.getArray('array').insert(1, ['world'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['hello'])
+  t.compare(doc.getArray('array').toArray(), ['hello', 'world'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testEmptyRestoreSnapshot = tc => {
+  const doc = new Doc({ gc: false })
+  const snap = snapshot(doc)
+  snap.sv.set(9999, 0)
+  doc.getArray().insert(0, ['world'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray().toArray(), [])
+  t.compare(doc.getArray().toArray(), ['world'])
+
+  // now this snapshot reflects the latest state. It shoult still work.
+  const snap2 = snapshot(doc)
+  const docRestored2 = createDocFromSnapshot(doc, snap2)
+  t.compare(docRestored2.getArray().toArray(), ['world'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRestoreSnapshotWithSubType = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, [new YMap()])
+  const subMap = doc.getArray('array').get(0)
+  subMap.set('key1', 'value1')
+
+  const snap = snapshot(doc)
+  subMap.set('key2', 'value2')
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toJSON(), [{
+    key1: 'value1'
+  }])
+  t.compare(doc.getArray('array').toJSON(), [{
+    key1: 'value1',
+    key2: 'value2'
+  }])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRestoreDeletedItem1 = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1', 'item2'])
+
+  const snap = snapshot(doc)
+  doc.getArray('array').delete(0)
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['item1', 'item2'])
+  t.compare(doc.getArray('array').toArray(), ['item2'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRestoreLeftItem = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1'])
+  doc.getMap('map').set('test', 1)
+  doc.getArray('array').insert(0, ['item0'])
+
+  const snap = snapshot(doc)
+  doc.getArray('array').delete(1)
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['item0', 'item1'])
+  t.compare(doc.getArray('array').toArray(), ['item0'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testDeletedItemsBase = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1'])
+  doc.getArray('array').delete(0)
+  const snap = snapshot(doc)
+  doc.getArray('array').insert(0, ['item0'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), [])
+  t.compare(doc.getArray('array').toArray(), ['item0'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testDeletedItems2 = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1', 'item2', 'item3'])
+  doc.getArray('array').delete(1)
+  const snap = snapshot(doc)
+  doc.getArray('array').insert(0, ['item0'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['item1', 'item3'])
+  t.compare(doc.getArray('array').toArray(), ['item0', 'item1', 'item3'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testDependentChanges = tc => {
+  const { array0, array1, testConnector } = init(tc, { users: 2 })
+
+  if (!array0.doc) {
+    throw new Error('no document 0')
+  }
+  if (!array1.doc) {
+    throw new Error('no document 1')
+  }
+
+  /**
+   * @type Doc
+   */
+  const doc0 = array0.doc
+  /**
+   * @type Doc
+   */
+  const doc1 = array1.doc
+
+  doc0.gc = false
+  doc1.gc = false
+
+  array0.insert(0, ['user1item1'])
+  testConnector.syncAll()
+  array1.insert(1, ['user2item1'])
+  testConnector.syncAll()
+
+  const snap = snapshot(array0.doc)
+
+  array0.insert(2, ['user1item2'])
+  testConnector.syncAll()
+  array1.insert(3, ['user2item2'])
+  testConnector.syncAll()
+
+  const docRestored0 = createDocFromSnapshot(array0.doc, snap)
+  t.compare(docRestored0.getArray('array').toArray(), ['user1item1', 'user2item1'])
+
+  const docRestored1 = createDocFromSnapshot(array1.doc, snap)
+  t.compare(docRestored1.getArray('array').toArray(), ['user1item1', 'user2item1'])
+}

tests/testHelper.js

@@ -1,11 +1,3 @@
-import * as Y from '../src/index.js'
-
-import {
-  createDeleteSetFromStructStore,
-  getStateVector,
-  Item,
-  DeleteItem, DeleteSet, StructStore, Doc // eslint-disable-line
-} from '../src/internals.js'
 
 import * as t from 'lib0/testing.js'
 import * as prng from 'lib0/prng.js'
@@ -13,8 +5,14 @@ import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
 import * as syncProtocol from 'y-protocols/sync.js'
 import * as object from 'lib0/object.js'
+import * as Y from '../src/internals.js'
 export * from '../src/internals.js'
 
+if (typeof window !== 'undefined') {
+  // @ts-ignore
+  window.Y = Y // eslint-disable-line
+}
+
 /**
  * @param {TestYInstance} y // publish message created by `y` to all other online clients
  * @param {Uint8Array} m
@@ -29,7 +27,7 @@ const broadcastMessage = (y, m) => {
   }
 }
 
-export class TestYInstance extends Doc {
+export class TestYInstance extends Y.Doc {
   /**
    * @param {TestConnector} testConnector
    * @param {number} clientID
@@ -230,7 +228,7 @@ export class TestConnector {
  * @param {t.TestCase} tc
  * @param {{users?:number}} conf
  * @param {InitTestObjectCallback<T>} [initTestObject]
- * @return {{testObjects:Array<any>,testConnector:TestConnector,users:Array<TestYInstance>,array0:Y.Array<any>,array1:Y.Array<any>,array2:Y.Array<any>,map0:Y.Map<any>,map1:Y.Map<any>,map2:Y.Map<any>,map3:Y.Map<any>,text0:Y.Text,text1:Y.Text,text2:Y.Text,xml0:Y.XmlElement,xml1:Y.XmlElement,xml2:Y.XmlElement}}
+ * @return {{testObjects:Array<any>,testConnector:TestConnector,users:Array<TestYInstance>,array0:Y.YArray<any>,array1:Y.YArray<any>,array2:Y.YArray<any>,map0:Y.YMap<any>,map1:Y.YMap<any>,map2:Y.YMap<any>,map3:Y.YMap<any>,text0:Y.YText,text1:Y.YText,text2:Y.YText,xml0:Y.YXmlElement,xml1:Y.YXmlElement,xml2:Y.YXmlElement}}
  */
 export const init = (tc, { users = 5 } = {}, initTestObject) => {
   /**
@@ -240,19 +238,27 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => {
     users: []
   }
   const gen = tc.prng
+  // choose an encoding approach at random
+  if (prng.bool(gen)) {
+    Y.useV2Encoding()
+  } else {
+    Y.useV1Encoding()
+  }
+
   const testConnector = new TestConnector(gen)
   result.testConnector = testConnector
   for (let i = 0; i < users; i++) {
     const y = testConnector.createY(i)
     y.clientID = i
     result.users.push(y)
-    result['array' + i] = y.get('array', Y.Array)
-    result['map' + i] = y.get('map', Y.Map)
-    result['xml' + i] = y.get('xml', Y.XmlElement)
-    result['text' + i] = y.get('text', Y.Text)
+    result['array' + i] = y.getArray('array')
+    result['map' + i] = y.getMap('map')
+    result['xml' + i] = y.get('xml', Y.YXmlElement)
+    result['text' + i] = y.getText('text')
   }
   testConnector.syncAll()
   result.testObjects = result.users.map(initTestObject || (() => null))
+  Y.useV1Encoding()
   return /** @type {any} */ (result)
 }
 
@@ -270,7 +276,7 @@ export const compare = users => {
   while (users[0].tc.flushAllMessages()) {}
   const userArrayValues = users.map(u => u.getArray('array').toJSON())
   const userMapValues = users.map(u => u.getMap('map').toJSON())
-  const userXmlValues = users.map(u => u.get('xml', Y.XmlElement).toString())
+  const userXmlValues = users.map(u => u.get('xml', Y.YXmlElement).toString())
   const userTextValues = users.map(u => u.getText('text').toDelta())
   for (const u of users) {
     t.assert(u.store.pendingDeleteReaders.length === 0)
@@ -299,23 +305,23 @@ export const compare = users => {
     t.compare(userXmlValues[i], userXmlValues[i + 1])
     t.compare(userTextValues[i].map(/** @param {any} a */ a => typeof a.insert === 'string' ? a.insert : ' ').join('').length, users[i].getText('text').length)
     t.compare(userTextValues[i], userTextValues[i + 1])
-    t.compare(getStateVector(users[i].store), getStateVector(users[i + 1].store))
-    compareDS(createDeleteSetFromStructStore(users[i].store), createDeleteSetFromStructStore(users[i + 1].store))
+    t.compare(Y.getStateVector(users[i].store), Y.getStateVector(users[i + 1].store))
+    compareDS(Y.createDeleteSetFromStructStore(users[i].store), Y.createDeleteSetFromStructStore(users[i + 1].store))
     compareStructStores(users[i].store, users[i + 1].store)
   }
   users.map(u => u.destroy())
 }
 
 /**
- * @param {Item?} a
- * @param {Item?} b
+ * @param {Y.Item?} a
+ * @param {Y.Item?} b
  * @return {boolean}
  */
 export const compareItemIDs = (a, b) => a === b || (a !== null && b != null && Y.compareIDs(a.id, b.id))
 
 /**
- * @param {StructStore} ss1
- * @param {StructStore} ss2
+ * @param {Y.StructStore} ss1
+ * @param {Y.StructStore} ss2
  */
 export const compareStructStores = (ss1, ss2) => {
   t.assert(ss1.clients.size === ss2.clients.size)
@@ -330,13 +336,14 @@ export const compareStructStores = (ss1, ss2) => {
         s1.constructor !== s2.constructor ||
         !Y.compareIDs(s1.id, s2.id) ||
         s1.deleted !== s2.deleted ||
+        // @ts-ignore
         s1.length !== s2.length
       ) {
         t.fail('Structs dont match')
       }
-      if (s1 instanceof Item) {
+      if (s1 instanceof Y.Item) {
         if (
-          !(s2 instanceof Item) ||
+          !(s2 instanceof Y.Item) ||
           !((s1.left === null && s2.left === null) || (s1.left !== null && s2.left !== null && Y.compareIDs(s1.left.lastId, s2.left.lastId))) ||
           !compareItemIDs(s1.right, s2.right) ||
           !Y.compareIDs(s1.origin, s2.origin) ||
@@ -356,13 +363,13 @@ export const compareStructStores = (ss1, ss2) => {
 }
 
 /**
- * @param {DeleteSet} ds1
- * @param {DeleteSet} ds2
+ * @param {Y.DeleteSet} ds1
+ * @param {Y.DeleteSet} ds2
  */
 export const compareDS = (ds1, ds2) => {
   t.assert(ds1.clients.size === ds2.clients.size)
-  for (const [client, deleteItems1] of ds1.clients) {
-    const deleteItems2 = /** @type {Array<DeleteItem>} */ (ds2.clients.get(client))
+  ds1.clients.forEach((deleteItems1, client) => {
+    const deleteItems2 = /** @type {Array<Y.DeleteItem>} */ (ds2.clients.get(client))
     t.assert(deleteItems2 !== undefined && deleteItems1.length === deleteItems2.length)
     for (let i = 0; i < deleteItems1.length; i++) {
       const di1 = deleteItems1[i]
@@ -371,7 +378,7 @@ export const compareDS = (ds1, ds2) => {
         t.fail('DeleteSets dont match')
       }
     }
-  }
+  })
 }
 
 /**
@@ -393,21 +400,21 @@ export const applyRandomTests = (tc, mods, iterations, initTestObject) => {
   const result = init(tc, { users: 5 }, initTestObject)
   const { testConnector, users } = result
   for (let i = 0; i < iterations; i++) {
-    if (prng.int31(gen, 0, 100) <= 2) {
+    if (prng.int32(gen, 0, 100) <= 2) {
       // 2% chance to disconnect/reconnect a random user
       if (prng.bool(gen)) {
         testConnector.disconnectRandom()
       } else {
         testConnector.reconnectRandom()
       }
-    } else if (prng.int31(gen, 0, 100) <= 1) {
+    } else if (prng.int32(gen, 0, 100) <= 1) {
       // 1% chance to flush all
       testConnector.flushAllMessages()
-    } else if (prng.int31(gen, 0, 100) <= 50) {
+    } else if (prng.int32(gen, 0, 100) <= 50) {
       // 50% chance to flush a random message
       testConnector.flushRandomMessage()
     }
-    const user = prng.int31(gen, 0, users.length - 1)
+    const user = prng.int32(gen, 0, users.length - 1)
     const test = prng.oneOf(gen, mods)
     test(users[user], gen, result.testObjects[user])
   }

tests/undo-redo.tests.js

@@ -53,6 +53,28 @@ export const testUndoText = tc => {
   t.compare(text0.toDelta(), [{ insert: 'b' }, { insert: 'cxy', attributes: { bold: true } }, { insert: 'z' }])
 }
 
+/**
+ * Test case to fix #241
+ * @param {t.TestCase} tc
+ */
+export const testDoubleUndo = tc => {
+  const doc = new Y.Doc()
+  const text = doc.getText()
+  text.insert(0, '1221')
+
+  const manager = new Y.UndoManager(text)
+
+  text.insert(2, '3')
+  text.insert(3, '3')
+
+  manager.undo()
+  manager.undo()
+
+  text.insert(2, '3')
+
+  t.compareStrings(text.toString(), '12321')
+}
+
 /**
  * @param {t.TestCase} tc
  */

tests/y-array.tests.js

@@ -5,6 +5,18 @@ import * as t from 'lib0/testing.js'
 import * as prng from 'lib0/prng.js'
 import * as math from 'lib0/math.js'
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testBasicUpdate = tc => {
+  const doc1 = new Y.Doc()
+  const doc2 = new Y.Doc()
+  doc1.getArray('array').insert(0, ['hi'])
+  const update = Y.encodeStateAsUpdate(doc1)
+  Y.applyUpdate(doc2, update)
+  t.compare(doc2.getArray('array').toArray(), ['hi'])
+}
+
 /**
  * @param {t.TestCase} tc
  */
@@ -335,23 +347,26 @@ const arrayTransactions = [
     const yarray = user.getArray('array')
     var uniqueNumber = getUniqueNumber()
     var content = []
-    var len = prng.int31(gen, 1, 4)
+    var len = prng.int32(gen, 1, 4)
     for (var i = 0; i < len; i++) {
       content.push(uniqueNumber)
     }
-    var pos = prng.int31(gen, 0, yarray.length)
+    var pos = prng.int32(gen, 0, yarray.length)
+    const oldContent = yarray.toArray()
     yarray.insert(pos, content)
+    oldContent.splice(pos, 0, ...content)
+    t.compareArrays(yarray.toArray(), oldContent) // we want to make sure that fastSearch markers insert at the correct position
   },
   function insertTypeArray (user, gen) {
     const yarray = user.getArray('array')
-    var pos = prng.int31(gen, 0, yarray.length)
+    var pos = prng.int32(gen, 0, yarray.length)
     yarray.insert(pos, [new Y.Array()])
     var array2 = yarray.get(pos)
     array2.insert(0, [1, 2, 3, 4])
   },
   function insertTypeMap (user, gen) {
     const yarray = user.getArray('array')
-    var pos = prng.int31(gen, 0, yarray.length)
+    var pos = prng.int32(gen, 0, yarray.length)
     yarray.insert(pos, [new Y.Map()])
     var map = yarray.get(pos)
     map.set('someprop', 42)
@@ -362,17 +377,20 @@ const arrayTransactions = [
     const yarray = user.getArray('array')
     var length = yarray.length
     if (length > 0) {
-      var somePos = prng.int31(gen, 0, length - 1)
-      var delLength = prng.int31(gen, 1, math.min(2, length - somePos))
+      var somePos = prng.int32(gen, 0, length - 1)
+      var delLength = prng.int32(gen, 1, math.min(2, length - somePos))
       if (prng.bool(gen)) {
         var type = yarray.get(somePos)
         if (type.length > 0) {
-          somePos = prng.int31(gen, 0, type.length - 1)
-          delLength = prng.int31(gen, 0, math.min(2, type.length - somePos))
+          somePos = prng.int32(gen, 0, type.length - 1)
+          delLength = prng.int32(gen, 0, math.min(2, type.length - somePos))
           type.delete(somePos, delLength)
         }
       } else {
+        const oldContent = yarray.toArray()
         yarray.delete(somePos, delLength)
+        oldContent.splice(somePos, delLength)
+        t.compareArrays(yarray.toArray(), oldContent)
       }
     }
   }
@@ -381,8 +399,8 @@ const arrayTransactions = [
 /**
  * @param {t.TestCase} tc
  */
-export const testRepeatGeneratingYarrayTests4 = tc => {
-  applyRandomTests(tc, arrayTransactions, 4)
+export const testRepeatGeneratingYarrayTests6 = tc => {
+  applyRandomTests(tc, arrayTransactions, 6)
 }
 
 /**

tests/y-map.tests.js

@@ -8,6 +8,33 @@ import * as Y from '../src/index.js'
 import * as t from 'lib0/testing.js'
 import * as prng from 'lib0/prng.js'
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testMapHavingIterableAsConstructorParamTests = tc => {
+  const { map0 } = init(tc, { users: 1 })
+
+  const m1 = new Y.Map(Object.entries({ number: 1, string: 'hello' }))
+  map0.set('m1', m1)
+  t.assert(m1.get('number') === 1)
+  t.assert(m1.get('string') === 'hello')
+
+  const m2 = new Y.Map([
+    ['object', { x: 1 }],
+    ['boolean', true]
+  ])
+  map0.set('m2', m2)
+  t.assert(m2.get('object').x === 1)
+  t.assert(m2.get('boolean') === true)
+
+  const m3 = new Y.Map([...m1, ...m2])
+  map0.set('m3', m3)
+  t.assert(m3.get('number') === 1)
+  t.assert(m3.get('string') === 'hello')
+  t.assert(m3.get('object').x === 1)
+  t.assert(m3.get('boolean') === true)
+}
+
 /**
  * @param {t.TestCase} tc
  */
@@ -33,6 +60,7 @@ export const testBasicMapTests = tc => {
   t.assert(map0.get('boolean1') === true, 'client 0 computed the change (boolean)')
   t.compare(map0.get('object'), { key: { key2: 'value' } }, 'client 0 computed the change (object)')
   t.assert(map0.get('y-map').get('y-array').get(0) === -1, 'client 0 computed the change (type)')
+  t.assert(map0.size === 6, 'client 0 map has correct size')
 
   users[2].connect()
   testConnector.flushAllMessages()
@@ -43,6 +71,7 @@ export const testBasicMapTests = tc => {
   t.assert(map1.get('boolean1') === true, 'client 1 computed the change (boolean)')
   t.compare(map1.get('object'), { key: { key2: 'value' } }, 'client 1 received the update (object)')
   t.assert(map1.get('y-map').get('y-array').get(0) === -1, 'client 1 received the update (type)')
+  t.assert(map1.size === 6, 'client 1 map has correct size')
 
   // compare disconnected user
   t.assert(map2.get('number') === 1, 'client 2 received the update (number) - was disconnected')
@@ -130,6 +159,20 @@ export const testGetAndSetOfMapPropertyWithConflict = tc => {
   compare(users)
 }
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testSizeAndDeleteOfMapProperty = tc => {
+  const { map0 } = init(tc, { users: 1 })
+  map0.set('stuff', 'c0')
+  map0.set('otherstuff', 'c1')
+  t.assert(map0.size === 2, `map size is ${map0.size} expected 2`)
+  map0.delete('stuff')
+  t.assert(map0.size === 1, `map size after delete is ${map0.size}, expected 1`)
+  map0.delete('otherstuff')
+  t.assert(map0.size === 0, `map size after delete is ${map0.size}, expected 0`)
+}
+
 /**
  * @param {t.TestCase} tc
  */
@@ -454,7 +497,7 @@ const mapTransactions = [
  * @param {t.TestCase} tc
  */
 export const testRepeatGeneratingYmapTests10 = tc => {
-  applyRandomTests(tc, mapTransactions, 10)
+  applyRandomTests(tc, mapTransactions, 3)
 }
 
 /**

tests/y-text.tests.js

@@ -1,5 +1,8 @@
 import * as Y from './testHelper.js'
 import * as t from 'lib0/testing.js'
+import * as prng from 'lib0/prng.js'
+import * as math from 'lib0/math.js'
+
 const { init, compare } = Y
 
 /**
@@ -158,3 +161,425 @@ export const testToJson = tc => {
   text0.insert(0, 'abc', { bold: true })
   t.assert(text0.toJSON() === 'abc', 'toJSON returns the unformatted text')
 }
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testToDeltaEmbedAttributes = tc => {
+  const { text0 } = init(tc, { users: 1 })
+  text0.insert(0, 'ab', { bold: true })
+  text0.insertEmbed(1, { image: 'imageSrc.png' }, { width: 100 })
+  const delta0 = text0.toDelta()
+  t.compare(delta0, [{ insert: 'a', attributes: { bold: true } }, { insert: { image: 'imageSrc.png' }, attributes: { width: 100 } }, { insert: 'b', attributes: { bold: true } }])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testToDeltaEmbedNoAttributes = tc => {
+  const { text0 } = init(tc, { users: 1 })
+  text0.insert(0, 'ab', { bold: true })
+  text0.insertEmbed(1, { image: 'imageSrc.png' })
+  const delta0 = text0.toDelta()
+  t.compare(delta0, [{ insert: 'a', attributes: { bold: true } }, { insert: { image: 'imageSrc.png' } }, { insert: 'b', attributes: { bold: true } }], 'toDelta does not set attributes key when no attributes are present')
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testFormattingRemoved = tc => {
+  const { text0 } = init(tc, { users: 1 })
+  text0.insert(0, 'ab', { bold: true })
+  text0.delete(0, 2)
+  t.assert(Y.getTypeChildren(text0).length === 1)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testFormattingRemovedInMidText = tc => {
+  const { text0 } = init(tc, { users: 1 })
+  text0.insert(0, '1234')
+  text0.insert(2, 'ab', { bold: true })
+  text0.delete(2, 2)
+  t.assert(Y.getTypeChildren(text0).length === 3)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testInsertAndDeleteAtRandomPositions = tc => {
+  const N = 100000
+  const { text0 } = init(tc, { users: 1 })
+  const gen = tc.prng
+
+  // create initial content
+  // let expectedResult = init
+  text0.insert(0, prng.word(gen, N / 2, N / 2))
+
+  // apply changes
+  for (let i = 0; i < N; i++) {
+    const pos = prng.uint32(gen, 0, text0.length)
+    if (prng.bool(gen)) {
+      const len = prng.uint32(gen, 1, 5)
+      const word = prng.word(gen, 0, len)
+      text0.insert(pos, word)
+      // expectedResult = expectedResult.slice(0, pos) + word + expectedResult.slice(pos)
+    } else {
+      const len = prng.uint32(gen, 0, math.min(3, text0.length - pos))
+      text0.delete(pos, len)
+      // expectedResult = expectedResult.slice(0, pos) + expectedResult.slice(pos + len)
+    }
+  }
+  // t.compareStrings(text0.toString(), expectedResult)
+  t.describe('final length', '' + text0.length)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testAppendChars = tc => {
+  const N = 10000
+  const { text0 } = init(tc, { users: 1 })
+
+  // apply changes
+  for (let i = 0; i < N; i++) {
+    text0.insert(text0.length, 'a')
+  }
+  t.assert(text0.length === N)
+}
+
+const id = Y.createID(0, 0)
+const c = new Y.ContentString('a')
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testBestCase = tc => {
+  const N = 2000000
+  const items = new Array(N)
+  t.measureTime('time to create two million items in the best case', () => {
+    const parent = /** @type {any} */ ({})
+    let prevItem = null
+    for (let i = 0; i < N; i++) {
+      /**
+       * @type {Y.Item}
+       */
+      const n = new Y.Item(Y.createID(0, 0), null, null, null, null, null, null, c)
+      // items.push(n)
+      items[i] = n
+      n.right = prevItem
+      n.rightOrigin = prevItem ? id : null
+      n.content = c
+      n.parent = parent
+      prevItem = n
+    }
+  })
+  const newArray = new Array(N)
+  t.measureTime('time to copy two million items to new Array', () => {
+    for (let i = 0; i < N; i++) {
+      newArray[i] = items[i]
+    }
+  })
+}
+
+const tryGc = () => {
+  if (typeof global !== 'undefined' && global.gc) {
+    global.gc()
+  }
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testLargeFragmentedDocument = tc => {
+  const itemsToInsert = 1000000
+  let update = /** @type {any} */ (null)
+  ;(() => {
+    const doc1 = new Y.Doc()
+    const text0 = doc1.getText('txt')
+    tryGc()
+    t.measureTime(`time to insert ${itemsToInsert} items`, () => {
+      doc1.transact(() => {
+        for (let i = 0; i < itemsToInsert; i++) {
+          text0.insert(0, '0')
+        }
+      })
+    })
+    tryGc()
+    t.measureTime('time to encode document', () => {
+      update = Y.encodeStateAsUpdateV2(doc1)
+    })
+    t.describe('Document size:', update.byteLength)
+  })()
+  ;(() => {
+    const doc2 = new Y.Doc()
+    tryGc()
+    t.measureTime(`time to apply ${itemsToInsert} updates`, () => {
+      Y.applyUpdateV2(doc2, update)
+    })
+  })()
+}
+
+// RANDOM TESTS
+
+let charCounter = 0
+
+/**
+ * Random tests for pure text operations without formatting.
+ *
+ * @type Array<function(any,prng.PRNG):void>
+ */
+const textChanges = [
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // insert text
+    const ytext = y.getText('text')
+    const insertPos = prng.int32(gen, 0, ytext.length)
+    const text = charCounter++ + prng.word(gen)
+    const prevText = ytext.toString()
+    ytext.insert(insertPos, text)
+    t.compareStrings(ytext.toString(), prevText.slice(0, insertPos) + text + prevText.slice(insertPos))
+  },
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // delete text
+    const ytext = y.getText('text')
+    const contentLen = ytext.toString().length
+    const insertPos = prng.int32(gen, 0, contentLen)
+    const overwrite = math.min(prng.int32(gen, 0, contentLen - insertPos), 2)
+    const prevText = ytext.toString()
+    ytext.delete(insertPos, overwrite)
+    t.compareStrings(ytext.toString(), prevText.slice(0, insertPos) + prevText.slice(insertPos + overwrite))
+  }
+]
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges5 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 5))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges30 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 30))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges40 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 40))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges50 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 50))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges70 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 70))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges90 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 90))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateTextChanges300 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, textChanges, 300))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+const marks = [
+  { bold: true },
+  { italic: true },
+  { italic: true, color: '#888' }
+]
+
+const marksChoices = [
+  undefined,
+  ...marks
+]
+
+/**
+ * Random tests for all features of y-text (formatting, embeds, ..).
+ *
+ * @type Array<function(any,prng.PRNG):void>
+ */
+const qChanges = [
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // insert text
+    const ytext = y.getText('text')
+    const insertPos = prng.int32(gen, 0, ytext.length)
+    const attrs = prng.oneOf(gen, marksChoices)
+    const text = charCounter++ + prng.word(gen)
+    ytext.insert(insertPos, text, attrs)
+  },
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // insert embed
+    const ytext = y.getText('text')
+    const insertPos = prng.int32(gen, 0, ytext.length)
+    ytext.insertEmbed(insertPos, { image: 'https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png' })
+  },
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // delete text
+    const ytext = y.getText('text')
+    const contentLen = ytext.toString().length
+    const insertPos = prng.int32(gen, 0, contentLen)
+    const overwrite = math.min(prng.int32(gen, 0, contentLen - insertPos), 2)
+    ytext.delete(insertPos, overwrite)
+  },
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // format text
+    const ytext = y.getText('text')
+    const contentLen = ytext.toString().length
+    const insertPos = prng.int32(gen, 0, contentLen)
+    const overwrite = math.min(prng.int32(gen, 0, contentLen - insertPos), 2)
+    const format = prng.oneOf(gen, marks)
+    ytext.format(insertPos, overwrite, format)
+  },
+  /**
+   * @param {Y.Doc} y
+   * @param {prng.PRNG} gen
+   */
+  (y, gen) => { // insert codeblock
+    const ytext = y.getText('text')
+    const insertPos = prng.int32(gen, 0, ytext.toString().length)
+    const text = charCounter++ + prng.word(gen)
+    const ops = []
+    if (insertPos > 0) {
+      ops.push({ retain: insertPos })
+    }
+    ops.push({ insert: text }, { insert: '\n', format: { 'code-block': true } })
+    ytext.applyDelta(ops)
+  }
+]
+
+/**
+ * @param {any} result
+ */
+const checkResult = result => {
+  for (let i = 1; i < result.testObjects.length; i++) {
+    const p1 = result.users[i].getText('text').toDelta()
+    const p2 = result.users[i].getText('text').toDelta()
+    t.compare(p1, p2)
+  }
+  // Uncomment this to find formatting-cleanup issues
+  // const cleanups = Y.cleanupYTextFormatting(result.users[0].getText('text'))
+  // t.assert(cleanups === 0)
+  return result
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges1 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, qChanges, 1))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges2 = tc => {
+  const { users } = checkResult(Y.applyRandomTests(tc, qChanges, 2))
+  const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+  t.assert(cleanups === 0)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges2Repeat = tc => {
+  for (let i = 0; i < 1000; i++) {
+    const { users } = checkResult(Y.applyRandomTests(tc, qChanges, 2))
+    const cleanups = Y.cleanupYTextFormatting(users[0].getText('text'))
+    t.assert(cleanups === 0)
+  }
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges3 = tc => {
+  checkResult(Y.applyRandomTests(tc, qChanges, 3))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges30 = tc => {
+  checkResult(Y.applyRandomTests(tc, qChanges, 30))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges40 = tc => {
+  checkResult(Y.applyRandomTests(tc, qChanges, 40))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges70 = tc => {
+  checkResult(Y.applyRandomTests(tc, qChanges, 70))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges100 = tc => {
+  checkResult(Y.applyRandomTests(tc, qChanges, 100))
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatGenerateQuillChanges300 = tc => {
+  checkResult(Y.applyRandomTests(tc, qChanges, 300))
+}

tsconfig.json

@@ -39,10 +39,7 @@
     "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
     "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
     "paths": {
-      "yjs": ["./src/index.js"],
-      "lib0/*": ["node_modules/lib0/*"],
-      "lib0/set.js": ["node_modules/lib0/set.js"],
-      "lib0/function.js": ["node_modules/lib0/function.js"]
+      "yjs": ["./src/index.js"]
     },                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
     // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
     // "typeRoots": [],                       /* List of folders to include type definitions from. */
@@ -60,9 +57,8 @@
     /* Experimental Options */
     // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
     // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
-    "maxNodeModuleJsDepth": 5,
+    // "maxNodeModuleJsDepth": 0,
     // "types": ["./src/utils/typedefs.js"]
   },
-  "include": ["./src/**/*.js", "./tests/**/*.js"],
-  "exclude": ["../lib0/**/*", "node_modules/**/*", "dist", "dist/**/*.js"]
+  "include": ["./src/**/*.js", "./tests/**/*.js"]
 }