Implement experimental new encoder 🚀

.jsdoc.json

@@ -17,10 +17,13 @@
     "useCollapsibles": true,
     "collapse": true,
     "resources": {
-      "yjs.dev": "Yjs website"
+      "yjs.dev": "Website",
+      "docs.yjs.dev": "Docs",
+      "discuss.yjs.dev": "Forum",
+      "https://gitter.im/Yjs/community": "Chat"
     },
     "logo": {
-      "url": "https://user-images.githubusercontent.com/5553757/48975307-61efb100-f06d-11e8-9177-ee895e5916e5.png",
+      "url": "https://yjs.dev/images/logo/yjs-512x512.png",
       "width": "162px",
       "height": "162px",
       "link": "/"
@@ -35,7 +38,7 @@
     ],
     "default": {
       "staticFiles": {
-          "include": ["examples/"]
+          "include": []
       }
     }
   },
@@ -44,7 +47,6 @@
     "encoding": "utf8",
     "private": false,
     "recurse": true,
-    "template": "./node_modules/tui-jsdoc-template",
+    "template": "./node_modules/tui-jsdoc-template"
-    "tutorials": "./examples"
   }
 }

README.md

@@ -958,5 +958,6 @@ Yjs and all related projects are [**MIT licensed**](./LICENSE).
 Yjs is based on my research as a student at the [RWTH
 i5](http://dbis.rwth-aachen.de/). Now I am working on Yjs in my spare time.
 
-Fund this project by donating on [Patreon](https://www.patreon.com/dmonad) or
+Fund this project by donating on [GitHub Sponsors](https://github.com/sponsors/dmonad)
-hiring [me](https://github.com/dmonad) for professional support.
+or hiring [me](https://github.com/dmonad) as a contractor for your collaborative
+app.

package-lock.json

@@ -1429,9 +1429,9 @@
       "dev": true
     },
     "isomorphic.js": {
-      "version": "0.1.3",
+      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.1.4.tgz",
-      "integrity": "sha512-pabBRLDwYefSsNS+qCazJ97o7P5xDTrNoxSYFTM09JlZTxPrOEPGKekwqUy3/Np4C4PHnVUXHYsZPOix0jELsA=="
+      "integrity": "sha512-t9zbgkjE7f9f2M6OSW49YEq0lUrSdAllBbWFUZoeck/rnnFae6UlhmDtXWs48VJY3ZpryCoZsRiAiKD44hPIGQ=="
     },
     "js-tokens": {
       "version": "4.0.0",
@@ -1548,9 +1548,9 @@
       }
     },
     "lib0": {
-      "version": "0.2.29",
+      "version": "0.2.32",
-      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.29.tgz",
+      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.32.tgz",
-      "integrity": "sha512-bcQqmh3bUDVXwZrAJnekTpNk0uaShRg1bHMK7uzBIDFAWWdMXXCUtQXO/d/XsIxCiskgHTqF4jiQmDdoPCMVIw==",
+      "integrity": "sha512-cHHKhHTojtvFSsthTk+CKuD17jMHIxuZxYpTzXj9TeQLPNoGNDPl6ax+J6eFETVe3ZvPMh3V0nGfJgGo6QgSvA==",
       "requires": {
         "isomorphic.js": "^0.1.3"
       }
@@ -1716,18 +1716,18 @@
       "dev": true
     },
     "markdownlint": {
-      "version": "0.20.3",
+      "version": "0.20.4",
-      "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.20.3.tgz",
+      "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.20.4.tgz",
-      "integrity": "sha512-J93s59tGvSFvAPWVUtEgxqPI0CHayTx1Z8poj1/4UJAquHGPIruWRMurkRldiNbgBiaQ4OOt15rHZbFfU6u05A==",
+      "integrity": "sha512-jpfaPgjT0OpeBbemjYNZbzGG3hCLcAIvrm/pEY3+q/szDScG6ZonDacqySVRJAv9glbo8y4wBPJ0wgW17+9GGA==",
       "dev": true,
       "requires": {
         "markdown-it": "10.0.0"
       }
     },
     "markdownlint-cli": {
-      "version": "0.23.1",
+      "version": "0.23.2",
-      "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.23.1.tgz",
+      "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.23.2.tgz",
-      "integrity": "sha512-UARWuPILksAcVLTosUv1F1tLognNYQ/qjLRIgWwQAYqdl3QQrTPurU/X9Z2jrdAJYlOim868QsufxjYJpH0K7Q==",
+      "integrity": "sha512-OSl5OZ8xzGN6z355cqRkiq67zPi3reJimklaF72p0554q85Dng5ToOjjSB9tDKZebSt85jX8cp+ruoQlPqOsPA==",
       "dev": true,
       "requires": {
         "commander": "~2.9.0",
@@ -1739,8 +1739,8 @@
         "jsonc-parser": "~2.2.0",
         "lodash.differencewith": "~4.5.0",
         "lodash.flatten": "~4.4.0",
-        "markdownlint": "~0.20.3",
+        "markdownlint": "~0.20.4",
-        "markdownlint-rule-helpers": "~0.10.0",
+        "markdownlint-rule-helpers": "~0.11.0",
         "minimatch": "~3.0.4",
         "minimist": "~1.2.5",
         "rc": "~1.2.7"
@@ -1770,9 +1770,9 @@
       }
     },
     "markdownlint-rule-helpers": {
-      "version": "0.10.0",
+      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/markdownlint-rule-helpers/-/markdownlint-rule-helpers-0.10.0.tgz",
+      "resolved": "https://registry.npmjs.org/markdownlint-rule-helpers/-/markdownlint-rule-helpers-0.11.0.tgz",
-      "integrity": "sha512-0e8VUTjNdQwS7hTyNan9oOLsy4a7KEsXo3fxKMDRFRk6Jn+pLB3iKZ3mj/m6ECrlOUCxPYYmgOmmyk3bSdbIvw==",
+      "integrity": "sha512-PhGii9dOiDJDXxiRMpK8N0FM9powprvRPsXALgkjlSPTwLh6ymH+iF3iUe3nq8KGu26tclFBlLL5xAGy/zb7FA==",
       "dev": true
     },
     "marked": {
@@ -2786,9 +2786,9 @@
       "dev": true
     },
     "typescript": {
-      "version": "3.9.3",
+      "version": "3.9.6",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.3.tgz",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.6.tgz",
-      "integrity": "sha512-D/wqnB2xzNFIcoBG9FG8cXRDjiqSTbG2wd8DMZeQyJlP1vfTkIxH4GKveWaEBYySKIg+USu+E+EDIR47SqnaMQ==",
+      "integrity": "sha512-Pspx3oKAPJtjNwE92YS05HQoY7z2SFyOpHo9MqJor3BXAGNaPUs83CuVp9VISFkSjyRfiTpmKuAYGJB7S7hOxw==",
       "dev": true
     },
     "uc.micro": {

package.json

@@ -18,7 +18,7 @@
     "lint": "markdownlint README.md && standard && tsc",
     "docs": "rm -rf docs; jsdoc --configure ./.jsdoc.json --verbose --readme ./README.md --package ./package.json || true",
     "serve-docs": "npm run docs && http-server ./docs/",
-    "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000",
+    "preversion": "npm run lint && PRODUCTION=1 npm run dist && npm run docs && node ./dist/tests.cjs --repitition-time 1000 && test -e dist/src/index.d.ts && test -e dist/yjs.cjs && test -e dist/yjs.cjs",
     "debug": "concurrently 'http-server -o test.html' 'npm run watch'",
     "trace-deopt": "clear && rollup -c  && node --trace-deopt dist/test.cjs",
     "trace-opt": "clear && rollup -c  && node --trace-opt dist/test.cjs"
@@ -60,7 +60,7 @@
   },
   "homepage": "https://yjs.dev",
   "dependencies": {
-    "lib0": "^0.2.29"
+    "lib0": "^0.2.32"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^11.1.0",
@@ -68,12 +68,12 @@
     "concurrently": "^3.6.1",
     "http-server": "^0.12.3",
     "jsdoc": "^3.6.4",
-    "markdownlint-cli": "^0.23.1",
+    "markdownlint-cli": "^0.23.2",
     "rollup": "^1.32.1",
     "rollup-cli": "^1.0.9",
     "standard": "^14.3.4",
     "tui-jsdoc-template": "^1.2.2",
-    "typescript": "^3.9.3",
+    "typescript": "^3.9.6",
     "y-protocols": "^0.2.3"
   }
 }

src/index.js

@@ -48,12 +48,18 @@ export {
   typeMapGetSnapshot,
   iterateDeletedStructs,
   applyUpdate,
+  applyUpdateV2,
   readUpdate,
+  readUpdateV2,
   encodeStateAsUpdate,
+  encodeStateAsUpdateV2,
   encodeStateVector,
+  encodeStateVectorV2,
   UndoManager,
   decodeSnapshot,
   encodeSnapshot,
+  decodeSnapshotV2,
+  encodeSnapshotV2,
   isDeleted,
   isParentOf,
   equalSnapshots,

src/internals.js

@@ -1,6 +1,8 @@
 
 export * from './utils/DeleteSet.js'
 export * from './utils/Doc.js'
+export * from './utils/UpdateDecoder.js'
+export * from './utils/UpdateEncoder.js'
 export * from './utils/encoding.js'
 export * from './utils/EventHandler.js'
 export * from './utils/ID.js'

src/structs/AbstractStruct.js

@@ -1,9 +1,8 @@
 
 import {
-  StructStore, ID, Transaction // eslint-disable-line
+  AbstractUpdateEncoder, ID, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js' // eslint-disable-line
 import * as error from 'lib0/error.js'
 
 export class AbstractStruct {
@@ -35,7 +34,7 @@ export class AbstractStruct {
   }
 
   /**
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    * @param {number} offset
    * @param {number} encodingRef
    */

src/structs/ContentAny.js

@@ -1,10 +1,7 @@
 import {
-  Transaction, Item, StructStore // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 export class ContentAny {
   /**
    * @param {Array<any>} arr
@@ -77,15 +74,15 @@ export class ContentAny {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
     const len = this.arr.length
-    encoding.writeVarUint(encoder, len - offset)
+    encoder.writeLen(len - offset)
     for (let i = offset; i < len; i++) {
       const c = this.arr[i]
-      encoding.writeAny(encoder, c)
+      encoder.writeAny(c)
     }
   }
 
@@ -98,14 +95,14 @@ export class ContentAny {
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentAny}
  */
 export const readContentAny = decoder => {
-  const len = decoding.readVarUint(decoder)
+  const len = decoder.readLen()
   const cs = []
   for (let i = 0; i < len; i++) {
-    cs.push(decoding.readAny(decoder))
+    cs.push(decoder.readAny())
   }
   return new ContentAny(cs)
 }

src/structs/ContentBinary.js

@@ -1,10 +1,7 @@
 import {
-  StructStore, Item, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-import * as buffer from 'lib0/buffer.js'
 import * as error from 'lib0/error.js'
 
 export class ContentBinary {
@@ -73,11 +70,11 @@ export class ContentBinary {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarUint8Array(encoder, this.content)
+    encoder.writeBuf(this.content)
   }
 
   /**
@@ -89,7 +86,7 @@ export class ContentBinary {
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentBinary}
  */
-export const readContentBinary = decoder => new ContentBinary(buffer.copyUint8Array(decoding.readVarUint8Array(decoder)))
+export const readContentBinary = decoder => new ContentBinary(decoder.readBuf())

src/structs/ContentDeleted.js

@@ -1,12 +1,9 @@
 
 import {
   addToDeleteSet,
-  StructStore, Item, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 export class ContentDeleted {
   /**
    * @param {number} len
@@ -67,7 +64,7 @@ export class ContentDeleted {
    * @param {Item} item
    */
   integrate (transaction, item) {
-    addToDeleteSet(transaction.deleteSet, item.id, this.len)
+    addToDeleteSet(transaction.deleteSet, item.id.client, item.id.clock, this.len)
     item.markDeleted()
   }
 
@@ -80,11 +77,11 @@ export class ContentDeleted {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarUint(encoder, this.len - offset)
+    encoder.writeLen(this.len - offset)
   }
 
   /**
@@ -98,7 +95,7 @@ export class ContentDeleted {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentDeleted}
  */
-export const readContentDeleted = decoder => new ContentDeleted(decoding.readVarUint(decoder))
+export const readContentDeleted = decoder => new ContentDeleted(decoder.readLen())

src/structs/ContentEmbed.js

@@ -1,10 +1,8 @@
 
 import {
-  StructStore, Item, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Item, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 import * as error from 'lib0/error.js'
 
 /**
@@ -76,11 +74,11 @@ export class ContentEmbed {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarString(encoder, JSON.stringify(this.embed))
+    encoder.writeJSON(this.embed)
   }
 
   /**
@@ -94,7 +92,7 @@ export class ContentEmbed {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentEmbed}
  */
-export const readContentEmbed = decoder => new ContentEmbed(JSON.parse(decoding.readVarString(decoder)))
+export const readContentEmbed = decoder => new ContentEmbed(decoder.readJSON())

src/structs/ContentFormat.js

@@ -1,10 +1,8 @@
 
 import {
-  Item, StructStore, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Item, StructStore, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 import * as error from 'lib0/error.js'
 
 /**
@@ -78,12 +76,12 @@ export class ContentFormat {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarString(encoder, this.key)
+    encoder.writeKey(this.key)
-    encoding.writeVarString(encoder, JSON.stringify(this.value))
+    encoder.writeJSON(this.value)
   }
 
   /**
@@ -95,7 +93,7 @@ export class ContentFormat {
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentFormat}
  */
-export const readContentFormat = decoder => new ContentFormat(decoding.readVarString(decoder), JSON.parse(decoding.readVarString(decoder)))
+export const readContentFormat = decoder => new ContentFormat(decoder.readString(), decoder.readJSON())

src/structs/ContentJSON.js

@@ -1,10 +1,7 @@
 import {
-  Transaction, Item, StructStore // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 /**
  * @private
  */
@@ -80,15 +77,15 @@ export class ContentJSON {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
     const len = this.arr.length
-    encoding.writeVarUint(encoder, len - offset)
+    encoder.writeLen(len - offset)
     for (let i = offset; i < len; i++) {
       const c = this.arr[i]
-      encoding.writeVarString(encoder, c === undefined ? 'undefined' : JSON.stringify(c))
+      encoder.writeString(c === undefined ? 'undefined' : JSON.stringify(c))
     }
   }
 
@@ -103,14 +100,14 @@ export class ContentJSON {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentJSON}
  */
 export const readContentJSON = decoder => {
-  const len = decoding.readVarUint(decoder)
+  const len = decoder.readLen()
   const cs = []
   for (let i = 0; i < len; i++) {
-    const c = decoding.readVarString(decoder)
+    const c = decoder.readString()
     if (c === 'undefined') {
       cs.push(undefined)
     } else {

src/structs/ContentString.js

@@ -1,10 +1,7 @@
 import {
-  Transaction, Item, StructStore // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Transaction, Item, StructStore // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 /**
  * @private
  */
@@ -80,11 +77,11 @@ export class ContentString {
    */
   gc (store) {}
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeVarString(encoder, offset === 0 ? this.str : this.str.slice(offset))
+    encoder.writeString(offset === 0 ? this.str : this.str.slice(offset))
   }
 
   /**
@@ -98,7 +95,7 @@ export class ContentString {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentString}
  */
-export const readContentString = decoder => new ContentString(decoding.readVarString(decoder))
+export const readContentString = decoder => new ContentString(decoder.readString())

src/structs/ContentType.js

@@ -7,15 +7,13 @@ import {
   readYXmlFragment,
   readYXmlHook,
   readYXmlText,
-  ID, StructStore, Transaction, Item, YEvent, AbstractType // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Transaction, Item, YEvent, AbstractType // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js' // eslint-disable-line
-import * as decoding from 'lib0/decoding.js'
 import * as error from 'lib0/error.js'
 
 /**
- * @type {Array<function(decoding.Decoder):AbstractType<any>>}
+ * @type {Array<function(AbstractUpdateDecoder):AbstractType<any>>}
  * @private
  */
 export const typeRefs = [
@@ -150,7 +148,7 @@ export class ContentType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
@@ -168,7 +166,7 @@ export class ContentType {
 /**
  * @private
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {ContentType}
  */
-export const readContentType = decoder => new ContentType(typeRefs[decoding.readVarUint(decoder)](decoder))
+export const readContentType = decoder => new ContentType(typeRefs[decoder.readTypeRef()](decoder))

src/structs/GC.js

@@ -2,11 +2,9 @@
 import {
   AbstractStruct,
   addStruct,
-  StructStore, Transaction, ID // eslint-disable-line
+  AbstractUpdateEncoder, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-
 export const structGCRefNumber = 0
 
 /**
@@ -41,12 +39,12 @@ export class GC extends AbstractStruct {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
-    encoding.writeUint8(encoder, structGCRefNumber)
+    encoder.writeInfo(structGCRefNumber)
-    encoding.writeVarUint(encoder, this.length - offset)
+    encoder.writeLen(this.length - offset)
   }
 
   /**

src/structs/Item.js

@@ -1,7 +1,5 @@
 
 import {
-  readID,
-  writeID,
   GC,
   getState,
   AbstractStruct,
@@ -23,12 +21,10 @@ import {
   readContentFormat,
   readContentType,
   addChangedTypeToTransaction,
-  Doc, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction // eslint-disable-line
 } from '../internals.js'
 
 import * as error from 'lib0/error.js'
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 import * as maplib from 'lib0/map.js'
 import * as set from 'lib0/set.js'
 import * as binary from 'lib0/binary.js'
@@ -574,7 +570,7 @@ export class Item extends AbstractStruct {
         parent._length -= this.length
       }
       this.markDeleted()
-      addToDeleteSet(transaction.deleteSet, this.id, this.length)
+      addToDeleteSet(transaction.deleteSet, this.id.client, this.id.clock, this.length)
       maplib.setIfUndefined(transaction.changed, parent, set.create).add(this.parentSub)
       this.content.delete(transaction)
     }
@@ -602,7 +598,7 @@ export class Item extends AbstractStruct {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    * @param {number} offset
    */
   write (encoder, offset) {
@@ -613,12 +609,12 @@ export class Item extends AbstractStruct {
       (origin === null ? 0 : binary.BIT8) | // origin is defined
       (rightOrigin === null ? 0 : binary.BIT7) | // right origin is defined
       (parentSub === null ? 0 : binary.BIT6) // parentSub is non-null
-    encoding.writeUint8(encoder, info)
+    encoder.writeInfo(info)
     if (origin !== null) {
-      writeID(encoder, origin)
+      encoder.writeLeftID(origin)
     }
     if (rightOrigin !== null) {
-      writeID(encoder, rightOrigin)
+      encoder.writeRightID(rightOrigin)
     }
     if (origin === null && rightOrigin === null) {
       const parent = /** @type {AbstractType<any>} */ (this.parent)
@@ -627,14 +623,14 @@ export class Item extends AbstractStruct {
         // parent type on y._map
         // find the correct key
         const ykey = findRootTypeKey(parent)
-        encoding.writeVarUint(encoder, 1) // write parentYKey
+        encoder.writeParentInfo(true) // write parentYKey
-        encoding.writeVarString(encoder, ykey)
+        encoder.writeString(ykey)
       } else {
-        encoding.writeVarUint(encoder, 0) // write parent id
+        encoder.writeParentInfo(false) // write parent id
-        writeID(encoder, parentItem.id)
+        encoder.writeLeftID(parentItem.id)
       }
       if (parentSub !== null) {
-        encoding.writeVarString(encoder, parentSub)
+        encoder.writeString(parentSub)
       }
     }
     this.content.write(encoder, offset)
@@ -642,15 +638,15 @@ export class Item extends AbstractStruct {
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @param {number} info
  */
-const readItemContent = (decoder, info) => contentRefs[info & binary.BITS5](decoder)
+export const readItemContent = (decoder, info) => contentRefs[info & binary.BITS5](decoder)
 
 /**
  * A lookup map for reading Item content.
  *
- * @type {Array<function(decoding.Decoder):AbstractContent>}
+ * @type {Array<function(AbstractUpdateDecoder):AbstractContent>}
  */
 export const contentRefs = [
   () => { throw error.unexpectedCase() }, // GC is not ItemContent
@@ -741,7 +737,7 @@ export class AbstractContent {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    * @param {number} offset
    */
   write (encoder, offset) {
@@ -755,38 +751,3 @@ export class AbstractContent {
     throw error.methodUnimplemented()
   }
 }
-
-/**
- * @param {decoding.Decoder} decoder
- * @param {ID} id
- * @param {number} info
- * @param {Doc} doc
- */
-export const readItem = (decoder, id, info, doc) => {
-  /**
-   * The item that was originally to the left of this item.
-   * @type {ID | null}
-   */
-  const origin = (info & binary.BIT8) === binary.BIT8 ? readID(decoder) : null
-  /**
-   * The item that was originally to the right of this item.
-   * @type {ID | null}
-   */
-  const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? readID(decoder) : null
-  const canCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
-  const hasParentYKey = canCopyParentInfo ? decoding.readVarUint(decoder) === 1 : false
-  /**
-   * If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
-   * and we read the next string as parentYKey.
-   * It indicates how we store/retrieve parent from `y.share`
-   * @type {string|null}
-   */
-  const parentYKey = canCopyParentInfo && hasParentYKey ? decoding.readVarString(decoder) : null
-
-  return new Item(
-    id, null, origin, null, rightOrigin,
-    canCopyParentInfo && !hasParentYKey ? readID(decoder) : (parentYKey ? doc.get(parentYKey) : null), // parent
-    canCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoding.readVarString(decoder) : null, // parentSub
-    /** @type {AbstractContent} */ (readItemContent(decoder, info)) // item content
-  )
-}

src/types/AbstractType.js

@@ -11,13 +11,12 @@ import {
   ContentAny,
   ContentBinary,
   getItemCleanStart,
-  ID, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
+  AbstractUpdateEncoder, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
 } from '../internals.js'
 
 import * as map from 'lib0/map.js'
 import * as iterator from 'lib0/iterator.js'
 import * as error from 'lib0/error.js'
-import * as encoding from 'lib0/encoding.js' // eslint-disable-line
 
 /**
  * Accumulate all (list) children of a type and return them as an Array.
@@ -116,7 +115,7 @@ export class AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) { }
 

src/types/YArray.js

@@ -15,12 +15,9 @@ import {
   YArrayRefID,
   callTypeObservers,
   transact,
-  Doc, Transaction, Item // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line
 } from '../internals.js'
 
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
-import * as encoding from 'lib0/encoding.js'
-
 /**
  * Event that describes the changes on a YArray
  * @template T
@@ -204,15 +201,15 @@ export class YArray extends AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YArrayRefID)
+    encoder.writeTypeRef(YArrayRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  *
  * @private
  * @function

src/types/YMap.js

@@ -14,11 +14,9 @@ import {
   YMapRefID,
   callTypeObservers,
   transact,
-  Doc, Transaction, Item // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, Transaction, Item // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
 import * as iterator from 'lib0/iterator.js'
 
 /**
@@ -229,15 +227,15 @@ export class YMap extends AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YMapRefID)
+    encoder.writeTypeRef(YMapRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  *
  * @private
  * @function

src/types/YText.js

@@ -20,11 +20,9 @@ import {
   splitSnapshotAffectedStructs,
   iterateDeletedStructs,
   iterateStructs,
-  ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
 } from '../internals.js'
 
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
-import * as encoding from 'lib0/encoding.js'
 import * as object from 'lib0/object.js'
 import * as map from 'lib0/map.js'
 
@@ -1096,15 +1094,15 @@ export class YText extends AbstractType {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YTextRefID)
+    encoder.writeTypeRef(YTextRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YText}
  *
  * @private

src/types/YXmlElement.js

@@ -8,12 +8,9 @@ import {
   typeMapGetAll,
   typeListForEach,
   YXmlElementRefID,
-  Snapshot, Doc, Item // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Snapshot, Doc, Item // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
-
 /**
  * An YXmlElement imitates the behavior of a
  * {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Dom Element}.
@@ -181,18 +178,18 @@ export class YXmlElement extends YXmlFragment {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YXmlElementRefID)
+    encoder.writeTypeRef(YXmlElementRefID)
-    encoding.writeVarString(encoder, this.nodeName)
+    encoder.writeKey(this.nodeName)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlElement}
  *
  * @function
  */
-export const readYXmlElement = decoder => new YXmlElement(decoding.readVarString(decoder))
+export const readYXmlElement = decoder => new YXmlElement(decoder.readKey())

src/types/YXmlFragment.js

@@ -14,12 +14,9 @@ import {
   YXmlFragmentRefID,
   callTypeObservers,
   transact,
-  Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder, Doc, ContentType, Transaction, Item, YXmlText, YXmlHook, Snapshot // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
-
 /**
  * Define the elements to which a set of CSS queries apply.
  * {@link https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors|CSS_Selectors}
@@ -325,15 +322,15 @@ export class YXmlFragment extends AbstractType {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YXmlFragmentRefID)
+    encoder.writeTypeRef(YXmlFragmentRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlFragment}
  *
  * @private

src/types/YXmlHook.js

@@ -1,10 +1,9 @@
 
 import {
   YMap,
-  YXmlHookRefID
+  YXmlHookRefID,
+  AbstractUpdateDecoder, AbstractUpdateEncoder // eslint-disable-line
 } from '../internals.js'
-import * as encoding from 'lib0/encoding.js'
-import * as decoding from 'lib0/decoding.js'
 
 /**
  * You can manage binding to a custom type with YXmlHook.
@@ -66,21 +65,20 @@ export class YXmlHook extends YMap {
    *
    * This is called when this Item is sent to a remote peer.
    *
-   * @param {encoding.Encoder} encoder The encoder to write data to.
+   * @param {AbstractUpdateEncoder} encoder The encoder to write data to.
    */
   _write (encoder) {
-    super._write(encoder)
+    encoder.writeTypeRef(YXmlHookRefID)
-    encoding.writeVarUint(encoder, YXmlHookRefID)
+    encoder.writeKey(this.hookName)
-    encoding.writeVarString(encoder, this.hookName)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlHook}
  *
  * @private
  * @function
  */
 export const readYXmlHook = decoder =>
-  new YXmlHook(decoding.readVarString(decoder))
+  new YXmlHook(decoder.readKey())

src/types/YXmlText.js

@@ -1,8 +1,9 @@
 
-import { YText, YXmlTextRefID } from '../internals.js'
+import {
-
+  YText,
-import * as encoding from 'lib0/encoding.js'
+  YXmlTextRefID,
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
+  AbstractUpdateDecoder, AbstractUpdateEncoder // eslint-disable-line
+} from '../internals.js'
 
 /**
  * Represents text in a Dom Element. In the future this type will also handle
@@ -78,15 +79,15 @@ export class YXmlText extends YText {
   }
 
   /**
-   * @param {encoding.Encoder} encoder
+   * @param {AbstractUpdateEncoder} encoder
    */
   _write (encoder) {
-    encoding.writeVarUint(encoder, YXmlTextRefID)
+    encoder.writeTypeRef(YXmlTextRefID)
   }
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractUpdateDecoder} decoder
  * @return {YXmlText}
  *
  * @private

src/utils/DeleteSet.js

@@ -3,9 +3,8 @@ import {
   findIndexSS,
   getState,
   splitItem,
-  createID,
   iterateStructs,
-  Item, AbstractStruct, GC, StructStore, Transaction, ID // eslint-disable-line
+  AbstractUpdateDecoder, AbstractDSDecoder, AbstractDSEncoder, DSDecoderV2, DSEncoderV2, Item, GC, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as array from 'lib0/array.js'
@@ -163,14 +162,15 @@ export const mergeDeleteSets = dss => {
 
 /**
  * @param {DeleteSet} ds
- * @param {ID} id
+ * @param {number} client
+ * @param {number} clock
  * @param {number} length
  *
  * @private
  * @function
  */
-export const addToDeleteSet = (ds, id, length) => {
+export const addToDeleteSet = (ds, client, clock, length) => {
-  map.setIfUndefined(ds.clients, id.client, () => []).push(new DeleteItem(id.clock, length))
+  map.setIfUndefined(ds.clients, client, () => []).push(new DeleteItem(clock, length))
 }
 
 export const createDeleteSet = () => new DeleteSet()
@@ -210,28 +210,29 @@ export const createDeleteSetFromStructStore = ss => {
 }
 
 /**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractDSEncoder} encoder
  * @param {DeleteSet} ds
  *
  * @private
  * @function
  */
 export const writeDeleteSet = (encoder, ds) => {
-  encoding.writeVarUint(encoder, ds.clients.size)
+  encoding.writeVarUint(encoder.restEncoder, ds.clients.size)
   ds.clients.forEach((dsitems, client) => {
-    encoding.writeVarUint(encoder, client)
+    encoder.resetDsCurVal()
+    encoding.writeVarUint(encoder.restEncoder, client)
     const len = dsitems.length
-    encoding.writeVarUint(encoder, len)
+    encoding.writeVarUint(encoder.restEncoder, len)
     for (let i = 0; i < len; i++) {
       const item = dsitems[i]
-      encoding.writeVarUint(encoder, item.clock)
+      encoder.writeDsClock(item.clock)
-      encoding.writeVarUint(encoder, item.len)
+      encoder.writeDsLen(item.len)
     }
   })
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @param {AbstractDSDecoder} decoder
  * @return {DeleteSet}
  *
  * @private
@@ -239,19 +240,27 @@ export const writeDeleteSet = (encoder, ds) => {
  */
 export const readDeleteSet = decoder => {
   const ds = new DeleteSet()
-  const numClients = decoding.readVarUint(decoder)
+  const numClients = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < numClients; i++) {
-    const client = decoding.readVarUint(decoder)
+    decoder.resetDsCurVal()
-    const numberOfDeletes = decoding.readVarUint(decoder)
+    const client = decoding.readVarUint(decoder.restDecoder)
-    for (let i = 0; i < numberOfDeletes; i++) {
+    const numberOfDeletes = decoding.readVarUint(decoder.restDecoder)
-      addToDeleteSet(ds, createID(client, decoding.readVarUint(decoder)), decoding.readVarUint(decoder))
+    if (numberOfDeletes > 0) {
+      const dsField = map.setIfUndefined(ds.clients, client, () => [])
+      for (let i = 0; i < numberOfDeletes; i++) {
+        dsField.push(new DeleteItem(decoder.readDsClock(), decoder.readDsLen()))
+      }
     }
   }
   return ds
 }
 
 /**
- * @param {decoding.Decoder} decoder
+ * @todo YDecoder also contains references to String and other Decoders. Would make sense to exchange YDecoder.toUint8Array for YDecoder.DsToUint8Array()..
+ */
+
+/**
+ * @param {AbstractDSDecoder} decoder
  * @param {Transaction} transaction
  * @param {StructStore} store
  *
@@ -260,18 +269,19 @@ export const readDeleteSet = decoder => {
  */
 export const readAndApplyDeleteSet = (decoder, transaction, store) => {
   const unappliedDS = new DeleteSet()
-  const numClients = decoding.readVarUint(decoder)
+  const numClients = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < numClients; i++) {
-    const client = decoding.readVarUint(decoder)
+    decoder.resetDsCurVal()
-    const numberOfDeletes = decoding.readVarUint(decoder)
+    const client = decoding.readVarUint(decoder.restDecoder)
+    const numberOfDeletes = decoding.readVarUint(decoder.restDecoder)
     const structs = store.clients.get(client) || []
     const state = getState(store, client)
     for (let i = 0; i < numberOfDeletes; i++) {
-      const clock = decoding.readVarUint(decoder)
+      const clock = decoder.readDsClock()
-      const len = decoding.readVarUint(decoder)
+      const clockEnd = clock + decoder.readDsLen()
       if (clock < state) {
-        if (state < clock + len) {
+        if (state < clockEnd) {
-          addToDeleteSet(unappliedDS, createID(client, state), clock + len - state)
+          addToDeleteSet(unappliedDS, client, state, clockEnd - state)
         }
         let index = findIndexSS(structs, clock)
         /**
@@ -288,10 +298,10 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => {
         while (index < structs.length) {
           // @ts-ignore
           struct = structs[index++]
-          if (struct.id.clock < clock + len) {
+          if (struct.id.clock < clockEnd) {
             if (!struct.deleted) {
-              if (clock + len < struct.id.clock + struct.length) {
+              if (clockEnd < struct.id.clock + struct.length) {
-                structs.splice(index, 0, splitItem(transaction, struct, clock + len - struct.id.clock))
+                structs.splice(index, 0, splitItem(transaction, struct, clockEnd - struct.id.clock))
               }
               struct.delete(transaction)
             }
@@ -300,14 +310,14 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => {
           }
         }
       } else {
-        addToDeleteSet(unappliedDS, createID(client, clock), len)
+        addToDeleteSet(unappliedDS, client, clock, clockEnd - clock)
       }
     }
   }
   if (unappliedDS.clients.size > 0) {
     // TODO: no need for encoding+decoding ds anymore
-    const unappliedDSEncoder = encoding.createEncoder()
+    const unappliedDSEncoder = new DSEncoderV2()
     writeDeleteSet(unappliedDSEncoder, unappliedDS)
-    store.pendingDeleteReaders.push(decoding.createDecoder(encoding.toUint8Array(unappliedDSEncoder)))
+    store.pendingDeleteReaders.push(new DSDecoderV2(decoding.createDecoder((unappliedDSEncoder.toUint8Array()))))
   }
 }

src/utils/PermanentUserData.js

@@ -5,11 +5,11 @@ import {
   readDeleteSet,
   writeDeleteSet,
   createDeleteSet,
-  ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line
+  DSEncoderV1, DSDecoderV1, ID, DeleteSet, YArrayEvent, Transaction, Doc // eslint-disable-line
 } from '../internals.js'
 
 import * as decoding from 'lib0/decoding.js'
-import * as encoding from 'lib0/encoding.js'
+
 import { mergeDeleteSets, isDeleted } from './DeleteSet.js'
 
 export class PermanentUserData {
@@ -46,12 +46,12 @@ export class PermanentUserData {
         event.changes.added.forEach(item => {
           item.content.getContent().forEach(encodedDs => {
             if (encodedDs instanceof Uint8Array) {
-              this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(decoding.createDecoder(encodedDs))]))
+              this.dss.set(userDescription, mergeDeleteSets([this.dss.get(userDescription) || createDeleteSet(), readDeleteSet(new DSDecoderV1(decoding.createDecoder(encodedDs)))]))
             }
           })
         })
       })
-      this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(decoding.createDecoder(encodedDs)))))
+      this.dss.set(userDescription, mergeDeleteSets(ds.map(encodedDs => readDeleteSet(new DSDecoderV1(encodedDs)))))
       ids.observe(/** @param {YArrayEvent<any>} event */ event =>
         event.changes.added.forEach(item => item.content.getContent().forEach(addClientId))
       )
@@ -97,11 +97,11 @@ export class PermanentUserData {
               user.get('ids').push([clientid])
             }
           })
-          const encoder = encoding.createEncoder()
+          const encoder = new DSEncoderV1()
           const ds = this.dss.get(userDescription)
           if (ds) {
             writeDeleteSet(encoder, ds)
-            user.get('ds').push([encoding.toUint8Array(encoder)])
+            user.get('ds').push([encoder.toUint8Array()])
           }
         }
       }, 0)
@@ -111,9 +111,9 @@ export class PermanentUserData {
         const yds = user.get('ds')
         const ds = transaction.deleteSet
         if (transaction.local && ds.clients.size > 0 && filter(transaction, ds)) {
-          const encoder = encoding.createEncoder()
+          const encoder = new DSEncoderV1()
           writeDeleteSet(encoder, ds)
-          yds.push([encoding.toUint8Array(encoder)])
+          yds.push([encoder.toUint8Array()])
         }
       })
     })

src/utils/Snapshot.js

@@ -12,13 +12,13 @@ import {
   createDeleteSet,
   createID,
   getState,
-  Transaction, Doc, DeleteSet, Item // eslint-disable-line
+  AbstractDSDecoder, AbstractDSEncoder, DSEncoderV1, DSEncoderV2, DSDecoderV1, DSDecoderV2, Transaction, Doc, DeleteSet, Item // eslint-disable-line
 } from '../internals.js'
 
 import * as map from 'lib0/map.js'
 import * as set from 'lib0/set.js'
-import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
+import { DefaultDSEncoder } from './encoding.js'
 
 export class Snapshot {
   /**
@@ -74,23 +74,35 @@ export const equalSnapshots = (snap1, snap2) => {
 
 /**
  * @param {Snapshot} snapshot
+ * @param {AbstractDSEncoder} [encoder]
  * @return {Uint8Array}
  */
-export const encodeSnapshot = snapshot => {
+export const encodeSnapshotV2 = (snapshot, encoder = new DSEncoderV2()) => {
-  const encoder = encoding.createEncoder()
   writeDeleteSet(encoder, snapshot.ds)
   writeStateVector(encoder, snapshot.sv)
-  return encoding.toUint8Array(encoder)
+  return encoder.toUint8Array()
+}
+
+/**
+ * @param {Snapshot} snapshot
+ * @return {Uint8Array}
+ */
+export const encodeSnapshot = snapshot => encodeSnapshotV2(snapshot, new DefaultDSEncoder())
+
+/**
+ * @param {Uint8Array} buf
+ * @param {AbstractDSDecoder} [decoder]
+ * @return {Snapshot}
+ */
+export const decodeSnapshotV2 = (buf, decoder = new DSDecoderV2(decoding.createDecoder(buf))) => {
+  return new Snapshot(readDeleteSet(decoder), readStateVector(decoder))
 }
 
 /**
  * @param {Uint8Array} buf
  * @return {Snapshot}
  */
-export const decodeSnapshot = buf => {
+export const decodeSnapshot = buf => decodeSnapshotV2(buf, new DSDecoderV1(decoding.createDecoder(buf)))
-  const decoder = decoding.createDecoder(buf)
-  return new Snapshot(readDeleteSet(decoder), readStateVector(decoder))
-}
 
 /**
  * @param {DeleteSet} ds

src/utils/StructStore.js

@@ -2,12 +2,11 @@
 import {
   GC,
   splitItem,
-  AbstractStruct, Transaction, ID, Item // eslint-disable-line
+  Transaction, ID, Item, DSDecoderV2 // eslint-disable-line
 } from '../internals.js'
 
 import * as math from 'lib0/math.js'
 import * as error from 'lib0/error.js'
-import * as decoding from 'lib0/decoding.js' // eslint-disable-line
 
 export class StructStore {
   constructor () {
@@ -31,7 +30,7 @@ export class StructStore {
      */
     this.pendingStack = []
     /**
-     * @type {Array<decoding.Decoder>}
+     * @type {Array<DSDecoderV2>}
      */
     this.pendingDeleteReaders = []
   }

src/utils/Transaction.js

@@ -11,15 +11,16 @@ import {
   Item,
   generateNewClientId,
   createID,
-  GC, StructStore, ID, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line
+  AbstractUpdateEncoder, GC, StructStore, UpdateEncoderV1, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line
 } from '../internals.js'
 
-import * as encoding from 'lib0/encoding.js'
 import * as map from 'lib0/map.js'
 import * as math from 'lib0/math.js'
 import * as set from 'lib0/set.js'
 import * as logging from 'lib0/logging.js'
 import { callAll } from 'lib0/function.js'
+import { DefaultUpdateEncoder } from './encoding.js'
+import { UpdateEncoderV2 } from './UpdateEncoder.js'
 
 /**
  * A transaction is created for every change on the Yjs model. It is possible
@@ -107,17 +108,18 @@ export class Transaction {
 }
 
 /**
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Transaction} transaction
+ * @return {boolean} Whether data was written.
  */
-export const computeUpdateMessageFromTransaction = transaction => {
+export const writeUpdateMessageFromTransaction = (encoder, transaction) => {
   if (transaction.deleteSet.clients.size === 0 && !map.any(transaction.afterState, (clock, client) => transaction.beforeState.get(client) !== clock)) {
-    return null
+    return false
   }
-  const encoder = encoding.createEncoder()
   sortAndMergeDeleteSet(transaction.deleteSet)
   writeStructsFromTransaction(encoder, transaction)
   writeDeleteSet(encoder, transaction.deleteSet)
-  return encoder
+  return true
 }
 
 /**
@@ -322,9 +324,17 @@ const cleanupTransactions = (transactionCleanups, i) => {
       // @todo Merge all the transactions into one and provide send the data as a single update message
       doc.emit('afterTransactionCleanup', [transaction, doc])
       if (doc._observers.has('update')) {
-        const updateMessage = computeUpdateMessageFromTransaction(transaction)
+        const encoder = new DefaultUpdateEncoder()
-        if (updateMessage !== null) {
+        const hasContent = writeUpdateMessageFromTransaction(encoder, transaction)
-          doc.emit('update', [encoding.toUint8Array(updateMessage), transaction.origin, doc])
+        if (hasContent) {
+          doc.emit('update', [encoder.toUint8Array(), transaction.origin, doc])
+        }
+      }
+      if (doc._observers.has('updateV2')) {
+        const encoder = new UpdateEncoderV2()
+        const hasContent = writeUpdateMessageFromTransaction(encoder, transaction)
+        if (hasContent) {
+          doc.emit('updateV2', [encoder.toUint8Array(), transaction.origin, doc])
         }
       }
       if (transactionCleanups.length <= i + 1) {

src/utils/UpdateDecoder.js

@@ -0,0 +1,392 @@
+import * as buffer from 'lib0/buffer.js'
+import * as error from 'lib0/error.js'
+import * as decoding from 'lib0/decoding.js'
+import {
+  ID, createID
+} from '../internals.js'
+
+export class AbstractDSDecoder {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    this.restDecoder = decoder
+    error.methodUnimplemented()
+  }
+
+  resetDsCurVal () { }
+
+  /**
+   * @return {number}
+   */
+  readDsClock () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {number}
+   */
+  readDsLen () {
+    error.methodUnimplemented()
+  }
+}
+
+export class AbstractUpdateDecoder extends AbstractDSDecoder {
+  /**
+   * @return {ID}
+   */
+  readLeftID () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {ID}
+   */
+  readRightID () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Read the next client id.
+   * Use this in favor of readID whenever possible to reduce the number of objects created.
+   *
+   * @return {number}
+   */
+  readClient () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readInfo () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {string}
+   */
+  readString () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {boolean} isKey
+   */
+  readParentInfo () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readTypeRef () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @return {number} len
+   */
+  readLen () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {any}
+   */
+  readAny () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  readBuf () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Legacy implementation uses JSON parse. We use any-decoding in v2.
+   *
+   * @return {any}
+   */
+  readJSON () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @return {string}
+   */
+  readKey () {
+    error.methodUnimplemented()
+  }
+}
+
+export class DSDecoderV1 {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    this.restDecoder = decoder
+  }
+
+  resetDsCurVal () {
+    // nop
+  }
+
+  /**
+   * @return {number}
+   */
+  readDsClock () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * @return {number}
+   */
+  readDsLen () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+}
+
+export class UpdateDecoderV1 extends DSDecoderV1 {
+  /**
+   * @return {ID}
+   */
+  readLeftID () {
+    return createID(decoding.readVarUint(this.restDecoder), decoding.readVarUint(this.restDecoder))
+  }
+
+  /**
+   * @return {ID}
+   */
+  readRightID () {
+    return createID(decoding.readVarUint(this.restDecoder), decoding.readVarUint(this.restDecoder))
+  }
+
+  /**
+   * Read the next client id.
+   * Use this in favor of readID whenever possible to reduce the number of objects created.
+   */
+  readClient () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readInfo () {
+    return decoding.readUint8(this.restDecoder)
+  }
+
+  /**
+   * @return {string}
+   */
+  readString () {
+    return decoding.readVarString(this.restDecoder)
+  }
+
+  /**
+   * @return {boolean} isKey
+   */
+  readParentInfo () {
+    return decoding.readVarUint(this.restDecoder) === 1
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readTypeRef () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @return {number} len
+   */
+  readLen () {
+    return decoding.readVarUint(this.restDecoder)
+  }
+
+  /**
+   * @return {any}
+   */
+  readAny () {
+    return decoding.readAny(this.restDecoder)
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  readBuf () {
+    return buffer.copyUint8Array(decoding.readVarUint8Array(this.restDecoder))
+  }
+
+  /**
+   * Legacy implementation uses JSON parse. We use any-decoding in v2.
+   *
+   * @return {any}
+   */
+  readJSON () {
+    return JSON.parse(decoding.readVarString(this.restDecoder))
+  }
+
+  /**
+   * @return {string}
+   */
+  readKey () {
+    return decoding.readVarString(this.restDecoder)
+  }
+}
+
+export class DSDecoderV2 {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    this.dsCurrVal = 0
+    this.restDecoder = decoder
+  }
+
+  resetDsCurVal () {
+    this.dsCurrVal = 0
+  }
+
+  readDsClock () {
+    this.dsCurrVal += decoding.readVarUint(this.restDecoder)
+    return this.dsCurrVal
+  }
+
+  readDsLen () {
+    const diff = decoding.readVarUint(this.restDecoder) + 1
+    this.dsCurrVal += diff
+    return diff
+  }
+}
+
+export class UpdateDecoderV2 extends DSDecoderV2 {
+  /**
+   * @param {decoding.Decoder} decoder
+   */
+  constructor (decoder) {
+    super(decoder)
+    /**
+     * List of cached keys. If the keys[id] does not exist, we read a new key
+     * from stringEncoder and push it to keys.
+     *
+     * @type {Array<string>}
+     */
+    this.keys = []
+    decoding.readUint8(decoder) // read feature flag - currently unused
+    this.keyClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.clientDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.leftClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.rightClockDecoder = new decoding.IntDiffOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.infoDecoder = new decoding.RleDecoder(decoding.readVarUint8Array(decoder), decoding.readUint8)
+    this.stringDecoder = new decoding.StringDecoder(decoding.readVarUint8Array(decoder))
+    this.parentInfoDecoder = new decoding.RleDecoder(decoding.readVarUint8Array(decoder), decoding.readUint8)
+    this.typeRefDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder))
+    this.lenDecoder = new decoding.UintOptRleDecoder(decoding.readVarUint8Array(decoder))
+  }
+
+  /**
+   * @return {ID}
+   */
+  readLeftID () {
+    return new ID(this.clientDecoder.read(), this.leftClockDecoder.read())
+  }
+
+  /**
+   * @return {ID}
+   */
+  readRightID () {
+    return new ID(this.clientDecoder.read(), this.rightClockDecoder.read())
+  }
+
+  /**
+   * Read the next client id.
+   * Use this in favor of readID whenever possible to reduce the number of objects created.
+   */
+  readClient () {
+    return this.clientDecoder.read()
+  }
+
+  /**
+   * @return {number} info An unsigned 8-bit integer
+   */
+  readInfo () {
+    return /** @type {number} */ (this.infoDecoder.read())
+  }
+
+  /**
+   * @return {string}
+   */
+  readString () {
+    return this.stringDecoder.read()
+  }
+
+  /**
+   * @return {boolean}
+   */
+  readParentInfo () {
+    return this.parentInfoDecoder.read() === 1
+  }
+
+  /**
+   * @return {number} An unsigned 8-bit integer
+   */
+  readTypeRef () {
+    return this.typeRefDecoder.read()
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @return {number}
+   */
+  readLen () {
+    return this.lenDecoder.read()
+  }
+
+  /**
+   * @return {any}
+   */
+  readAny () {
+    return decoding.readAny(this.restDecoder)
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  readBuf () {
+    return decoding.readVarUint8Array(this.restDecoder)
+  }
+
+  /**
+   * This is mainly here for legacy purposes.
+   *
+   * Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder.
+   *
+   * @return {any}
+   */
+  readJSON () {
+    return decoding.readAny(this.restDecoder)
+  }
+
+  /**
+   * @return {string}
+   */
+  readKey () {
+    const keyClock = this.keyClockDecoder.read()
+    if (keyClock < this.keys.length) {
+      return this.keys[keyClock]
+    } else {
+      const key = this.stringDecoder.read()
+      this.keys.push(key)
+      return key
+    }
+  }
+}

src/utils/UpdateEncoder.js

@@ -0,0 +1,408 @@
+
+import * as error from 'lib0/error.js'
+import * as encoding from 'lib0/encoding.js'
+
+import {
+  ID // eslint-disable-line
+} from '../internals.js'
+
+export class AbstractDSEncoder {
+  constructor () {
+    this.restEncoder = encoding.createEncoder()
+  }
+
+  /**
+   * @return {Uint8Array}
+   */
+  toUint8Array () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * Resets the ds value to 0.
+   * The v2 encoder uses this information to reset the initial diff value.
+   */
+  resetDsCurVal () { }
+
+  /**
+   * @param {number} clock
+   */
+  writeDsClock (clock) { }
+
+  /**
+   * @param {number} len
+   */
+  writeDsLen (len) { }
+}
+
+export class AbstractUpdateEncoder extends AbstractDSEncoder {
+  /**
+   * @return {Uint8Array}
+   */
+  toUint8Array () {
+    error.methodUnimplemented()
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeLeftID (id) { }
+
+  /**
+   * @param {ID} id
+   */
+  writeRightID (id) { }
+
+  /**
+   * Use writeClient and writeClock instead of writeID if possible.
+   * @param {number} client
+   */
+  writeClient (client) { }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeInfo (info) { }
+
+  /**
+   * @param {string} s
+   */
+  writeString (s) { }
+
+  /**
+   * @param {boolean} isYKey
+   */
+  writeParentInfo (isYKey) { }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeTypeRef (info) { }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @param {number} len
+   */
+  writeLen (len) { }
+
+  /**
+   * @param {any} any
+   */
+  writeAny (any) { }
+
+  /**
+   * @param {Uint8Array} buf
+   */
+  writeBuf (buf) { }
+
+  /**
+   * @param {any} embed
+   */
+  writeJSON (embed) { }
+
+  /**
+   * @param {string} key
+   */
+  writeKey (key) { }
+}
+
+export class DSEncoderV1 {
+  constructor () {
+    this.restEncoder = new encoding.Encoder()
+  }
+
+  toUint8Array () {
+    return encoding.toUint8Array(this.restEncoder)
+  }
+
+  resetDsCurVal () {
+    // nop
+  }
+
+  /**
+   * @param {number} clock
+   */
+  writeDsClock (clock) {
+    encoding.writeVarUint(this.restEncoder, clock)
+  }
+
+  /**
+   * @param {number} len
+   */
+  writeDsLen (len) {
+    encoding.writeVarUint(this.restEncoder, len)
+  }
+}
+
+export class UpdateEncoderV1 extends DSEncoderV1 {
+  /**
+   * @param {ID} id
+   */
+  writeLeftID (id) {
+    encoding.writeVarUint(this.restEncoder, id.client)
+    encoding.writeVarUint(this.restEncoder, id.clock)
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeRightID (id) {
+    encoding.writeVarUint(this.restEncoder, id.client)
+    encoding.writeVarUint(this.restEncoder, id.clock)
+  }
+
+  /**
+   * Use writeClient and writeClock instead of writeID if possible.
+   * @param {number} client
+   */
+  writeClient (client) {
+    encoding.writeVarUint(this.restEncoder, client)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeInfo (info) {
+    encoding.writeUint8(this.restEncoder, info)
+  }
+
+  /**
+   * @param {string} s
+   */
+  writeString (s) {
+    encoding.writeVarString(this.restEncoder, s)
+  }
+
+  /**
+   * @param {boolean} isYKey
+   */
+  writeParentInfo (isYKey) {
+    encoding.writeVarUint(this.restEncoder, isYKey ? 1 : 0)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeTypeRef (info) {
+    encoding.writeVarUint(this.restEncoder, info)
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @param {number} len
+   */
+  writeLen (len) {
+    encoding.writeVarUint(this.restEncoder, len)
+  }
+
+  /**
+   * @param {any} any
+   */
+  writeAny (any) {
+    encoding.writeAny(this.restEncoder, any)
+  }
+
+  /**
+   * @param {Uint8Array} buf
+   */
+  writeBuf (buf) {
+    encoding.writeVarUint8Array(this.restEncoder, buf)
+  }
+
+  /**
+   * @param {any} embed
+   */
+  writeJSON (embed) {
+    encoding.writeVarString(this.restEncoder, JSON.stringify(embed))
+  }
+
+  /**
+   * @param {string} key
+   */
+  writeKey (key) {
+    encoding.writeVarString(this.restEncoder, key)
+  }
+}
+
+export class DSEncoderV2 {
+  constructor () {
+    this.restEncoder = new encoding.Encoder() // encodes all the rest / non-optimized
+    this.dsCurrVal = 0
+  }
+
+  toUint8Array () {
+    return encoding.toUint8Array(this.restEncoder)
+  }
+
+  resetDsCurVal () {
+    this.dsCurrVal = 0
+  }
+
+  /**
+   * @param {number} clock
+   */
+  writeDsClock (clock) {
+    const diff = clock - this.dsCurrVal
+    this.dsCurrVal = clock
+    encoding.writeVarUint(this.restEncoder, diff)
+  }
+
+  /**
+   * @param {number} len
+   */
+  writeDsLen (len) {
+    if (len === 0) {
+      error.unexpectedCase()
+    }
+    encoding.writeVarUint(this.restEncoder, len - 1)
+    this.dsCurrVal += len
+  }
+}
+
+export class UpdateEncoderV2 extends DSEncoderV2 {
+  constructor () {
+    super()
+    /**
+     * @type {Map<string,number>}
+     */
+    this.keyMap = new Map()
+    /**
+     * Refers to the next uniqe key-identifier to me used.
+     * See writeKey method for more information.
+     *
+     * @type {number}
+     */
+    this.keyClock = 0
+    this.keyClockEncoder = new encoding.IntDiffOptRleEncoder()
+    this.clientEncoder = new encoding.UintOptRleEncoder()
+    this.leftClockEncoder = new encoding.IntDiffOptRleEncoder()
+    this.rightClockEncoder = new encoding.IntDiffOptRleEncoder()
+    this.infoEncoder = new encoding.RleEncoder(encoding.writeUint8)
+    this.stringEncoder = new encoding.StringEncoder()
+    this.parentInfoEncoder = new encoding.RleEncoder(encoding.writeUint8)
+    this.typeRefEncoder = new encoding.UintOptRleEncoder()
+    this.lenEncoder = new encoding.UintOptRleEncoder()
+  }
+
+  toUint8Array () {
+    const encoder = encoding.createEncoder()
+    encoding.writeUint8(encoder, 0) // this is a feature flag that we might use in the future
+    encoding.writeVarUint8Array(encoder, this.keyClockEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.clientEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.leftClockEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.rightClockEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, encoding.toUint8Array(this.infoEncoder))
+    encoding.writeVarUint8Array(encoder, this.stringEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, encoding.toUint8Array(this.parentInfoEncoder))
+    encoding.writeVarUint8Array(encoder, this.typeRefEncoder.toUint8Array())
+    encoding.writeVarUint8Array(encoder, this.lenEncoder.toUint8Array())
+    // @note The rest encoder is appended! (note the missing var)
+    encoding.writeUint8Array(encoder, encoding.toUint8Array(this.restEncoder))
+    return encoding.toUint8Array(encoder)
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeLeftID (id) {
+    this.clientEncoder.write(id.client)
+    this.leftClockEncoder.write(id.clock)
+  }
+
+  /**
+   * @param {ID} id
+   */
+  writeRightID (id) {
+    this.clientEncoder.write(id.client)
+    this.rightClockEncoder.write(id.clock)
+  }
+
+  /**
+   * @param {number} client
+   */
+  writeClient (client) {
+    this.clientEncoder.write(client)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeInfo (info) {
+    this.infoEncoder.write(info)
+  }
+
+  /**
+   * @param {string} s
+   */
+  writeString (s) {
+    this.stringEncoder.write(s)
+  }
+
+  /**
+   * @param {boolean} isYKey
+   */
+  writeParentInfo (isYKey) {
+    this.parentInfoEncoder.write(isYKey ? 1 : 0)
+  }
+
+  /**
+   * @param {number} info An unsigned 8-bit integer
+   */
+  writeTypeRef (info) {
+    this.typeRefEncoder.write(info)
+  }
+
+  /**
+   * Write len of a struct - well suited for Opt RLE encoder.
+   *
+   * @param {number} len
+   */
+  writeLen (len) {
+    this.lenEncoder.write(len)
+  }
+
+  /**
+   * @param {any} any
+   */
+  writeAny (any) {
+    encoding.writeAny(this.restEncoder, any)
+  }
+
+  /**
+   * @param {Uint8Array} buf
+   */
+  writeBuf (buf) {
+    encoding.writeVarUint8Array(this.restEncoder, buf)
+  }
+
+  /**
+   * This is mainly here for legacy purposes.
+   *
+   * Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder.
+   *
+   * @param {any} embed
+   */
+  writeJSON (embed) {
+    encoding.writeAny(this.restEncoder, embed)
+  }
+
+  /**
+   * Property keys are often reused. For example, in y-prosemirror the key `bold` might
+   * occur very often. For a 3d application, the key `position` might occur very often.
+   *
+   * We cache these keys in a Map and refer to them via a unique number.
+   *
+   * @param {string} key
+   */
+  writeKey (key) {
+    const clock = this.keyMap.get(key)
+    if (clock === undefined) {
+      this.keyClockEncoder.write(this.keyClock++)
+      this.stringEncoder.write(key)
+    } else {
+      this.keyClockEncoder.write(this.keyClock++)
+    }
+  }
+}

src/utils/encoding.js

@@ -1,7 +1,8 @@
 
 /**
  * @module encoding
- *
+ */
+/*
  * We use the first five bits in the info flag for determining the type of the struct.
  *
  * 0: GC
@@ -16,8 +17,6 @@
 
 import {
   findIndexSS,
-  writeID,
-  readID,
   getState,
   createID,
   getStateVector,
@@ -25,16 +24,36 @@ import {
   writeDeleteSet,
   createDeleteSetFromStructStore,
   transact,
-  readItem,
+  readItemContent,
-  Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line
+  UpdateDecoderV1,
+  UpdateDecoderV2,
+  UpdateEncoderV1,
+  UpdateEncoderV2,
+  DSDecoderV2,
+  DSEncoderV2,
+  DSDecoderV1,
+  DSEncoderV1,
+  AbstractDSEncoder, AbstractDSDecoder, AbstractUpdateEncoder, AbstractUpdateDecoder, AbstractContent, Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding.js'
 import * as decoding from 'lib0/decoding.js'
 import * as binary from 'lib0/binary.js'
 
+export let DefaultDSEncoder = DSEncoderV1
+export let DefaultDSDecoder = DSDecoderV1
+export let DefaultUpdateEncoder = UpdateEncoderV1
+export let DefaultUpdateDecoder = UpdateDecoderV1
+
+export const useV2Encoding = () => {
+  DefaultDSEncoder = DSEncoderV2
+  DefaultDSDecoder = DSDecoderV2
+  DefaultUpdateEncoder = UpdateEncoderV2
+  DefaultUpdateDecoder = UpdateDecoderV2
+}
+
 /**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Array<GC|Item>} structs All structs by `client`
  * @param {number} client
  * @param {number} clock write structs starting with `ID(client,clock)`
@@ -45,8 +64,9 @@ const writeStructs = (encoder, structs, client, clock) => {
   // write first id
   const startNewStructs = findIndexSS(structs, clock)
   // write # encoded structs
-  encoding.writeVarUint(encoder, structs.length - startNewStructs)
+  encoding.writeVarUint(encoder.restEncoder, structs.length - startNewStructs)
-  writeID(encoder, createID(client, clock))
+  encoder.writeClient(client)
+  encoding.writeVarUint(encoder.restEncoder, clock)
   const firstStruct = structs[startNewStructs]
   // write first struct with an offset
   firstStruct.write(encoder, clock - firstStruct.id.clock)
@@ -56,7 +76,7 @@ const writeStructs = (encoder, structs, client, clock) => {
 }
 
 /**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {StructStore} store
  * @param {Map<number,number>} _sm
  *
@@ -78,7 +98,7 @@ export const writeClientsStructs = (encoder, store, _sm) => {
     }
   })
   // write # states that were updated
-  encoding.writeVarUint(encoder, sm.size)
+  encoding.writeVarUint(encoder.restEncoder, sm.size)
   // Write items with higher client ids first
   // This heavily improves the conflict algorithm.
   Array.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => {
@@ -88,7 +108,7 @@ export const writeClientsStructs = (encoder, store, _sm) => {
 }
 
 /**
- * @param {decoding.Decoder} decoder The decoder object to read data from.
+ * @param {AbstractUpdateDecoder} decoder The decoder object to read data from.
  * @param {Map<number,Array<GC|Item>>} clientRefs
  * @param {Doc} doc
  * @return {Map<number,Array<GC|Item>>}
@@ -97,21 +117,52 @@ export const writeClientsStructs = (encoder, store, _sm) => {
  * @function
  */
 export const readClientsStructRefs = (decoder, clientRefs, doc) => {
-  const numOfStateUpdates = decoding.readVarUint(decoder)
+  const numOfStateUpdates = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < numOfStateUpdates; i++) {
-    const numberOfStructs = decoding.readVarUint(decoder)
+    const numberOfStructs = decoding.readVarUint(decoder.restDecoder)
     /**
      * @type {Array<GC|Item>}
      */
     const refs = []
-    let { client, clock } = readID(decoder)
+    const client = decoder.readClient()
-    let info, struct
+    let clock = decoding.readVarUint(decoder.restDecoder)
     clientRefs.set(client, refs)
     for (let i = 0; i < numberOfStructs; i++) {
-      info = decoding.readUint8(decoder)
+      const info = decoder.readInfo()
-      struct = (binary.BITS5 & info) === 0 ? new GC(createID(client, clock), decoding.readVarUint(decoder)) : readItem(decoder, createID(client, clock), info, doc)
+      if ((binary.BITS5 & info) !== 0) {
-      refs.push(struct)
+        /**
-      clock += struct.length
+         * The item that was originally to the left of this item.
+         * @type {ID | null}
+         */
+        const origin = (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null
+        /**
+         * The item that was originally to the right of this item.
+         * @type {ID | null}
+         */
+        const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null
+        const canCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
+        const hasParentYKey = canCopyParentInfo ? decoder.readParentInfo() : false
+        /**
+         * If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
+         * and we read the next string as parentYKey.
+         * It indicates how we store/retrieve parent from `y.share`
+         * @type {string|null}
+         */
+        const parentYKey = canCopyParentInfo && hasParentYKey ? decoder.readString() : null
+
+        const struct = new Item(
+          createID(client, clock), null, origin, null, rightOrigin,
+          canCopyParentInfo && !hasParentYKey ? decoder.readLeftID() : (parentYKey ? doc.get(parentYKey) : null), // parent
+          canCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub
+          /** @type {AbstractContent} */ (readItemContent(decoder, info)) // item content
+        )
+        refs.push(struct)
+        clock += struct.length
+      } else {
+        const len = decoder.readLen()
+        refs.push(new GC(createID(client, clock), len))
+        clock += len
+      }
     }
   }
   return clientRefs
@@ -222,7 +273,7 @@ export const tryResumePendingDeleteReaders = (transaction, store) => {
 }
 
 /**
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Transaction} transaction
  *
  * @private
@@ -275,7 +326,7 @@ const cleanupPendingStructs = pendingClientsStructRefs => {
  *
  * This is called when data is received from a remote peer.
  *
- * @param {decoding.Decoder} decoder The decoder object to read data from.
+ * @param {AbstractUpdateDecoder} decoder The decoder object to read data from.
  * @param {Transaction} transaction
  * @param {StructStore} store
  *
@@ -299,15 +350,46 @@ export const readStructs = (decoder, transaction, store) => {
  * @param {decoding.Decoder} decoder
  * @param {Doc} ydoc
  * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
+ * @param {AbstractUpdateDecoder} [structDecoder]
  *
  * @function
  */
-export const readUpdate = (decoder, ydoc, transactionOrigin) =>
+export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = new UpdateDecoderV2(decoder)) =>
   transact(ydoc, transaction => {
-    readStructs(decoder, transaction, ydoc.store)
+    readStructs(structDecoder, transaction, ydoc.store)
-    readAndApplyDeleteSet(decoder, transaction, ydoc.store)
+    readAndApplyDeleteSet(structDecoder, transaction, ydoc.store)
   }, transactionOrigin, false)
 
+/**
+ * Read and apply a document update.
+ *
+ * This function has the same effect as `applyUpdate` but accepts an decoder.
+ *
+ * @param {decoding.Decoder} decoder
+ * @param {Doc} ydoc
+ * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
+ *
+ * @function
+ */
+export const readUpdate = (decoder, ydoc, transactionOrigin) => readUpdateV2(decoder, ydoc, transactionOrigin, new DefaultUpdateDecoder(decoder))
+
+/**
+ * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`.
+ *
+ * This function has the same effect as `readUpdate` but accepts an Uint8Array instead of a Decoder.
+ *
+ * @param {Doc} ydoc
+ * @param {Uint8Array} update
+ * @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
+ * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder]
+ *
+ * @function
+ */
+export const applyUpdateV2 = (ydoc, update, transactionOrigin, YDecoder = UpdateDecoderV2) => {
+  const decoder = decoding.createDecoder(update)
+  readUpdateV2(decoder, ydoc, transactionOrigin, new YDecoder(decoder))
+}
+
 /**
  * Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`.
  *
@@ -319,14 +401,13 @@ export const readUpdate = (decoder, ydoc, transactionOrigin) =>
  *
  * @function
  */
-export const applyUpdate = (ydoc, update, transactionOrigin) =>
+export const applyUpdate = (ydoc, update, transactionOrigin) => applyUpdateV2(ydoc, update, transactionOrigin, DefaultUpdateDecoder)
-  readUpdate(decoding.createDecoder(update), ydoc, transactionOrigin)
 
 /**
  * Write all the document as a single update message. If you specify the state of the remote client (`targetStateVector`) it will
  * only write the operations that are missing.
  *
- * @param {encoding.Encoder} encoder
+ * @param {AbstractUpdateEncoder} encoder
  * @param {Doc} doc
  * @param {Map<number,number>} [targetStateVector] The state of the target that receives the update. Leave empty to write all known structs
  *
@@ -345,31 +426,45 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map())
  *
  * @param {Doc} doc
  * @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs
+ * @param {AbstractUpdateEncoder} [encoder]
  * @return {Uint8Array}
  *
  * @function
  */
-export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => {
+export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => {
-  const encoder = encoding.createEncoder()
   const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector)
   writeStateAsUpdate(encoder, doc, targetStateVector)
-  return encoding.toUint8Array(encoder)
+  return encoder.toUint8Array()
 }
 
+/**
+ * Write all the document as a single update message that can be applied on the remote document. If you specify the state of the remote client (`targetState`) it will
+ * only write the operations that are missing.
+ *
+ * Use `writeStateAsUpdate` instead if you are working with lib0/encoding.js#Encoder
+ *
+ * @param {Doc} doc
+ * @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs
+ * @return {Uint8Array}
+ *
+ * @function
+ */
+export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => encodeStateAsUpdateV2(doc, encodedTargetStateVector, new DefaultUpdateEncoder())
+
 /**
  * Read state vector from Decoder and return as Map
  *
- * @param {decoding.Decoder} decoder
+ * @param {AbstractDSDecoder} decoder
  * @return {Map<number,number>} Maps `client` to the number next expected `clock` from that client.
  *
  * @function
  */
 export const readStateVector = decoder => {
   const ss = new Map()
-  const ssLength = decoding.readVarUint(decoder)
+  const ssLength = decoding.readVarUint(decoder.restDecoder)
   for (let i = 0; i < ssLength; i++) {
-    const client = decoding.readVarUint(decoder)
+    const client = decoding.readVarUint(decoder.restDecoder)
-    const clock = decoding.readVarUint(decoder)
+    const clock = decoding.readVarUint(decoder.restDecoder)
     ss.set(client, clock)
   }
   return ss
@@ -383,28 +478,34 @@ export const readStateVector = decoder => {
  *
  * @function
  */
-export const decodeStateVector = decodedState => readStateVector(decoding.createDecoder(decodedState))
+export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
 
 /**
- * Write State Vector to `lib0/encoding.js#Encoder`.
+ * Read decodedState and return State as Map.
  *
- * @param {encoding.Encoder} encoder
+ * @param {Uint8Array} decodedState
+ * @return {Map<number,number>} Maps `client` to the number next expected `clock` from that client.
+ *
+ * @function
+ */
+export const decodeStateVector = decodedState => readStateVector(new DefaultDSDecoder(decoding.createDecoder(decodedState)))
+
+/**
+ * @param {AbstractDSEncoder} encoder
  * @param {Map<number,number>} sv
  * @function
  */
 export const writeStateVector = (encoder, sv) => {
-  encoding.writeVarUint(encoder, sv.size)
+  encoding.writeVarUint(encoder.restEncoder, sv.size)
   sv.forEach((clock, client) => {
-    encoding.writeVarUint(encoder, client)
+    encoding.writeVarUint(encoder.restEncoder, client) // @todo use a special client decoder that is based on mapping
-    encoding.writeVarUint(encoder, clock)
+    encoding.writeVarUint(encoder.restEncoder, clock)
   })
   return encoder
 }
 
 /**
- * Write State Vector to `lib0/encoding.js#Encoder`.
+ * @param {AbstractDSEncoder} encoder
- *
- * @param {encoding.Encoder} encoder
  * @param {Doc} doc
  *
  * @function
@@ -415,12 +516,22 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod
  * Encode State as Uint8Array.
  *
  * @param {Doc} doc
+ * @param {AbstractDSEncoder} [encoder]
  * @return {Uint8Array}
  *
  * @function
  */
-export const encodeStateVector = doc => {
+export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => {
-  const encoder = encoding.createEncoder()
   writeDocumentStateVector(encoder, doc)
-  return encoding.toUint8Array(encoder)
+  return encoder.toUint8Array()
 }
+
+/**
+ * Encode State as Uint8Array.
+ *
+ * @param {Doc} doc
+ * @return {Uint8Array}
+ *
+ * @function
+ */
+export const encodeStateVector = doc => encodeStateVectorV2(doc, new DefaultDSEncoder())

tests/testHelper.js

@@ -394,21 +394,21 @@ export const applyRandomTests = (tc, mods, iterations, initTestObject) => {
   const result = init(tc, { users: 5 }, initTestObject)
   const { testConnector, users } = result
   for (let i = 0; i < iterations; i++) {
-    if (prng.int31(gen, 0, 100) <= 2) {
+    if (prng.int32(gen, 0, 100) <= 2) {
       // 2% chance to disconnect/reconnect a random user
       if (prng.bool(gen)) {
         testConnector.disconnectRandom()
       } else {
         testConnector.reconnectRandom()
       }
-    } else if (prng.int31(gen, 0, 100) <= 1) {
+    } else if (prng.int32(gen, 0, 100) <= 1) {
       // 1% chance to flush all
       testConnector.flushAllMessages()
-    } else if (prng.int31(gen, 0, 100) <= 50) {
+    } else if (prng.int32(gen, 0, 100) <= 50) {
       // 50% chance to flush a random message
       testConnector.flushRandomMessage()
     }
-    const user = prng.int31(gen, 0, users.length - 1)
+    const user = prng.int32(gen, 0, users.length - 1)
     const test = prng.oneOf(gen, mods)
     test(users[user], gen, result.testObjects[user])
   }

tests/y-array.tests.js

@@ -5,6 +5,18 @@ import * as t from 'lib0/testing.js'
 import * as prng from 'lib0/prng.js'
 import * as math from 'lib0/math.js'
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testBasicUpdate = tc => {
+  const doc1 = new Y.Doc()
+  const doc2 = new Y.Doc()
+  doc1.getArray('array').insert(0, ['hi'])
+  const update = Y.encodeStateAsUpdate(doc1)
+  Y.applyUpdate(doc2, update)
+  t.compare(doc2.getArray('array').toArray(), ['hi'])
+}
+
 /**
  * @param {t.TestCase} tc
  */
@@ -335,23 +347,23 @@ const arrayTransactions = [
     const yarray = user.getArray('array')
     var uniqueNumber = getUniqueNumber()
     var content = []
-    var len = prng.int31(gen, 1, 4)
+    var len = prng.int32(gen, 1, 4)
     for (var i = 0; i < len; i++) {
       content.push(uniqueNumber)
     }
-    var pos = prng.int31(gen, 0, yarray.length)
+    var pos = prng.int32(gen, 0, yarray.length)
     yarray.insert(pos, content)
   },
   function insertTypeArray (user, gen) {
     const yarray = user.getArray('array')
-    var pos = prng.int31(gen, 0, yarray.length)
+    var pos = prng.int32(gen, 0, yarray.length)
     yarray.insert(pos, [new Y.Array()])
     var array2 = yarray.get(pos)
     array2.insert(0, [1, 2, 3, 4])
   },
   function insertTypeMap (user, gen) {
     const yarray = user.getArray('array')
-    var pos = prng.int31(gen, 0, yarray.length)
+    var pos = prng.int32(gen, 0, yarray.length)
     yarray.insert(pos, [new Y.Map()])
     var map = yarray.get(pos)
     map.set('someprop', 42)
@@ -362,13 +374,13 @@ const arrayTransactions = [
     const yarray = user.getArray('array')
     var length = yarray.length
     if (length > 0) {
-      var somePos = prng.int31(gen, 0, length - 1)
+      var somePos = prng.int32(gen, 0, length - 1)
-      var delLength = prng.int31(gen, 1, math.min(2, length - somePos))
+      var delLength = prng.int32(gen, 1, math.min(2, length - somePos))
       if (prng.bool(gen)) {
         var type = yarray.get(somePos)
         if (type.length > 0) {
-          somePos = prng.int31(gen, 0, type.length - 1)
+          somePos = prng.int32(gen, 0, type.length - 1)
-          delLength = prng.int31(gen, 0, math.min(2, type.length - somePos))
+          delLength = prng.int32(gen, 0, math.min(2, type.length - somePos))
           type.delete(somePos, delLength)
         }
       } else {

tests/y-text.tests.js

@@ -215,7 +215,7 @@ const tryGc = () => {
  * @param {t.TestCase} tc
  */
 export const testLargeFragmentedDocument = tc => {
-  const itemsToInsert = 2000000
+  const itemsToInsert = 1000000
   let update = /** @type {any} */ (null)
   ;(() => {
     const doc1 = new Y.Doc()
@@ -230,14 +230,15 @@ export const testLargeFragmentedDocument = tc => {
     })
     tryGc()
     t.measureTime('time to encode document', () => {
-      update = Y.encodeStateAsUpdate(doc1)
+      update = Y.encodeStateAsUpdateV2(doc1)
     })
+    t.describe('Document size:', update.byteLength)
   })()
   ;(() => {
     const doc2 = new Y.Doc()
     tryGc()
     t.measureTime(`time to apply ${itemsToInsert} updates`, () => {
-      Y.applyUpdate(doc2, update)
+      Y.applyUpdateV2(doc2, update)
     })
   })()
 }