v13.0.0-9 -- distribution files

.gitignore

@@ -1,15 +1,3 @@
 node_modules
 bower_components
-build
-build_test
-.directory
-.codio
-.settings
-.jshintignore
-.jshintrc
-.validate.json
-/y.js
-/y.js.map
-/y-*
-.vscode
-jsconfig.json
+/y.*

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "dist"]
-	path = dist
-	url = https://github.com/y-js/yjs.git
-	branch = dist

README.md

@@ -22,6 +22,7 @@ is a list of the modules we know of:
 |[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC|
 |[websockets](https://github.com/y-js/y-websockets-client) | Set up [a central server](https://github.com/y-js/y-websockets-client), and connect to it via websockets |
 |[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))|
+|[ipfs](https://github.com/ipfs-labs/y-ipfs-connector) | Connector for the [Interplanetary File System](https://ipfs.io/)!|
 |[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios|
 
 ##### Database adapters

examples/bower.json

@@ -9,16 +9,6 @@
   "license": "MIT",
   "ignore": [],
   "dependencies": {
-    "yjs": "latest",
-    "y-array": "latest",
-    "y-map": "latest",
-    "y-memory": "latest",
-    "y-richtext": "latest",
-    "y-webrtc": "latest",
-    "y-websockets-client": "latest",
-    "y-text": "latest",
-    "y-indexeddb": "latest",
-    "y-xml": "latest",
     "quill": "^1.0.0-rc.2",
     "ace": "~1.2.3",
     "ace-builds": "~1.2.3",

examples/chat/index.html

@@ -12,7 +12,12 @@
       <input name="message" type="text" style="width:60%;">
       <input type="submit" value="Send">
     </form>
-    <script src="../bower_components/yjs/y.js"></script>
+    <script src="../../y.js"></script>
+    <script src="../../../y-array/y-array.js"></script>
+    <script src="../../../y-map/dist/y-map.js"></script>
+    <script src="../../../y-text/dist/y-text.js"></script>
+    <script src="../../../y-memory/y-memory.js"></script>
+    <script src="../../../y-websockets-client/dist/y-websockets-client.js"></script>
     <script src="./index.js"></script>
   </body>
 </html>

examples/chat/index.js

@@ -1,5 +1,4 @@
-/* @flow */
-/* global Y */
+/* global Y, chat */
 
 // initialize a shared object. This function call returns a promise!
 Y({
@@ -17,10 +16,10 @@ Y({
 }).then(function (y) {
   window.yChat = y
   // This functions inserts a message at the specified position in the DOM
-  function appendMessage(message, position) { 
+  function appendMessage (message, position) {
     var p = document.createElement('p')
     var uname = document.createElement('span')
-    uname.appendChild(document.createTextNode(message.username + ": "))
+    uname.appendChild(document.createTextNode(message.username + ': '))
     p.appendChild(uname)
     p.appendChild(document.createTextNode(message.message))
     document.querySelector('#chat').insertBefore(p, chat.children[position] || null)
@@ -28,23 +27,22 @@ Y({
   // This function makes sure that only 7 messages exist in the chat history.
   // The rest is deleted
   function cleanupChat () {
-    var len
-    while ((len = y.share.chat.length) > 7) {
-      y.share.chat.delete(0)        
+    if (y.share.chat.length > 7) {
+      y.share.chat.delete(0, y.chat.length - 7)
     }
   }
   // Insert the initial content
   y.share.chat.toArray().forEach(appendMessage)
   cleanupChat()
-  
+
   // whenever content changes, make sure to reflect the changes in the DOM
   y.share.chat.observe(function (event) {
     if (event.type === 'insert') {
-      for (var i = 0; i < event.length; i++) {
+      for (let i = 0; i < event.length; i++) {
         appendMessage(event.values[i], event.index + i)
       }
     } else if (event.type === 'delete') {
-      for (var i = 0; i < event.length; i++) {
+      for (let i = 0; i < event.length; i++) {
         chat.children[event.index].remove()
       }
     }
@@ -54,8 +52,8 @@ Y({
   document.querySelector('#chatform').onsubmit = function (event) {
     // the form is submitted
     var message = {
-      username: this.querySelector("[name=username]").value,
-      message: this.querySelector("[name=message]").value
+      username: this.querySelector('[name=username]').value,
+      message: this.querySelector('[name=message]').value
     }
     if (message.username.length > 0 && message.message.length > 0) {
       if (y.share.chat.length > 6) {
@@ -66,10 +64,10 @@ Y({
       // This will call the observe function (see line 40)
       // and reflect the change in the DOM
       y.share.chat.push([message])
-      this.querySelector("[name=message]").value = ""
+      this.querySelector('[name=message]').value = ''
     }
     // Do not send this form!
     event.preventDefault()
     return false
   }
-})
+})

examples/drawing/index.html

@@ -12,7 +12,11 @@
   </style>
   <button type="button" id="clearDrawingCanvas">Clear Drawing</button>
   <svg id="drawingCanvas" viewbox="0 0 100 100" width="100%"></svg>
-  <script src="../bower_components/yjs/y.js"></script>
+  <script src="../../y.js"></script>
+  <script src="../../../y-array/y-array.js"></script>
+  <script src="../../../y-map/dist/y-map.js"></script>
+  <script src="../../../y-memory/y-memory.js"></script>
+  <script src="../../../y-websockets-client/y-websockets-client.js"></script>
   <script src="../bower_components/d3/d3.js"></script>
   <script src="./index.js"></script>
 </body>

examples/drawing/index.js

@@ -7,8 +7,8 @@ Y({
   },
   connector: {
     name: 'websockets-client',
-    room: 'drawing-example'
-    // url: 'localhost:1234'
+    room: 'drawing-example',
+    url: 'localhost:1234'
   },
   sourceDir: '/bower_components',
   share: {

examples/jigsaw/index.html

@@ -16,7 +16,10 @@
       <path d="M 325.63989,452.69678 C 317.92163,423.11035 287.36353,445.9126 268.64673,447.82227 C 250.27417,449.69727 239.77563,436.57373 239.77563,419.32568 C 239.77563,402.07812 254.02368,395.70361 265.27222,394.5791 C 276.52075,393.4541 282.14526,396.45361 297.89331,407.32715 C 313.64136,418.20117 326.3894,399.45313 327.8894,392.7041 C 329.3894,385.95508 320.76538,342.83545 320.01538,330.46191 C 319.75855,326.2207 320.56372,321.21484 321.58911,315.6377 C 315.92114,317.13525 309.58862,318.82715 302.39282,320.71338 C 263.77222,330.83691 232.27661,318.46338 221.40259,308.71484 C 210.52905,298.96582 229.27661,284.71777 234.90112,279.09326 C 240.52563,273.46924 243.90015,262.59521 239.77563,247.22217 C 235.65112,231.84912 201.90503,222.10059 179.03296,239.34814 C 156.16089,256.59619 180.90796,285.09277 188.40698,293.3418 C 195.90601,301.59033 186.15698,312.08936 170.03442,313.58887 C 160.60962,314.46582 119.15894,313.67676 86.340576,312.87012 C 85.573975,347.74561 84.581299,386.15088 83.794922,402.07812 C 82.295166,432.44922 109.29175,422.32568 115.66577,420.82568 C 122.04028,419.32568 126.16479,409.57715 143.03735,408.45215 C 185.9231,405.59326 186.09985,466.69629 144.16235,467.69482 C 128.41431,468.06982 113.79126,451.19678 108.16675,447.44727 C 102.54272,443.69775 87.919433,442.94775 83.794922,457.9458 C 82.01709,464.41113 78.118652,481.65137 78.098144,496.18994 C 78.071045,515.38037 82.295166,531.81201 82.295166,531.81201 C 82.295166,531.81201 105.54224,526.5625 149.41187,526.5625 C 193.28149,526.5625 199.65552,547.93506 194.78101,558.80859 C 189.90649,569.68213 181.28296,568.93213 179.40796,583.18066 C 172.7063,634.11133 253.34106,631.08203 249.14917,584.68018 C 247.96948,571.62354 237.16528,571.66699 232.27661,557.68359 C 222.17944,528.80273 244.64966,523.56299 257.39819,524.68799 C 263.59351,525.23437 290.95679,529.73389 320.75757,531.21582 C 321.22437,531.23877 321.69312,531.25928 322.16089,531.28125 C 326.09985,518.06592 329.95825,503.87646 330.1394,498.44092 C 330.51392,487.19238 330.1394,469.94434 325.63989,452.69678 z " style="fill:#d3ea9d;stroke:#000000" id="path2508"/>
 		</g>
   </svg>
-  <script src="../bower_components/yjs/y.js"></script>
+  <script src="../../y.js"></script>
+  <script src="../../../y-map/dist/y-map.js"></script>
+  <script src="../../../y-memory/y-memory.js"></script>
+  <script src="../../../y-websockets-client/y-websockets-client.js"></script>
   <script src="../bower_components/d3/d3.js"></script>
   <script src="./index.js"></script>
 </body>

examples/jigsaw/index.js

@@ -8,9 +8,9 @@ Y({
   },
   connector: {
     name: 'websockets-client',
-    room: 'Puzzle-example'
+    room: 'Puzzle-example',
+    url: 'http://localhost:1234'
   },
-  sourceDir: '/bower_components',
   share: {
     piece1: 'Map',
     piece2: 'Map',
@@ -19,11 +19,16 @@ Y({
   }
 }).then(function (y) {
   window.yJigsaw = y
-  var origin // mouse start position - translation of piece  
+  var origin // mouse start position - translation of piece
   var drag = d3.behavior.drag()
     .on('dragstart', function (params) {
       // get the translation of the element
-      var translation = d3.select(this).attr('transform').slice(10,-1).split(',').map(Number)
+      var translation = d3
+        .select(this)
+        .attr('transform')
+        .slice(10, -1)
+        .split(',')
+        .map(Number)
       // mouse coordinates
       var mouse = d3.mouse(this.parentNode)
       origin = {
@@ -31,11 +36,11 @@ Y({
         y: mouse[1] - translation[1]
       }
     })
-    .on("drag", function(){
+    .on('drag', function () {
       var mouse = d3.mouse(this.parentNode)
       var x = mouse[0] - origin.x // =^= mouse - mouse at dragstart + translation at dragstart
       var y = mouse[1] - origin.y
-      d3.select(this).attr("transform", "translate(" + x + "," + y + ")")
+      d3.select(this).attr('transform', 'translate(' + x + ',' + y + ')')
     })
     .on('dragend', function (piece, i) {
       // save the current translation of the puzzle piece
@@ -46,24 +51,24 @@ Y({
     })
 
   var data = [y.share.piece1, y.share.piece2, y.share.piece3, y.share.piece4]
-  var pieces = d3.select(document.querySelector("#puzzle-example")).selectAll("path").data(data)
+  var pieces = d3.select(document.querySelector('#puzzle-example')).selectAll('path').data(data)
 
   pieces
     .classed('draggable', true)
-    .attr("transform", function (piece) {
+    .attr('transform', function (piece) {
       var translation = piece.get('translation') || {x: 0, y: 0}
-      return "translate(" + translation.x + "," + translation.y + ")"
+      return 'translate(' + translation.x + ',' + translation.y + ')'
     }).call(drag)
 
-  data.forEach(function(piece){
+  data.forEach(function (piece) {
     piece.observe(function () {
       // whenever a property of a piece changes, update the translation of the pieces
       pieces
         .transition()
-        .attr("transform", function (piece) {
+        .attr('transform', function (piece) {
           var translation = piece.get('translation') || {x: 0, y: 0}
-          return "translate(" + translation.x + "," + translation.y + ")"
+          return 'translate(' + translation.x + ',' + translation.y + ')'
         })
     })
   })
-})
+})

examples/monaco/index.js

@@ -1,8 +1,8 @@
-/* global Y */
+/* global Y, monaco */
 
-require.config({ paths: { 'vs': '../node_modules/monaco-editor/min/vs' }})
-require(['vs/editor/editor.main'], function() {
+require.config({ paths: { 'vs': '../node_modules/monaco-editor/min/vs' } })
 
+require(['vs/editor/editor.main'], function () {
   // Initialize a shared object. This function call returns a promise!
   Y({
     db: {
@@ -28,4 +28,3 @@ require(['vs/editor/editor.main'], function() {
     y.share.monaco.bindMonaco(editor)
   })
 })
-

examples/package.json

@@ -6,5 +6,11 @@
   "license": "MIT",
   "dependencies": {
     "monaco-editor": "^0.8.3"
+  },
+  "devDependencies": {
+    "standard": "^10.0.2"
+  },
+  "standard": {
+    "ignore": ["bower_components"]
   }
 }

examples/quill/index.html

@@ -4,8 +4,8 @@
   <!-- quill does not include dist files! We are using the hosted version instead -->
   <!--link rel="stylesheet" href="../bower_components/quill/dist/quill.snow.css" /-->
   <link href="https://cdn.quilljs.com/1.0.4/quill.snow.css" rel="stylesheet">
-  <link href="//cdnjs.cloudflare.com/ajax/libs/KaTeX/0.5.1/katex.min.css" rel="stylesheet">
-  <link href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.2.0/styles/monokai-sublime.min.css" rel="stylesheet">
+  <link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.5.1/katex.min.css" rel="stylesheet">
+  <link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.2.0/styles/monokai-sublime.min.css" rel="stylesheet">
   <style>
     #quill-container {
       border: 1px solid gray;
@@ -19,13 +19,17 @@
     </div>
   </div>
 
-  <script src="//cdnjs.cloudflare.com/ajax/libs/KaTeX/0.5.1/katex.min.js" type="text/javascript"></script>
-  <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.2.0/highlight.min.js" type="text/javascript"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.5.1/katex.min.js" type="text/javascript"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.2.0/highlight.min.js" type="text/javascript"></script>
   <script src="https://cdn.quilljs.com/1.0.4/quill.js"></script>
   <!-- quill does not include dist files! We are using the hosted version instead (see above)
   <script src="../bower_components/quill/dist/quill.js"></script>
   -->
-  <script src="../bower_components/yjs/y.js"></script>
+  <script src="../../y.js"></script>
+  <script src="../../../y-array/y-array.js"></script>
+  <script src="../../../y-richtext/dist/y-richtext.js"></script>
+  <script src="../../../y-memory/y-memory.js"></script>
+  <script src="../../../y-websockets-client/y-websockets-client.js"></script>
   <script src="./index.js"></script>
 </body>
 </html>

examples/quill/index.js

@@ -8,7 +8,8 @@ Y({
   },
   connector: {
     name: 'websockets-client',
-    room: 'richtext-example-quill-1.0-test'
+    room: 'richtext-example-quill-1.0-test',
+    url: 'http://localhost:1234'
   },
   sourceDir: '/bower_components',
   share: {
@@ -29,11 +30,11 @@ Y({
         [{ script: 'sub' }, { script: 'super' }],
         ['link', 'image'],
         ['link', 'code-block'],
-        [{list: 'ordered' }]
+        [{ list: 'ordered' }]
       ]
     },
     theme: 'snow'
-  });
+  })
   // bind quill to richtext type
   y.share.richtext.bind(window.quill)
-})
+})

examples/serviceworker/index.js

@@ -1,13 +1,13 @@
 /* global Y, Quill */
 
 // register yjs service worker
-if('serviceWorker' in navigator){
+if ('serviceWorker' in navigator) {
   // Register service worker
   // it is important to copy yjs-sw-template to the root directory!
-  navigator.serviceWorker.register('./yjs-sw-template.js').then(function(reg){
-    console.log("Yjs service worker registration succeeded. Scope is " + reg.scope);
-  }).catch(function(err){
-    console.error("Yjs service worker registration failed with error " + err);
+  navigator.serviceWorker.register('./yjs-sw-template.js').then(function (reg) {
+    console.log('Yjs service worker registration succeeded. Scope is ' + reg.scope)
+  }).catch(function (err) {
+    console.error('Yjs service worker registration failed with error ' + err)
   })
 }
 
@@ -39,7 +39,7 @@ Y({
         [{ script: 'sub' }, { script: 'super' }],
         ['link', 'image'],
         ['link', 'code-block'],
-        [{list: 'ordered' }]
+        [{ list: 'ordered' }]
       ]
     },
     theme: 'snow'

examples/textarea/index.html

@@ -2,7 +2,11 @@
 <html>
 <body>
   <textarea style="width:80%;" rows=40 id="textfield" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
-  <script src="../bower_components/yjs/y.js"></script>
+  <script src="../../y.js"></script>
+  <script src="../../../y-array/y-array.js"></script>
+  <script src="../../../y-text/dist/y-text.js"></script>
+  <script src="../../../y-memory/y-memory.js"></script>
+  <script src="../../../y-websockets-client/y-websockets-client.js"></script>
   <script src="./index.js"></script>
 </body>
 </html>

examples/textarea/index.js

@@ -7,12 +7,13 @@ Y({
   },
   connector: {
     name: 'websockets-client',
-    room: 'Textarea-example'
-    // url: '127.0.0.1:1234'
+    room: 'Textarea-example',
+    url: 'http://127.0.0.1:1234'
   },
   sourceDir: '/bower_components',
   share: {
-    textarea: 'Text' // y.share.textarea is of type Y.Text
+    textarea: 'Text', // y.share.textarea is of type Y.Text
+    test: 'Array'
   }
 }).then(function (y) {
   window.yTextarea = y

package.json

@@ -1,21 +1,25 @@
 {
   "name": "yjs",
-  "version": "13.0.0-0",
+  "version": "13.0.0-9",
   "description": "A framework for real-time p2p shared editing on any data",
-  "main": "./src/y.js",
+  "main": "./y.node.js",
+  "browser": "./y.js",
+  "module": "./src/y.js",
   "scripts": {
+    "test": "npm run lint",
+    "debug": "concurrently 'rollup -wc rollup.test.js' 'cutest-serve y.test.js -o'",
     "lint": "standard",
-    "dist": "rollup -c rollup.dist.js",
-    "serve": "concurrently 'serve examples' 'rollup -wc rollup.dist.js -o examples/bower_components/yjs/y.js'"
+    "dist": "rollup -c rollup.browser.js; rollup -c rollup.node.js",
+    "postversion": "npm run dist",
+    "postpublish": "tag-dist-files --overwrite-existing-tag"
   },
-  "pre-commit": [
-    "lint",
-    "test"
+  "files": [
+    "y.*"
   ],
   "standard": {
     "ignore": [
-      "./y.js",
-      "./y.js.map"
+      "/y.js",
+      "/y.js.map"
     ]
   },
   "repository": {
@@ -40,12 +44,13 @@
   "homepage": "http://y-js.org",
   "devDependencies": {
     "babel-cli": "^6.24.1",
-    "babel-preset-latest": "^6.24.1",
     "babel-plugin-external-helpers": "^6.22.0",
     "babel-plugin-transform-regenerator": "^6.24.1",
     "babel-plugin-transform-runtime": "^6.23.0",
+    "babel-preset-latest": "^6.24.1",
     "chance": "^1.0.9",
     "concurrently": "^3.4.0",
+    "cutest": "^0.1.9",
     "rollup-plugin-babel": "^2.7.1",
     "rollup-plugin-commonjs": "^8.0.2",
     "rollup-plugin-inject": "^2.0.0",
@@ -54,9 +59,11 @@
     "rollup-plugin-uglify": "^1.0.2",
     "rollup-regenerator-runtime": "^6.23.1",
     "rollup-watch": "^3.2.2",
-    "standard": "^10.0.2"
+    "standard": "^10.0.2",
+    "tag-dist-files": "^0.1.6"
   },
   "dependencies": {
-    "debug": "^2.6.8"
+    "debug": "^2.6.8",
+    "utf-8": "^1.0.0"
   }
 }

rollup.node.js

@@ -0,0 +1,26 @@
+import nodeResolve from 'rollup-plugin-node-resolve'
+import commonjs from 'rollup-plugin-commonjs'
+var pkg = require('./package.json')
+
+export default {
+  entry: 'src/y.js',
+  moduleName: 'Y',
+  format: 'umd',
+  plugins: [
+    nodeResolve({
+      main: true,
+      module: true,
+      browser: true
+    }),
+    commonjs()
+  ],
+  dest: 'y.node.js',
+  sourceMap: true,
+  banner: `
+/**
+ * ${pkg.name} - ${pkg.description}
+ * @version v${pkg.version}
+ * @license ${pkg.license}
+ */
+`
+}

rollup.test.js

@@ -3,8 +3,8 @@ import commonjs from 'rollup-plugin-commonjs'
 import multiEntry from 'rollup-plugin-multi-entry'
 
 export default {
-  entry: 'tests/*.js',
-  moduleName: 'y-array-tests',
+  entry: 'test/*',
+  moduleName: 'y-tests',
   format: 'umd',
   plugins: [
     nodeResolve({
@@ -15,6 +15,6 @@ export default {
     commonjs(),
     multiEntry()
   ],
-  dest: 'y-array.test.js',
+  dest: 'y.test.js',
   sourceMap: true
 }

src/Connector.js

@@ -1,40 +1,18 @@
-/* @flow */
-'use strict'
-
-function canRead (auth) { return auth === 'read' || auth === 'write' }
-function canWrite (auth) { return auth === 'write' }
+import { BinaryEncoder, BinaryDecoder } from './Encoding.js'
+import { sendSyncStep1, computeMessageSyncStep1, computeMessageSyncStep2, computeMessageUpdate } from './MessageHandler.js'
 
 export default function extendConnector (Y/* :any */) {
   class AbstractConnector {
-    /* ::
-    y: YConfig;
-    role: SyncRole;
-    connections: Object;
-    isSynced: boolean;
-    userEventListeners: Array<Function>;
-    whenSyncedListeners: Array<Function>;
-    currentSyncTarget: ?UserId;
-    syncingClients: Array<UserId>;
-    forwardToSyncingClients: boolean;
-    debug: boolean;
-    syncStep2: Promise;
-    userId: UserId;
-    send: Function;
-    broadcast: Function;
-    broadcastOpBuffer: Array<Operation>;
-    protocolVersion: number;
-    */
     /*
       opts contains the following information:
        role : String Role of this client ("master" or "slave")
-       userId : String Uniquely defines the user.
-       debug: Boolean Whether to print debug messages (optional)
     */
     constructor (y, opts) {
       this.y = y
       if (opts == null) {
         opts = {}
       }
+      this.opts = opts
       // Prefer to receive untransformed operations. This does only work if
       // this client receives operations from only one other client.
       // In particular, this does not work with y-webrtc.
@@ -51,29 +29,18 @@ export default function extendConnector (Y/* :any */) {
       this.logMessage = Y.debug('y:connector-message')
       this.y.db.forwardAppliedOperations = opts.forwardAppliedOperations || false
       this.role = opts.role
-      this.connections = {}
+      this.connections = new Map()
       this.isSynced = false
       this.userEventListeners = []
       this.whenSyncedListeners = []
       this.currentSyncTarget = null
-      this.syncingClients = []
-      this.forwardToSyncingClients = opts.forwardToSyncingClients !== false
       this.debug = opts.debug === true
       this.broadcastOpBuffer = []
       this.protocolVersion = 11
       this.authInfo = opts.auth || null
       this.checkAuth = opts.checkAuth || function () { return Promise.resolve('write') } // default is everyone has write access
-      if (opts.generateUserId === true) {
-        this.setUserId(Y.utils.generateGuid())
-      }
-    }
-    resetAuth (auth) {
-      if (this.authInfo !== auth) {
-        this.authInfo = auth
-        this.broadcast({
-          type: 'auth',
-          auth: this.authInfo
-        })
+      if (opts.generateUserId !== false) {
+        this.setUserId(Y.utils.generateUserId())
       }
     }
     reconnect () {
@@ -82,25 +49,27 @@ export default function extendConnector (Y/* :any */) {
     }
     disconnect () {
       this.log('discronnecting..')
-      this.connections = {}
+      this.connections = new Map()
       this.isSynced = false
       this.currentSyncTarget = null
-      this.syncingClients = []
       this.whenSyncedListeners = []
       this.y.db.stopGarbageCollector()
       return this.y.db.whenTransactionsFinished()
     }
     repair () {
       this.log('Repairing the state of Yjs. This can happen if messages get lost, and Yjs detects that something is wrong. If this happens often, please report an issue here: https://github.com/y-js/yjs/issues')
-      for (var name in this.connections) {
-        this.connections[name].isSynced = false
-      }
+      this.connections.forEach(user => { user.isSynced = false })
       this.isSynced = false
       this.currentSyncTarget = null
       this.findNextSyncTarget()
     }
     setUserId (userId) {
       if (this.userId == null) {
+        if (!Number.isInteger(userId)) {
+          let err = new Error('UserId must be an integer!')
+          this.y.emit('error', err)
+          throw err
+        }
         this.log('Set userId to "%s"', userId)
         this.userId = userId
         return this.y.db.setUserId(userId)
@@ -115,16 +84,13 @@ export default function extendConnector (Y/* :any */) {
       this.userEventListeners = this.userEventListeners.filter(g => f !== g)
     }
     userLeft (user) {
-      if (this.connections[user] != null) {
-        this.log('User left: %s', user)
-        delete this.connections[user]
+      if (this.connections.has(user)) {
+        this.log('%s: User left %s', this.userId, user)
+        this.connections.delete(user)
         if (user === this.currentSyncTarget) {
           this.currentSyncTarget = null
           this.findNextSyncTarget()
         }
-        this.syncingClients = this.syncingClients.filter(function (cli) {
-          return cli !== user
-        })
         for (var f of this.userEventListeners) {
           f({
             action: 'userLeft',
@@ -137,17 +103,21 @@ export default function extendConnector (Y/* :any */) {
       if (role == null) {
         throw new Error('You must specify the role of the joined user!')
       }
-      if (this.connections[user] != null) {
+      if (this.connections.has(user)) {
         throw new Error('This user already joined!')
       }
-      this.log('User joined: %s', user)
-      this.connections[user] = {
+      this.log('%s: User joined %s', this.userId, user)
+      this.connections.set(user, {
+        uid: user,
         isSynced: false,
-        role: role
-      }
+        role: role,
+        processAfterAuth: [],
+        auth: null,
+        receivedSyncStep2: false
+      })
       let defer = {}
       defer.promise = new Promise(function (resolve) { defer.resolve = resolve })
-      this.connections[user].syncStep2 = defer
+      this.connections.get(user).syncStep2 = defer
       for (var f of this.userEventListeners) {
         f({
           action: 'userJoined',
@@ -169,13 +139,13 @@ export default function extendConnector (Y/* :any */) {
       }
     }
     findNextSyncTarget () {
-      if (this.currentSyncTarget != null) {
-        return // "The current sync has not finished!"
+      if (this.currentSyncTarget != null || this.role === 'slave') {
+        return // "The current sync has not finished or this is controlled by a master!"
       }
 
       var syncUser = null
-      for (var uid in this.connections) {
-        if (!this.connections[uid].isSynced) {
+      for (var [uid, user] of this.connections) {
+        if (!user.isSynced) {
           syncUser = uid
           break
         }
@@ -183,21 +153,7 @@ export default function extendConnector (Y/* :any */) {
       var conn = this
       if (syncUser != null) {
         this.currentSyncTarget = syncUser
-        this.y.db.requestTransaction(function * () {
-          var stateSet = yield * this.getStateSet()
-          var deleteSet = yield * this.getDeleteSet()
-          var answer = {
-            type: 'sync step 1',
-            stateSet: stateSet,
-            // deleteSet: deleteSet,
-            protocolVersion: conn.protocolVersion,
-            auth: conn.authInfo
-          }
-          if (conn.preferUntransformed && Object.keys(stateSet).length === 0) {
-            answer.preferUntransformed = true
-          }
-          conn.send(syncUser, answer)
-        })
+        sendSyncStep1(this, syncUser)
       } else {
         if (!conn.isSynced) {
           this.y.db.requestTransaction(function * () {
@@ -216,13 +172,19 @@ export default function extendConnector (Y/* :any */) {
         }
       }
     }
-    send (uid, message) {
-      this.log('Send \'%s\' to %s', message.type, uid)
-      this.logMessage('Message: %j', message)
+    send (uid, buffer) {
+      if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
+        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages')
+      }
+      this.log('%s: Send \'%y\' to %s', this.userId, buffer, uid)
+      this.logMessage('Message: %Y', buffer)
     }
-    broadcast (message) {
-      this.log('Broadcast \'%s\'', message.type)
-      this.logMessage('Message: %j', message)
+    broadcast (buffer) {
+      if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
+        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages')
+      }
+      this.log('%s: Broadcast \'%y\'', this.userId, buffer)
+      this.logMessage('Message: %Y', buffer)
     }
     /*
       Buffer operations, and broadcast them when ready.
@@ -234,11 +196,18 @@ export default function extendConnector (Y/* :any */) {
       var self = this
       function broadcastOperations () {
         if (self.broadcastOpBuffer.length > 0) {
-          self.broadcast({
-            type: 'update',
-            ops: self.broadcastOpBuffer
-          })
+          let encoder = new BinaryEncoder()
+          encoder.writeVarString(self.opts.room)
+          encoder.writeVarString('update')
+          let ops = self.broadcastOpBuffer
           self.broadcastOpBuffer = []
+          let length = ops.length
+          encoder.writeUint32(length)
+          for (var i = 0; i < length; i++) {
+            let op = ops[i]
+            Y.Struct[op.struct].binaryEncode(encoder, op)
+          }
+          self.broadcast(encoder.createBuffer())
         }
       }
       if (this.broadcastOpBuffer.length === 0) {
@@ -251,145 +220,67 @@ export default function extendConnector (Y/* :any */) {
     /*
       You received a raw message, and you know that it is intended for Yjs. Then call this function.
     */
-    receiveMessage (sender/* :UserId */, message/* :Message */) {
+    receiveMessage (sender, buffer) {
+      if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
+        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array!')
+      }
       if (sender === this.userId) {
-        return Promise.resolve()
+        return
       }
-      this.log('Receive \'%s\' from %s', message.type, sender)
-      this.logMessage('Message: %j', message)
-      if (message.protocolVersion != null && message.protocolVersion !== this.protocolVersion) {
-        this.log(
-          `You tried to sync with a yjs instance that has a different protocol version
-          (You: ${this.protocolVersion}, Client: ${message.protocolVersion}).
-          The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)!
-          `)
-        this.send(sender, {
-          type: 'sync stop',
-          protocolVersion: this.protocolVersion
-        })
-        return Promise.reject(new Error('Incompatible protocol version'))
-      }
-      if (message.auth != null && this.connections[sender] != null) {
-        // authenticate using auth in message
-        var auth = this.checkAuth(message.auth, this.y)
-        this.connections[sender].auth = auth
-        auth.then(auth => {
-          for (var f of this.userEventListeners) {
-            f({
-              action: 'userAuthenticated',
-              user: sender,
-              auth: auth
-            })
-          }
-        })
-      } else if (this.connections[sender] != null && this.connections[sender].auth == null) {
-        // authenticate without otherwise
-        this.connections[sender].auth = this.checkAuth(null, this.y)
-      }
-      if (this.connections[sender] != null && this.connections[sender].auth != null) {
-        return this.connections[sender].auth.then((auth) => {
-          if (message.type === 'sync step 1' && canRead(auth)) {
-            let conn = this
-            let m = message
-            let wait // wait for sync step 2 to complete
-            if (this.role === 'slave') {
-              wait = Promise.all(Object.keys(this.connections)
-                .map(uid => this.connections[uid])
-                .filter(conn => conn.role === 'master')
-                .map(conn => conn.syncStep2.promise)
-              )
-            } else {
-              wait = Promise.resolve()
-            }
-            wait.then(() => {
-              this.y.db.requestTransaction(function * () {
-                var currentStateSet = yield * this.getStateSet()
-                // TODO: remove
-                // if (canWrite(auth)) {
-                //  yield * this.applyDeleteSet(m.deleteSet)
-                // }
+      let decoder = new BinaryDecoder(buffer)
+      let encoder = new BinaryEncoder()
+      let roomname = decoder.readVarString() // read room name
+      encoder.writeVarString(roomname)
+      let messageType = decoder.readVarString()
+      let senderConn = this.connections.get(sender)
 
-                var ds = yield * this.getDeleteSet()
-                var answer = {
-                  type: 'sync step 2',
-                  stateSet: currentStateSet,
-                  deleteSet: ds,
-                  protocolVersion: this.protocolVersion,
-                  auth: this.authInfo
-                }
-                if (message.preferUntransformed === true && Object.keys(m.stateSet).length === 0) {
-                  answer.osUntransformed = yield * this.getOperationsUntransformed()
-                } else {
-                  answer.os = yield * this.getOperations(m.stateSet)
-                }
-                conn.send(sender, answer)
-                if (this.forwardToSyncingClients) {
-                  conn.syncingClients.push(sender)
-                  setTimeout(function () {
-                    conn.syncingClients = conn.syncingClients.filter(function (cli) {
-                      return cli !== sender
-                    })
-                    conn.send(sender, {
-                      type: 'sync done'
-                    })
-                  }, 5000) // TODO: conn.syncingClientDuration)
-                } else {
-                  conn.send(sender, {
-                    type: 'sync done'
-                  })
-                }
-              })
+      this.log('%s: Receive \'%s\' from %s', this.userId, messageType, sender)
+      this.logMessage('Message: %Y', buffer)
+
+      if (senderConn == null) {
+        throw new Error('Received message from unknown peer!')
+      }
+
+      if (messageType === 'sync step 1' || messageType === 'sync step 2') {
+        let auth = decoder.readVarUint()
+        if (senderConn.auth == null) {
+          senderConn.processAfterAuth.push([sender, buffer])
+
+          // check auth
+          return this.checkAuth(auth, this.y, sender).then(authPermissions => {
+            senderConn.auth = authPermissions
+            this.y.emit('userAuthenticated', {
+              user: senderConn.uid,
+              auth: authPermissions
             })
-          } else if (message.type === 'sync step 2' && canWrite(auth)) {
-            var db = this.y.db
-            let defer = this.connections[sender].syncStep2
-            let m = message
-            // apply operations first
-            db.requestTransaction(function * () {
-              yield * this.applyDeleteSet(m.deleteSet)
-              if (m.osUntransformed != null) {
-                yield * this.applyOperationsUntransformed(m.osUntransformed, m.stateSet)
-              } else {
-                this.store.apply(m.os)
-              }
-              defer.resolve()
-            })
-            /*/ then apply ds
-            db.whenTransactionsFinished().then(() => {
-              db.requestTransaction(function * () {
-                yield * this.applyDeleteSet(m.deleteSet)
-              })
-              defer.resolve()
-            })*/
-            return defer.promise
-          } else if (message.type === 'sync done') {
-            var self = this
-            this.connections[sender].syncStep2.promise.then(function () {
-              self._setSyncedWith(sender)
-            })
-          } else if (message.type === 'update' && canWrite(auth)) {
-            if (this.forwardToSyncingClients) {
-              for (var client of this.syncingClients) {
-                this.send(client, message)
-              }
+            return senderConn.syncStep2.promise
+          }).then(() => {
+            if (senderConn.processAfterAuth == null) {
+              return Promise.resolve()
             }
-            if (this.y.db.forwardAppliedOperations) {
-              var delops = message.ops.filter(function (o) {
-                return o.struct === 'Delete'
-              })
-              if (delops.length > 0) {
-                this.broadcastOps(delops)
-              }
-            }
-            this.y.db.apply(message.ops)
-          }
-        })
+            let messages = senderConn.processAfterAuth
+            senderConn.processAfterAuth = null
+            return Promise.all(messages.map(m =>
+              this.receiveMessage(m[0], m[1])
+            ))
+          })
+        }
+      }
+
+      if (messageType === 'sync step 1' && (senderConn.auth === 'write' || senderConn.auth === 'read')) {
+        // cannot wait for sync step 1 to finish, because we may wait for sync step 2 in sync step 1 (->lock)
+        computeMessageSyncStep1(decoder, encoder, this, senderConn, sender)
+        return this.y.db.whenTransactionsFinished()
+      } else if (messageType === 'sync step 2' && senderConn.auth === 'write') {
+        return computeMessageSyncStep2(decoder, encoder, this, senderConn, sender)
+      } else if (messageType === 'update' && senderConn.auth === 'write') {
+        return computeMessageUpdate(decoder, encoder, this, senderConn, sender)
       } else {
-        return Promise.reject(new Error('Unable to deliver message'))
+        Promise.reject(new Error('Unable to receive message'))
       }
     }
     _setSyncedWith (user) {
-      var conn = this.connections[user]
+      var conn = this.connections.get(user)
       if (conn != null) {
         conn.isSynced = true
       }

src/Connectors/Test.js

@@ -1,178 +0,0 @@
-/* global getRandom, async */
-'use strict'
-
-module.exports = function (Y) {
-  var globalRoom = {
-    users: {},
-    buffers: {},
-    removeUser: function (user) {
-      for (var i in this.users) {
-        this.users[i].userLeft(user)
-      }
-      delete this.users[user]
-      delete this.buffers[user]
-    },
-    addUser: function (connector) {
-      this.users[connector.userId] = connector
-      this.buffers[connector.userId] = {}
-      for (var uname in this.users) {
-        if (uname !== connector.userId) {
-          var u = this.users[uname]
-          u.userJoined(connector.userId, 'master')
-          connector.userJoined(u.userId, 'master')
-        }
-      }
-    },
-    whenTransactionsFinished: function () {
-      var self = this
-      return new Promise(function (resolve, reject) {
-        // The connector first has to send the messages to the db.
-        // Wait for the checkAuth-function to resolve
-        // The test lib only has a simple checkAuth function: `() => Promise.resolve()`
-        // Just add a function to the event-queue, in order to wait for the event.
-        // TODO: this may be buggy in test applications (but it isn't be for real-life apps)
-        setTimeout(function () {
-          var ps = []
-          for (var name in self.users) {
-            ps.push(self.users[name].y.db.whenTransactionsFinished())
-          }
-          Promise.all(ps).then(resolve, reject)
-        }, 10)
-      })
-    },
-    flushOne: function flushOne () {
-      var bufs = []
-      for (var receiver in globalRoom.buffers) {
-        let buff = globalRoom.buffers[receiver]
-        var push = false
-        for (let sender in buff) {
-          if (buff[sender].length > 0) {
-            push = true
-            break
-          }
-        }
-        if (push) {
-          bufs.push(receiver)
-        }
-      }
-      if (bufs.length > 0) {
-        var userId = getRandom(bufs)
-        let buff = globalRoom.buffers[userId]
-        let sender = getRandom(Object.keys(buff))
-        var m = buff[sender].shift()
-        if (buff[sender].length === 0) {
-          delete buff[sender]
-        }
-        var user = globalRoom.users[userId]
-        return user.receiveMessage(m[0], m[1]).then(function () {
-          return user.y.db.whenTransactionsFinished()
-        }, function () {})
-      } else {
-        return false
-      }
-    },
-    flushAll: function () {
-      return new Promise(function (resolve) {
-        // flushes may result in more created operations,
-        // flush until there is nothing more to flush
-        function nextFlush () {
-          var c = globalRoom.flushOne()
-          if (c) {
-            while (c) {
-              c = globalRoom.flushOne()
-            }
-            globalRoom.whenTransactionsFinished().then(nextFlush)
-          } else {
-            c = globalRoom.flushOne()
-            if (c) {
-              c.then(function () {
-                globalRoom.whenTransactionsFinished().then(nextFlush)
-              })
-            } else {
-              resolve()
-            }
-          }
-        }
-        globalRoom.whenTransactionsFinished().then(nextFlush)
-      })
-    }
-  }
-  Y.utils.globalRoom = globalRoom
-
-  var userIdCounter = 0
-
-  class Test extends Y.AbstractConnector {
-    constructor (y, options) {
-      if (options === undefined) {
-        throw new Error('Options must not be undefined!')
-      }
-      options.role = 'master'
-      options.forwardToSyncingClients = false
-      super(y, options)
-      this.setUserId((userIdCounter++) + '').then(() => {
-        globalRoom.addUser(this)
-      })
-      this.globalRoom = globalRoom
-      this.syncingClientDuration = 0
-    }
-    receiveMessage (sender, m) {
-      return super.receiveMessage(sender, JSON.parse(JSON.stringify(m)))
-    }
-    send (userId, message) {
-      var buffer = globalRoom.buffers[userId]
-      if (buffer != null) {
-        if (buffer[this.userId] == null) {
-          buffer[this.userId] = []
-        }
-        buffer[this.userId].push(JSON.parse(JSON.stringify([this.userId, message])))
-      }
-    }
-    broadcast (message) {
-      for (var key in globalRoom.buffers) {
-        var buff = globalRoom.buffers[key]
-        if (buff[this.userId] == null) {
-          buff[this.userId] = []
-        }
-        buff[this.userId].push(JSON.parse(JSON.stringify([this.userId, message])))
-      }
-    }
-    isDisconnected () {
-      return globalRoom.users[this.userId] == null
-    }
-    reconnect () {
-      if (this.isDisconnected()) {
-        globalRoom.addUser(this)
-        super.reconnect()
-      }
-      return Y.utils.globalRoom.flushAll()
-    }
-    disconnect () {
-      var waitForMe = Promise.resolve()
-      if (!this.isDisconnected()) {
-        globalRoom.removeUser(this.userId)
-        waitForMe = super.disconnect()
-      }
-      var self = this
-      return waitForMe.then(function () {
-        return self.y.db.whenTransactionsFinished()
-      })
-    }
-    flush () {
-      var self = this
-      return async(function * () {
-        var buff = globalRoom.buffers[self.userId]
-        while (Object.keys(buff).length > 0) {
-          var sender = getRandom(Object.keys(buff))
-          var m = buff[sender].shift()
-          if (buff[sender].length === 0) {
-            delete buff[sender]
-          }
-          yield this.receiveMessage(m[0], m[1])
-        }
-        yield self.whenTransactionsFinished()
-      })
-    }
-  }
-
-  Y.Test = Test
-}

src/Database.js

@@ -81,10 +81,6 @@ export default function extendDatabase (Y /* :any */) {
       function garbageCollect () {
         return os.whenTransactionsFinished().then(function () {
           if (os.gcTimeout > 0 && (os.gc1.length > 0 || os.gc2.length > 0)) {
-            // debug
-            if (os.y.connector.isSynced === false) {
-              debugger
-            }
             if (!os.y.connector.isSynced) {
               console.warn('gc should be empty when not synced!')
             }
@@ -124,7 +120,7 @@ export default function extendDatabase (Y /* :any */) {
     startGarbageCollector () {
       this.gc = this.dbOpts.gc
       if (this.gc) {
-        this.gcTimeout = !this.dbOpts.gcTimeout ? 50000 : this.dbOpts.gcTimeout
+        this.gcTimeout = !this.dbOpts.gcTimeout ? 100000 : this.dbOpts.gcTimeout
       } else {
         this.gcTimeout = -1
       }
@@ -310,10 +306,12 @@ export default function extendDatabase (Y /* :any */) {
       * check if it is an expected op (otherwise wait for it)
       * check if was deleted, apply a delete operation after op was applied
     */
-    apply (ops) {
+    applyOperations (decoder) {
       this.opsReceivedTimestamp = new Date()
-      for (var i = 0; i < ops.length; i++) {
-        var o = ops[i]
+      let length = decoder.readUint32()
+
+      for (var i = 0; i < length; i++) {
+        let o = Y.Struct.binaryDecodeOperation(decoder)
         if (o.id == null || o.id[0] !== this.y.connector.userId) {
           var required = Y.Struct[o.struct].requiredOps(o)
           if (o.requires != null) {
@@ -405,11 +403,11 @@ export default function extendDatabase (Y /* :any */) {
     whenOperationsExist: any;
     */
     * tryExecute (op) {
-      this.store.addToDebug('yield* this.store.tryExecute.call(this, ', JSON.stringify(op), ')')
+      this.store.addToDebug('yield * this.store.tryExecute.call(this, ', JSON.stringify(op), ')')
       if (op.struct === 'Delete') {
         yield * Y.Struct.Delete.execute.call(this, op)
         // this is now called in Transaction.deleteOperation!
-        // yield* this.store.operationAdded(this, op)
+        // yield * this.store.operationAdded(this, op)
       } else {
         // check if this op was defined
         var defined = yield * this.getInsertion(op.id)
@@ -594,7 +592,7 @@ export default function extendDatabase (Y /* :any */) {
       op.type = typedefinition[0].name
 
       this.requestTransaction(function * () {
-        if (op.id[0] === '_') {
+        if (op.id[0] === 0xFFFFFF) {
           yield * this.setOperation(op)
         } else {
           yield * this.applyCreatedOperations([op])

src/Encoding.js

@@ -0,0 +1,133 @@
+import utf8 from 'utf-8'
+
+const bits7 = 0b1111111
+const bits8 = 0b11111111
+
+export class BinaryEncoder {
+  constructor () {
+    this.data = []
+  }
+  get pos () {
+    return this.data.length
+  }
+  createBuffer () {
+    return Uint8Array.from(this.data).buffer
+  }
+  writeUint8 (num) {
+    this.data.push(num & bits8)
+  }
+  setUint8 (pos, num) {
+    this.data[pos] = num & bits8
+  }
+  writeUint16 (num) {
+    this.data.push(num & bits8, (num >>> 8) & bits8)
+  }
+  setUint16 (pos, num) {
+    this.data[pos] = num & bits8
+    this.data[pos + 1] = (num >>> 8) & bits8
+  }
+  writeUint32 (num) {
+    for (let i = 0; i < 4; i++) {
+      this.data.push(num & bits8)
+      num >>>= 8
+    }
+  }
+  setUint32 (pos, num) {
+    for (let i = 0; i < 4; i++) {
+      this.data[pos + i] = num & bits8
+      num >>>= 8
+    }
+  }
+  writeVarUint (num) {
+    while (num >= 0b10000000) {
+      this.data.push(0b10000000 | (bits7 & num))
+      num >>>= 7
+    }
+    this.data.push(bits7 & num)
+  }
+  writeVarString (str) {
+    let bytes = utf8.setBytesFromString(str)
+    let len = bytes.length
+    this.writeVarUint(len)
+    for (let i = 0; i < len; i++) {
+      this.data.push(bytes[i])
+    }
+  }
+  writeOpID (id) {
+    let user = id[0]
+    this.writeVarUint(user)
+    if (user !== 0xFFFFFF) {
+      this.writeVarUint(id[1])
+    } else {
+      this.writeVarString(id[1])
+    }
+  }
+}
+
+export class BinaryDecoder {
+  constructor (buffer) {
+    if (buffer instanceof ArrayBuffer) {
+      this.uint8arr = new Uint8Array(buffer)
+    } else if (buffer instanceof Uint8Array) {
+      this.uint8arr = buffer
+    } else {
+      throw new Error('Expected an ArrayBuffer or Uint8Array!')
+    }
+    this.pos = 0
+  }
+  skip8 () {
+    this.pos++
+  }
+  readUint8 () {
+    return this.uint8arr[this.pos++]
+  }
+  readUint32 () {
+    let uint =
+      this.uint8arr[this.pos] +
+      (this.uint8arr[this.pos + 1] << 8) +
+      (this.uint8arr[this.pos + 2] << 16) +
+      (this.uint8arr[this.pos + 3] << 24)
+    this.pos += 4
+    return uint
+  }
+  peekUint8 () {
+    return this.uint8arr[this.pos]
+  }
+  readVarUint () {
+    let num = 0
+    let len = 0
+    while (true) {
+      let r = this.uint8arr[this.pos++]
+      num = num | ((r & bits7) << len)
+      len += 7
+      if (r < 1 << 7) {
+        return num >>> 0 // return unsigned number!
+      }
+      if (len > 35) {
+        throw new Error('Integer out of range!')
+      }
+    }
+  }
+  readVarString () {
+    let len = this.readVarUint()
+    let bytes = new Array(len)
+    for (let i = 0; i < len; i++) {
+      bytes[i] = this.uint8arr[this.pos++]
+    }
+    return utf8.getStringFromBytes(bytes)
+  }
+  peekVarString () {
+    let pos = this.pos
+    let s = this.readVarString()
+    this.pos = pos
+    return s
+  }
+  readOpID () {
+    let user = this.readVarUint()
+    if (user !== 0xFFFFFF) {
+      return [user, this.readVarUint()]
+    } else {
+      return [user, this.readVarString()]
+    }
+  }
+}

src/MessageHandler.js

@@ -0,0 +1,185 @@
+
+import Y from './y.js'
+import { BinaryDecoder, BinaryEncoder } from './Encoding.js'
+
+export function formatYjsMessage (buffer) {
+  let decoder = new BinaryDecoder(buffer)
+  decoder.readVarString() // read roomname
+  let type = decoder.readVarString()
+  let strBuilder = []
+  strBuilder.push('\n === ' + type + ' ===\n')
+  if (type === 'update') {
+    logMessageUpdate(decoder, strBuilder)
+  } else if (type === 'sync step 1') {
+    logMessageSyncStep1(decoder, strBuilder)
+  } else if (type === 'sync step 2') {
+    logMessageSyncStep2(decoder, strBuilder)
+  } else {
+    strBuilder.push('-- Unknown message type - probably an encoding issue!!!')
+  }
+  return strBuilder.join('')
+}
+
+export function formatYjsMessageType (buffer) {
+  let decoder = new BinaryDecoder(buffer)
+  decoder.readVarString() // roomname
+  return decoder.readVarString()
+}
+
+export function logMessageUpdate (decoder, strBuilder) {
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    strBuilder.push(JSON.stringify(Y.Struct.binaryDecodeOperation(decoder)) + '\n')
+  }
+}
+
+export function computeMessageUpdate (decoder, encoder, conn) {
+  if (conn.y.db.forwardAppliedOperations) {
+    let messagePosition = decoder.pos
+    let len = decoder.readUint32()
+    let delops = []
+    for (let i = 0; i < len; i++) {
+      let op = Y.Struct.binaryDecodeOperation(decoder)
+      if (op.struct === 'Delete') {
+        delops.push(op)
+      }
+    }
+    if (delops.length > 0) {
+      conn.broadcastOps(delops)
+    }
+    decoder.pos = messagePosition
+  }
+  conn.y.db.applyOperations(decoder)
+}
+
+export function sendSyncStep1 (conn, syncUser) {
+  conn.y.db.requestTransaction(function * () {
+    let encoder = new BinaryEncoder()
+    encoder.writeVarString(conn.opts.room || '')
+    encoder.writeVarString('sync step 1')
+    encoder.writeVarString(conn.authInfo || '')
+    encoder.writeVarUint(conn.protocolVersion)
+    let preferUntransformed = conn.preferUntransformed && this.os.length === 0 // TODO: length may not be defined
+    encoder.writeUint8(preferUntransformed ? 1 : 0)
+    yield * this.writeStateSet(encoder)
+    conn.send(syncUser, encoder.createBuffer())
+  })
+}
+
+export function logMessageSyncStep1 (decoder, strBuilder) {
+  let auth = decoder.readVarString()
+  let protocolVersion = decoder.readVarUint()
+  let preferUntransformed = decoder.readUint8() === 1
+  strBuilder.push(`
+  - auth: "${auth}"
+  - protocolVersion: ${protocolVersion}
+  - preferUntransformed: ${preferUntransformed}
+`)
+  logSS(decoder, strBuilder)
+}
+
+export function computeMessageSyncStep1 (decoder, encoder, conn, senderConn, sender) {
+  let protocolVersion = decoder.readVarUint()
+  let preferUntransformed = decoder.readUint8() === 1
+
+  // check protocol version
+  if (protocolVersion !== conn.protocolVersion) {
+    console.warn(
+      `You tried to sync with a yjs instance that has a different protocol version
+      (You: ${protocolVersion}, Client: ${protocolVersion}).
+      The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)!
+      `)
+    conn.y.destroy()
+  }
+
+  // send sync step 2
+  conn.y.db.requestTransaction(function * () {
+    encoder.writeVarString('sync step 2')
+    encoder.writeVarString(conn.authInfo || '')
+
+    if (preferUntransformed) {
+      encoder.writeUint8(1)
+      yield * this.writeOperationsUntransformed(encoder)
+    } else {
+      encoder.writeUint8(0)
+      yield * this.writeOperations(encoder, decoder)
+    }
+
+    yield * this.writeDeleteSet(encoder)
+    conn.send(senderConn.uid, encoder.createBuffer())
+    senderConn.receivedSyncStep2 = true
+  })
+  if (conn.role === 'slave') {
+    sendSyncStep1(conn, sender)
+  }
+  return conn.y.db.whenTransactionsFinished()
+}
+
+export function logSS (decoder, strBuilder) {
+  strBuilder.push('  == SS: \n')
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let user = decoder.readVarUint()
+    let clock = decoder.readVarUint()
+    strBuilder.push(`     ${user}: ${clock}\n`)
+  }
+}
+
+export function logOS (decoder, strBuilder) {
+  strBuilder.push('  == OS: \n')
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let op = Y.Struct.binaryDecodeOperation(decoder)
+    strBuilder.push(JSON.stringify(op) + '\n')
+  }
+}
+
+export function logDS (decoder, strBuilder) {
+  strBuilder.push('  == DS: \n')
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let user = decoder.readVarUint()
+    strBuilder.push(`    User: ${user}: `)
+    let len2 = decoder.readVarUint()
+    for (let j = 0; j < len2; j++) {
+      let from = decoder.readVarUint()
+      let to = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      strBuilder.push(`[${from}, ${to}, ${gc}]`)
+    }
+  }
+}
+
+export function logMessageSyncStep2 (decoder, strBuilder) {
+  strBuilder.push('     - auth: ' + decoder.readVarString() + '\n')
+  let osTransformed = decoder.readUint8() === 1
+  strBuilder.push('     - osUntransformed: ' + osTransformed + '\n')
+  logOS(decoder, strBuilder)
+  if (osTransformed) {
+    logSS(decoder, strBuilder)
+  }
+  logDS(decoder, strBuilder)
+}
+
+export function computeMessageSyncStep2 (decoder, encoder, conn, senderConn, sender) {
+  var db = conn.y.db
+  let defer = senderConn.syncStep2
+
+  // apply operations first
+  db.requestTransaction(function * () {
+    let osUntransformed = decoder.readUint8()
+    if (osUntransformed === 1) {
+      yield * this.applyOperationsUntransformed(decoder)
+    } else {
+      this.store.applyOperations(decoder)
+    }
+  })
+  // then apply ds
+  db.requestTransaction(function * () {
+    yield * this.applyDeleteSet(decoder)
+  })
+  return db.whenTransactionsFinished().then(() => {
+    conn._setSyncedWith(sender)
+    defer.resolve()
+  })
+}

src/SpecHelper.js

@@ -1,404 +0,0 @@
-/* eslint-env browser, jasmine */
-
-/*
-  This is just a compilation of functions that help to test this library!
-*/
-
-// When testing, you store everything on the global object. We call it g
-
-var Y = require('./y.js')
-require('../../y-memory/src/Memory.js')(Y)
-require('../../y-array/src/Array.js')(Y)
-require('../../y-map/src/Map.js')(Y)
-require('../../y-indexeddb/src/IndexedDB.js')(Y)
-
-module.exports = Y
-
-var g
-if (typeof global !== 'undefined') {
-  g = global
-} else if (typeof window !== 'undefined') {
-  g = window
-} else {
-  throw new Error('No global object?')
-}
-g.g = g
-
-// Helper methods for the random number generator
-Math.seedrandom = require('seedrandom')
-
-g.generateRandomSeed = function generateRandomSeed () {
-  var seed
-  if (typeof window !== 'undefined' && window.location.hash.length > 1) {
-    seed = window.location.hash.slice(1) // first character is the hash!
-    console.warn('Using random seed that was specified in the url!')
-  } else {
-    seed = JSON.stringify(Math.random())
-  }
-  console.info('Using random seed: ' + seed)
-  g.setRandomSeed(seed)
-}
-
-g.setRandomSeed = function setRandomSeed (seed) {
-  Math.seedrandom.currentSeed = seed
-  Math.seedrandom(Math.seedrandom.currentSeed, { global: true })
-}
-
-g.generateRandomSeed()
-
-g.YConcurrencyTestingMode = true
-
-jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000
-
-g.describeManyTimes = function describeManyTimes (times, name, f) {
-  for (var i = 0; i < times; i++) {
-    describe(name, f)
-  }
-}
-
-/*
-  Wait for a specified amount of time (in ms). defaults to 5ms
-*/
-function wait (t) {
-  if (t == null) {
-    t = 0
-  }
-  return new Promise(function (resolve) {
-    setTimeout(function () {
-      resolve()
-    }, t)
-  })
-}
-g.wait = wait
-
-g.databases = ['memory']
-if (typeof window !== 'undefined') {
-  g.databases.push('indexeddb')
-} else {
-  g.databases.push('leveldb')
-}
-/*
-  returns a random element of o.
-  works on Object, and Array
-*/
-function getRandom (o) {
-  if (o instanceof Array) {
-    return o[Math.floor(Math.random() * o.length)]
-  } else if (o.constructor === Object) {
-    return o[getRandom(Object.keys(o))]
-  }
-}
-g.getRandom = getRandom
-
-function getRandomNumber (n) {
-  if (n == null) {
-    n = 9999
-  }
-  return Math.floor(Math.random() * n)
-}
-g.getRandomNumber = getRandomNumber
-
-function getRandomString () {
-  var chars = 'abcdefghijklmnopqrstuvwxyzäüöABCDEFGHIJKLMNOPQRSTUVWXYZÄÜÖ'
-  var char = chars[getRandomNumber(chars.length)] // ü\n\n\n\n\n\n\n'
-  var length = getRandomNumber(7)
-  var string = ''
-  for (var i = 0; i < length; i++) {
-    string += char
-  }
-  return string
-}
-g.getRandomString = getRandomString
-
-function * applyTransactions (relAmount, numberOfTransactions, objects, users, transactions, noReconnect) {
-  g.generateRandomSeed() // create a new seed, so we can re-create the behavior
-  for (var i = 0; i < numberOfTransactions * relAmount + 1; i++) {
-    var r = Math.random()
-    if (r > 0.95) {
-      // 10% chance of toggling concurrent user interactions.
-      // There will be an artificial delay until ops can be executed by the type,
-      // therefore, operations of the database will be (pre)transformed until user operations arrive
-      yield (function simulateConcurrentUserInteractions (type) {
-        if (!(type instanceof Y.utils.CustomType) && type.y instanceof Y.utils.CustomType) {
-          // usually we expect type to be a custom type. But in YXml we share an object {y: YXml, dom: Dom} instead
-          type = type.y
-        }
-        if (type.eventHandler.awaiting === 0 && type.eventHandler._debuggingAwaiting !== true) {
-          type.eventHandler.awaiting = 1
-          type.eventHandler._debuggingAwaiting = true
-        } else {
-          // fixAwaitingInType will handle _debuggingAwaiting
-          return fixAwaitingInType(type)
-        }
-      })(getRandom(objects))
-    } else if (r >= 0.5) {
-      // 40% chance to flush
-      yield Y.utils.globalRoom.flushOne() // flushes for some user.. (not necessarily 0)
-    } else if (noReconnect || r >= 0.05) {
-      // 45% chance to create operation
-      var done = getRandom(transactions)(getRandom(objects))
-      if (done != null) {
-        yield done
-      } else {
-        yield wait()
-      }
-      yield Y.utils.globalRoom.whenTransactionsFinished()
-    } else {
-      // 5% chance to disconnect/reconnect
-      var u = getRandom(users)
-      yield Promise.all(objects.map(fixAwaitingInType))
-      if (u.connector.isDisconnected()) {
-        yield u.reconnect()
-      } else {
-        yield u.disconnect()
-      }
-      yield Promise.all(objects.map(fixAwaitingInType))
-    }
-  }
-}
-
-function fixAwaitingInType (type) {
-  if (!(type instanceof Y.utils.CustomType) && type.y instanceof Y.utils.CustomType) {
-    // usually we expect type to be a custom type. But in YXml we share an object {y: YXml, dom: Dom} instead
-    type = type.y
-  }
-  return new Promise(function (resolve) {
-    type.os.whenTransactionsFinished().then(function () {
-      // _debuggingAwaiting artificially increases the awaiting property. We need to make sure that we only do that once / reverse the effect once
-      type.os.requestTransaction(function * () {
-        if (type.eventHandler.awaiting > 0 && type.eventHandler._debuggingAwaiting === true) {
-          type.eventHandler._debuggingAwaiting = false
-          yield * type.eventHandler.awaitOps(this, function * () { /* mock function */ })
-        }
-        wait(50).then(type.os.whenTransactionsFinished()).then(wait(50)).then(resolve)
-      })
-    })
-  })
-}
-g.fixAwaitingInType = fixAwaitingInType
-
-g.applyRandomTransactionsNoGCNoDisconnect = async(function * applyRandomTransactions (users, objects, transactions, numberOfTransactions) {
-  yield * applyTransactions(1, numberOfTransactions, objects, users, transactions, true)
-  yield Y.utils.globalRoom.flushAll()
-  yield Promise.all(objects.map(fixAwaitingInType))
-})
-
-g.applyRandomTransactionsAllRejoinNoGC = async(function * applyRandomTransactions (users, objects, transactions, numberOfTransactions) {
-  yield * applyTransactions(1, numberOfTransactions, objects, users, transactions)
-  yield Promise.all(objects.map(fixAwaitingInType))
-  yield Y.utils.globalRoom.flushAll()
-  yield Promise.all(objects.map(fixAwaitingInType))
-  for (var u in users) {
-    yield Promise.all(objects.map(fixAwaitingInType))
-    yield users[u].reconnect()
-    yield Promise.all(objects.map(fixAwaitingInType))
-  }
-  yield Promise.all(objects.map(fixAwaitingInType))
-  yield Y.utils.globalRoom.flushAll()
-  yield Promise.all(objects.map(fixAwaitingInType))
-  yield g.garbageCollectAllUsers(users)
-})
-
-g.applyRandomTransactionsWithGC = async(function * applyRandomTransactions (users, objects, transactions, numberOfTransactions) {
-  yield * applyTransactions(1, numberOfTransactions, objects, users.slice(1), transactions)
-  yield Y.utils.globalRoom.flushAll()
-  yield Promise.all(objects.map(fixAwaitingInType))
-  for (var u in users) {
-    // TODO: here, we enforce that two users never sync at the same time with u[0]
-    //       enforce that in the connector itself!
-    yield users[u].reconnect()
-  }
-  yield Y.utils.globalRoom.flushAll()
-  yield Promise.all(objects.map(fixAwaitingInType))
-  yield g.garbageCollectAllUsers(users)
-})
-
-g.garbageCollectAllUsers = async(function * garbageCollectAllUsers (users) {
-  yield Y.utils.globalRoom.flushAll()
-  for (var i in users) {
-    yield users[i].db.emptyGarbageCollector()
-  }
-})
-
-g.compareAllUsers = async(function * compareAllUsers (users) {
-  var s1, s2 // state sets
-  var ds1, ds2 // delete sets
-  var allDels1, allDels2 // all deletions
-  var db1 = [] // operation store of user1
-
-  yield Y.utils.globalRoom.flushAll()
-  yield g.garbageCollectAllUsers(users)
-  yield Y.utils.globalRoom.flushAll()
-
-  // disconnect, then reconnect all users
-  // We do this to make sure that the gc is updated by everyone
-  for (var i = 0; i < users.length; i++) {
-    yield users[i].disconnect()
-    yield wait()
-    yield users[i].reconnect()
-  }
-  yield wait()
-  yield Y.utils.globalRoom.flushAll()
-
-  // t1 and t2 basically do the same. They define t[1,2], ds[1,2], and allDels[1,2]
-  function * t1 () {
-    s1 = yield * this.getStateSet()
-    ds1 = yield * this.getDeleteSet()
-    allDels1 = []
-    yield * this.ds.iterate(this, null, null, function * (d) {
-      allDels1.push(d)
-    })
-  }
-  function * t2 () {
-    s2 = yield * this.getStateSet()
-    ds2 = yield * this.getDeleteSet()
-    allDels2 = []
-    yield * this.ds.iterate(this, null, null, function * (d) {
-      allDels2.push(d)
-    })
-  }
-
-  var buffer = Y.utils.globalRoom.buffers
-  for (var name in buffer) {
-    if (buffer[name].length > 0) {
-      // not all ops were transmitted..
-      debugger // eslint-disable-line
-    }
-  }
-
-  for (var uid = 0; uid < users.length; uid++) {
-    var u = users[uid]
-    u.db.requestTransaction(function * () {
-      var sv = yield * this.getStateVector()
-      for (var s of sv) {
-        yield * this.updateState(s.user)
-      }
-      // compare deleted ops against deleteStore
-      yield * this.os.iterate(this, null, null, function * (o) {
-        if (o.deleted === true) {
-          expect(yield * this.isDeleted(o.id)).toBeTruthy()
-        }
-      })
-      // compare deleteStore against deleted ops
-      var ds = []
-      yield * this.ds.iterate(this, null, null, function * (d) {
-        ds.push(d)
-      })
-      for (var j in ds) {
-        var d = ds[j]
-        for (var i = 0; i < d.len; i++) {
-          var o = yield * this.getInsertion([d.id[0], d.id[1] + i])
-          // gc'd or deleted
-          if (d.gc) {
-            expect(o).toBeFalsy()
-          } else {
-            expect(o.deleted).toBeTruthy()
-          }
-        }
-      }
-    })
-    // compare allDels tree
-    if (s1 == null) {
-      u.db.requestTransaction(function * () {
-        yield * t1.call(this)
-        yield * this.os.iterate(this, null, null, function * (o) {
-          o = Y.utils.copyObject(o)
-          delete o.origin
-          delete o.originOf
-          db1.push(o)
-        })
-      })
-    } else {
-      u.db.requestTransaction(function * () {
-        yield * t2.call(this)
-        var db2 = []
-        yield * this.os.iterate(this, null, null, function * (o) {
-          o = Y.utils.copyObject(o)
-          delete o.origin
-          delete o.originOf
-          db2.push(o)
-        })
-        expect(s1).toEqual(s2)
-        expect(allDels1).toEqual(allDels2) // inner structure
-        expect(ds1).toEqual(ds2) // exported structure
-        db2.forEach((o, i) => {
-          expect(db1[i]).toEqual(o)
-        })
-      })
-    }
-    yield u.db.whenTransactionsFinished()
-  }
-})
-
-g.createUsers = async(function * createUsers (self, numberOfUsers, database, initType) {
-  if (Y.utils.globalRoom.users[0] != null) {
-    yield Y.utils.globalRoom.flushAll()
-  }
-  // destroy old users
-  for (var u in Y.utils.globalRoom.users) {
-    Y.utils.globalRoom.users[u].y.destroy()
-  }
-  self.users = null
-
-  var promises = []
-  for (var i = 0; i < numberOfUsers; i++) {
-    promises.push(Y({
-      db: {
-        name: database,
-        namespace: 'User ' + i,
-        cleanStart: true,
-        gcTimeout: -1,
-        gc: true,
-        repairCheckInterval: -1
-      },
-      connector: {
-        name: 'Test',
-        debug: false
-      },
-      share: {
-        root: initType || 'Map'
-      }
-    }))
-  }
-  self.users = yield Promise.all(promises)
-  self.types = self.users.map(function (u) { return u.share.root })
-  return self.users
-})
-
-/*
-  Until async/await arrives in js, we use this function to wait for promises
-  by yielding them.
-*/
-function async (makeGenerator) {
-  return function (arg) {
-    var generator = makeGenerator.apply(this, arguments)
-
-    function handle (result) {
-      if (result.done) return Promise.resolve(result.value)
-
-      return Promise.resolve(result.value).then(function (res) {
-        return handle(generator.next(res))
-      }, function (err) {
-        return handle(generator.throw(err))
-      })
-    }
-    try {
-      return handle(generator.next())
-    } catch (ex) {
-      generator.throw(ex)
-      // return Promise.reject(ex)
-    }
-  }
-}
-g.async = async
-
-function logUsers (self) {
-  if (self.constructor === Array) {
-    self = {users: self}
-  }
-  self.users[0].db.logTable()
-  self.users[1].db.logTable()
-  self.users[2].db.logTable()
-}
-
-g.logUsers = logUsers

src/Struct.js

@@ -1,5 +1,7 @@
-/* @flow */
-'use strict'
+const CDELETE = 0
+const CINSERT = 1
+const CLIST = 2
+const CMAP = 3
 
 /*
  An operation also defines the structure of a type. This is why operation and
@@ -21,6 +23,20 @@
 */
 export default function extendStruct (Y) {
   var Struct = {
+    binaryDecodeOperation: function (decoder) {
+      let code = decoder.peekUint8()
+      if (code === CDELETE) {
+        return Y.Struct.Delete.binaryDecode(decoder)
+      } else if (code === CINSERT) {
+        return Y.Struct.Insert.binaryDecode(decoder)
+      } else if (code === CLIST) {
+        return Y.Struct.List.binaryDecode(decoder)
+      } else if (code === CMAP) {
+        return Y.Struct.Map.binaryDecode(decoder)
+      } else {
+        throw new Error('Unable to decode operation!')
+      }
+    },
     /* This is the only operation that is actually not a structure, because
     it is not stored in the OS. This is why it _does not_ have an id
 
@@ -36,6 +52,19 @@ export default function extendStruct (Y) {
           struct: 'Delete'
         }
       },
+      binaryEncode: function (encoder, op) {
+        encoder.writeUint8(CDELETE)
+        encoder.writeOpID(op.target)
+        encoder.writeVarUint(op.length || 0)
+      },
+      binaryDecode: function (decoder) {
+        decoder.skip8()
+        return {
+          target: decoder.readOpID(),
+          length: decoder.readVarUint(),
+          struct: 'Delete'
+        }
+      },
       requiredOps: function (op) {
         return [] // [op.target]
       },
@@ -77,6 +106,91 @@ export default function extendStruct (Y) {
 
         return e
       },
+      binaryEncode: function (encoder, op) {
+        encoder.writeUint8(CINSERT)
+        // compute info property
+        let contentIsText = op.content != null && op.content.every(c => typeof c === 'string' && c.length === 1)
+        let originIsLeft = Y.utils.compareIds(op.left, op.origin)
+        let info =
+          (op.parentSub != null ? 1 : 0) |
+          (op.opContent != null ? 2 : 0) |
+          (contentIsText ? 4 : 0) |
+          (originIsLeft ? 8 : 0) |
+          (op.left != null ? 16 : 0) |
+          (op.right != null ? 32 : 0) |
+          (op.origin != null ? 64 : 0)
+        encoder.writeUint8(info)
+        encoder.writeOpID(op.id)
+        encoder.writeOpID(op.parent)
+        if (info & 16) {
+          encoder.writeOpID(op.left)
+        }
+        if (info & 32) {
+          encoder.writeOpID(op.right)
+        }
+        if (!originIsLeft && info & 64) {
+          encoder.writeOpID(op.origin)
+        }
+        if (info & 1) {
+          // write parentSub
+          encoder.writeVarString(op.parentSub)
+        }
+        if (info & 2) {
+          // write opContent
+          encoder.writeOpID(op.opContent)
+        } else if (info & 4) {
+          // write text
+          encoder.writeVarString(op.content.join(''))
+        } else {
+          // convert to JSON and write
+          encoder.writeVarString(JSON.stringify(op.content))
+        }
+      },
+      binaryDecode: function (decoder) {
+        let op = {
+          struct: 'Insert'
+        }
+        decoder.skip8()
+        // get info property
+        let info = decoder.readUint8()
+
+        op.id = decoder.readOpID()
+        op.parent = decoder.readOpID()
+        if (info & 16) {
+          op.left = decoder.readOpID()
+        } else {
+          op.left = null
+        }
+        if (info & 32) {
+          op.right = decoder.readOpID()
+        } else {
+          op.right = null
+        }
+        if (info & 8) {
+          // origin is left
+          op.origin = op.left
+        } else if (info & 64) {
+          op.origin = decoder.readOpID()
+        } else {
+          op.origin = null
+        }
+        if (info & 1) {
+          // has parentSub
+          op.parentSub = decoder.readVarString()
+        }
+        if (info & 2) {
+          // has opContent
+          op.opContent = decoder.readOpID()
+        } else if (info & 4) {
+          // has pure text content
+          op.content = decoder.readVarString().split('')
+        } else {
+          // has mixed content
+          let s = decoder.readVarString()
+          op.content = JSON.parse(s)
+        }
+        return op
+      },
       requiredOps: function (op) {
         var ids = []
         if (op.left != null) {
@@ -299,14 +413,31 @@ export default function extendStruct (Y) {
           id: op.id,
           type: op.type
         }
-        if (op.requires != null) {
-          e.requires = op.requires
-        }
         if (op.info != null) {
           e.info = op.info
         }
         return e
       },
+      binaryEncode: function (encoder, op) {
+        encoder.writeUint8(CLIST)
+        encoder.writeOpID(op.id)
+        encoder.writeVarString(op.type)
+        let info = op.info != null ? JSON.stringify(op.info) : ''
+        encoder.writeVarString(info)
+      },
+      binaryDecode: function (decoder) {
+        decoder.skip8()
+        let op = {
+          id: decoder.readOpID(),
+          type: decoder.readVarString(),
+          struct: 'List'
+        }
+        let info = decoder.readVarString()
+        if (info.length > 0) {
+          op.info = JSON.parse(info)
+        }
+        return op
+      },
       requiredOps: function () {
         /*
         var ids = []
@@ -381,13 +512,36 @@ export default function extendStruct (Y) {
           map: {} // overwrite map!!
         }
         if (op.requires != null) {
-          e.requires = op.requires
+          e.requires = op.require
+          // TODO: !!
+          console.warn('requires is used! see same note above for List')
         }
         if (op.info != null) {
           e.info = op.info
         }
         return e
       },
+      binaryEncode: function (encoder, op) {
+        encoder.writeUint8(CMAP)
+        encoder.writeOpID(op.id)
+        encoder.writeVarString(op.type)
+        let info = op.info != null ? JSON.stringify(op.info) : ''
+        encoder.writeVarString(info)
+      },
+      binaryDecode: function (decoder) {
+        decoder.skip8()
+        let op = {
+          id: decoder.readOpID(),
+          type: decoder.readVarString(),
+          struct: 'Map',
+          map: {}
+        }
+        let info = decoder.readVarString()
+        if (info.length > 0) {
+          op.info = JSON.parse(info)
+        }
+        return op
+      },
       requiredOps: function () {
         return []
       },

src/Transaction.js

@@ -1,5 +1,3 @@
-/* @flow */
-'use strict'
 
 /*
   Partial definition of a transaction
@@ -96,7 +94,7 @@ export default function extendTransaction (Y) {
           send.push(Y.Struct[op.struct].encode(op))
         }
       }
-      if (this.store.y.connector.isSynced && send.length > 0) { // TODO: && !this.store.forwardAppliedOperations (but then i don't send delete ops)
+      if (send.length > 0) { // TODO: && !this.store.forwardAppliedOperations (but then i don't send delete ops)
         // is connected, and this is not going to be send in addOperation
         this.store.y.connector.broadcastOps(send)
       }
@@ -162,14 +160,14 @@ export default function extendTransaction (Y) {
             if (target.start != null) {
               // TODO: don't do it like this .. -.-
               yield * this.deleteList(target.start)
-              // yield* this.deleteList(target.id) -- do not gc itself because this may still get referenced
+              // yield * this.deleteList(target.id) -- do not gc itself because this may still get referenced
             }
             if (target.map != null) {
               for (var name in target.map) {
                 yield * this.deleteList(target.map[name])
               }
               // TODO: here to..  (see above)
-              // yield* this.deleteList(target.id) -- see above
+              // yield * this.deleteList(target.id) -- see above
             }
             if (target.opContent != null) {
               yield * this.deleteOperation(target.opContent)
@@ -223,7 +221,7 @@ export default function extendTransaction (Y) {
     */
     * markGarbageCollected (id, len) {
       // this.mem.push(["gc", id]);
-      this.store.addToDebug('yield* this.markGarbageCollected(', id, ', ', len, ')')
+      this.store.addToDebug('yield * this.markGarbageCollected(', id, ', ', len, ')')
       var n = yield * this.markDeleted(id, len)
       if (n.id[1] < id[1] && !n.gc) {
         // un-extend left
@@ -294,8 +292,9 @@ export default function extendTransaction (Y) {
                 yield * this.ds.put(n)
               } else {
                 // already gc'd
-                throw new Error('Cannot happen! (it dit though.. :()')
-                // return n
+                throw new Error(
+                  'DS reached an inconsistent state. Please report this issue!'
+                )
               }
             }
           } else {
@@ -366,6 +365,7 @@ export default function extendTransaction (Y) {
       operations that can be gc'd and add them to the garbage collector.
     */
     * garbageCollectAfterSync () {
+      // debugger
       if (this.store.gc1.length > 0 || this.store.gc2.length > 0) {
         console.warn('gc should be empty after sync')
       }
@@ -417,7 +417,7 @@ export default function extendTransaction (Y) {
       * reset origins of all right ops
     */
     * garbageCollectOperation (id) {
-      this.store.addToDebug('yield* this.garbageCollectOperation(', id, ')')
+      this.store.addToDebug('yield * this.garbageCollectOperation(', id, ')')
       var o = yield * this.getOperation(id)
       yield * this.markGarbageCollected(id, (o != null && o.content != null) ? o.content.length : 1) // always mark gc'd
       // if op exists, then clean that mess up..
@@ -459,16 +459,8 @@ export default function extendTransaction (Y) {
 
           if (o.originOf != null && o.originOf.length > 0) {
             // find new origin of right ops
-            // origin is the first left deleted operation
+            // origin is the first left operation
             var neworigin = o.left
-            var neworigin_ = null
-            while (neworigin != null) {
-              neworigin_ = yield * this.getInsertion(neworigin)
-              if (neworigin_.deleted) {
-                break
-              }
-              neworigin = neworigin_.left
-            }
 
             // reset origin of all right ops (except first right - duh!),
 
@@ -482,7 +474,7 @@ export default function extendTransaction (Y) {
             right.origin = neworigin
             // search until you find origin pointer to the left of o
             if (right.right != null) {
-              var i = yield* this.getOperation(right.right)
+              var i = yield * this.getOperation(right.right)
               var ids = [o.id, o.right]
               while (ids.some(function (id) {
                 return Y.utils.compareIds(id, i.origin)
@@ -490,14 +482,14 @@ export default function extendTransaction (Y) {
                 if (Y.utils.compareIds(i.origin, o.id)) {
                   // reset origin of i
                   i.origin = neworigin
-                  yield* this.setOperation(i)
+                  yield * this.setOperation(i)
                 }
                 // get next i
                 if (i.right == null) {
                   break
                 } else {
                   ids.push(i.id)
-                  i = yield* this.getOperation(i.right)
+                  i = yield * this.getOperation(i.right)
                 }
               }
             }
@@ -513,6 +505,7 @@ export default function extendTransaction (Y) {
               }
             }
             if (neworigin != null) {
+              var neworigin_ = yield * this.getInsertion(neworigin)
               if (neworigin_.originOf == null) {
                 neworigin_.originOf = o.originOf
               } else {
@@ -593,11 +586,20 @@ export default function extendTransaction (Y) {
       apply a delete set in order to get
       the state of the supplied ds
     */
-    * applyDeleteSet (ds) {
+    * applyDeleteSet (decoder) {
       var deletions = []
 
-      for (var user in ds) {
-        var dv = ds[user]
+      let dsLength = decoder.readUint32()
+      for (let i = 0; i < dsLength; i++) {
+        let user = decoder.readVarUint()
+        let dv = []
+        let dvLength = decoder.readVarUint()
+        for (let j = 0; j < dvLength; j++) {
+          let from = decoder.readVarUint()
+          let len = decoder.readVarUint()
+          let gc = decoder.readUint8() === 1
+          dv.push([from, len, gc])
+        }
         var pos = 0
         var d = dv[pos]
         yield * this.ds.iterate(this, [user, 0], [user, Number.MAX_VALUE], function * (n) {
@@ -691,21 +693,34 @@ export default function extendTransaction (Y) {
     /*
       A DeleteSet (ds) describes all the deleted ops in the OS
     */
-    * getDeleteSet () {
-      var ds = {}
+    * writeDeleteSet (encoder) {
+      var ds = new Map()
       yield * this.ds.iterate(this, null, null, function * (n) {
         var user = n.id[0]
         var counter = n.id[1]
         var len = n.len
         var gc = n.gc
-        var dv = ds[user]
+        var dv = ds.get(user)
         if (dv === void 0) {
           dv = []
-          ds[user] = dv
+          ds.set(user, dv)
         }
         dv.push([counter, len, gc])
       })
-      return ds
+      let keys = Array.from(ds.keys())
+      encoder.writeUint32(keys.length)
+      for (var i = 0; i < keys.length; i++) {
+        let user = keys[i]
+        let deletions = ds.get(user)
+        encoder.writeVarUint(user)
+        encoder.writeVarUint(deletions.length)
+        for (var j = 0; j < deletions.length; j++) {
+          let del = deletions[j]
+          encoder.writeVarUint(del[0])
+          encoder.writeVarUint(del[1])
+          encoder.writeUint8(del[2] ? 1 : 0)
+        }
+      }
     }
     * isDeleted (id) {
       var n = yield * this.ds.findWithUpperBound(id)
@@ -717,7 +732,8 @@ export default function extendTransaction (Y) {
     }
     * addOperation (op) {
       yield * this.os.put(op)
-      if (this.store.y.connector.isSynced && this.store.forwardAppliedOperations && typeof op.id[1] !== 'string') {
+      // case op is created by this user, op is already broadcasted in applyCreatedOperations
+      if (op.id[0] !== this.store.userId && this.store.forwardAppliedOperations && typeof op.id[1] !== 'string') {
         // is connected, and this is not going to be send in addOperation
         this.store.y.connector.broadcastOps([op])
       }
@@ -826,7 +842,7 @@ export default function extendTransaction (Y) {
     }
     * getOperation (id/* :any */)/* :Transaction<any> */ {
       var o = yield * this.os.find(id)
-      if (id[0] !== '_' || o != null) {
+      if (id[0] !== 0xFFFFFF || o != null) {
         return o
       } else { // type is string
         // generate this operation?
@@ -838,10 +854,10 @@ export default function extendTransaction (Y) {
           yield * this.setOperation(op)
           return op
         } else {
-          // won't be called. but just in case..
-          console.error('Unexpected case. How can this happen?')
-          debugger // eslint-disable-line
-          return null
+          throw new Error(
+            'Unexpected case. Operation cannot be generated correctly!' +
+            'Incompatible Yjs version?'
+          )
         }
       }
     }
@@ -883,6 +899,18 @@ export default function extendTransaction (Y) {
       })
       return ss
     }
+    * writeStateSet (encoder) {
+      let lenPosition = encoder.pos
+      let len = 0
+      encoder.writeUint32(0)
+      yield * this.ss.iterate(this, null, null, function * (n) {
+        encoder.writeVarUint(n.id[0])
+        encoder.writeVarUint(n.clock)
+        len++
+      })
+      encoder.setUint32(lenPosition, len)
+      return len === 0
+    }
     /*
       Here, we make all missing operations executable for the receiving user.
 
@@ -932,34 +960,43 @@ export default function extendTransaction (Y) {
     * getOperations (startSS) {
       // TODO: use bounds here!
       if (startSS == null) {
-        startSS = {}
+        startSS = new Map()
       }
       var send = []
 
       var endSV = yield * this.getStateVector()
-      for (var endState of endSV) {
-        var user = endState.user
-        if (user === '_') {
+      for (let endState of endSV) {
+        let user = endState.user
+        if (user === 0xFFFFFF) {
           continue
         }
-        var startPos = startSS[user] || 0
+        let startPos = startSS.get(user) || 0
         if (startPos > 0) {
           // There is a change that [user, startPos] is in a composed Insertion (with a smaller counter)
           // find out if that is the case
-          var firstMissing = yield * this.getInsertion([user, startPos])
+          let firstMissing = yield * this.getInsertion([user, startPos])
           if (firstMissing != null) {
             // update startPos
             startPos = firstMissing.id[1]
-            startSS[user] = startPos
           }
         }
+        startSS.set(user, startPos)
+      }
+      for (let endState of endSV) {
+        let user = endState.user
+        let startPos = startSS.get(user)
+        if (user === 0xFFFFFF) {
+          continue
+        }
         yield * this.os.iterate(this, [user, startPos], [user, Number.MAX_VALUE], function * (op) {
           op = Y.Struct[op.struct].encode(op)
           if (op.struct !== 'Insert') {
             send.push(op)
-          } else if (op.right == null || op.right[1] < (startSS[op.right[0]] || 0)) {
+          } else if (op.right == null || op.right[1] < (startSS.get(op.right[0]) || 0)) {
             // case 1. op.right is known
-            var o = op
+            // this case is only reached if op.right is known.
+            // => this is not called for op.left, as op.right is unknown
+            let o = op
             // Remember: ?
             // -> set op.right
             //    1. to the first operation that is known (according to startSS)
@@ -972,11 +1009,14 @@ export default function extendTransaction (Y) {
               if (o.left == null) {
                 op.left = null
                 send.push(op)
-                if (!Y.utils.compareIds(o.id, op.id)) {
+                /* not necessary, as o is already sent..
+                if (!Y.utils.compareIds(o.id, op.id) && o.id[1] >= (startSS.get(o.id[0]) || 0)) {
+                  // o is not op && o is unknown
                   o = Y.Struct[op.struct].encode(o)
                   o.right = missingOrigins[missingOrigins.length - 1].id
                   send.push(o)
                 }
+                */
                 break
               }
               o = yield * this.getInsertion(o.left)
@@ -984,7 +1024,7 @@ export default function extendTransaction (Y) {
               while (missingOrigins.length > 0 && Y.utils.matchesId(o, missingOrigins[missingOrigins.length - 1].origin)) {
                 missingOrigins.pop()
               }
-              if (o.id[1] < (startSS[o.id[0]] || 0)) {
+              if (o.id[1] < (startSS.get(o.id[0]) || 0)) {
                 // case 2. o is known
                 op.left = Y.utils.getLastId(o)
                 send.push(op)
@@ -996,8 +1036,10 @@ export default function extendTransaction (Y) {
                 op = Y.Struct[op.struct].encode(o)
                 op.right = newright
                 if (missingOrigins.length > 0) {
-                  debugger
-                  console.log('This should not happen .. :( please report this')
+                  throw new Error(
+                    'Reached inconsistent OS state.' +
+                    'Operations are not correctly connected.'
+                  )
                 }
                 missingOrigins = [op]
               } else {
@@ -1014,28 +1056,48 @@ export default function extendTransaction (Y) {
       }
       return send.reverse()
     }
+
+    * writeOperations (encoder, decoder) {
+      let ss = new Map()
+      let ssLength = decoder.readUint32()
+      for (let i = 0; i < ssLength; i++) {
+        let user = decoder.readVarUint()
+        let clock = decoder.readVarUint()
+        ss.set(user, clock)
+      }
+      let ops = yield * this.getOperations(ss)
+      encoder.writeUint32(ops.length)
+      for (let i = 0; i < ops.length; i++) {
+        let op = ops[i]
+        Y.Struct[op.struct].binaryEncode(encoder, Y.Struct[op.struct].encode(op))
+      }
+    }
     /*
      * Get the plain untransformed operations from the database.
      * You can apply these operations using .applyOperationsUntransformed(ops)
      *
      */
-    * getOperationsUntransformed () {
-      var ops = []
+    * writeOperationsUntransformed (encoder) {
+      let lenPosition = encoder.pos
+      let len = 0
+      encoder.writeUint32(0) // placeholder
       yield * this.os.iterate(this, null, null, function * (op) {
-        if (op.id[0] !== '_') {
-          ops.push(op)
+        if (op.id[0] !== 0xFFFFFF) {
+          len++
+          Y.Struct[op.struct].binaryEncode(encoder, Y.Struct[op.struct].encode(op))
         }
       })
-      return {
-        untransformed: ops
-      }
+      encoder.setUint32(lenPosition, len)
+      yield * this.writeStateSet(encoder)
     }
-    * applyOperationsUntransformed (m, stateSet) {
-      var ops = m.untransformed
-      for (var i = 0; i < ops.length; i++) {
-        var op = ops[i]
-        // create, and modify parent, if it is created implicitly
-        if (op.parent != null && op.parent[0] === '_') {
+    * applyOperationsUntransformed (decoder) {
+      let len = decoder.readUint32()
+      for (let i = 0; i < len; i++) {
+        let op = Y.Struct.binaryDecodeOperation(decoder)
+        yield * this.os.put(op)
+      }
+      yield * this.os.iterate(this, null, null, function * (op) {
+        if (op.parent != null) {
           if (op.struct === 'Insert') {
             // update parents .map/start/end properties
             if (op.parentSub != null && op.left == null) {
@@ -1055,12 +1117,14 @@ export default function extendTransaction (Y) {
             }
           }
         }
-        yield * this.os.put(op)
-      }
-      for (var user in stateSet) {
+      })
+      let stateSetLength = decoder.readUint32()
+      for (let i = 0; i < stateSetLength; i++) {
+        let user = decoder.readVarUint()
+        let clock = decoder.readVarUint()
         yield * this.ss.put({
           id: [user],
-          clock: stateSet[user]
+          clock: clock
         })
       }
     }
@@ -1075,7 +1139,7 @@ export default function extendTransaction (Y) {
       // or the o that has no origin to the right of op
       // (this is why we use the ids array)
       while (o.right != null) {
-        var right = yield* this.getOperation(o.right)
+        var right = yield * this.getOperation(o.right)
         if (o.right[1] < (startSS[o.right[0]] || 0) || !ids.some(function (id) {
           return Y.utils.compareIds(id, right.origin)
         })) {

src/Utils.js

@@ -1,3 +1,7 @@
+/* globals crypto */
+
+import { BinaryDecoder, BinaryEncoder } from './Encoding.js'
+
 /*
   EventHandler is an helper class for constructing custom types.
 
@@ -22,7 +26,10 @@
 */
 
 export default function Utils (Y) {
-  Y.utils = {}
+  Y.utils = {
+    BinaryDecoder: BinaryDecoder,
+    BinaryEncoder: BinaryEncoder
+  }
 
   Y.utils.bubbleEvent = function (type, event) {
     type.eventHandler.callEventListeners(event)
@@ -42,6 +49,32 @@ export default function Utils (Y) {
     }
   }
 
+  class NamedEventHandler {
+    constructor () {
+      this._eventListener = {}
+    }
+    on (name, f) {
+      if (this._eventListener[name] == null) {
+        this._eventListener[name] = []
+      }
+      this._eventListener[name].push(f)
+    }
+    off (name, f) {
+      if (name == null || f == null) {
+        throw new Error('You must specify event name and function!')
+      }
+      let listener = this._eventListener[name] || []
+      this._eventListener[name] = listener.filter(e => e !== f)
+    }
+    emit (name, value) {
+      (this._eventListener[name] || []).forEach(l => l(value))
+    }
+    destroy () {
+      this._eventListener = null
+    }
+  }
+  Y.utils.NamedEventHandler = NamedEventHandler
+
   class EventListenerHandler {
     constructor () {
       this.eventListeners = []
@@ -72,7 +105,12 @@ export default function Utils (Y) {
           }
           this.eventListeners[i](_event)
         } catch (e) {
-          console.error('Your observer threw an error. This error was caught so that Yjs still can ensure data consistency! In order to debug this error you have to check "Pause On Caught Exceptions"', e)
+          /*
+            Your observer threw an error. This error was caught so that Yjs
+            can ensure data consistency! In order to debug this error you
+            have to check "Pause On Caught Exceptions" in developer tools.
+          */
+          console.error(e)
         }
       }
     }
@@ -610,6 +648,7 @@ export default function Utils (Y) {
         }
       }
     }
+    return false
   }
   Y.utils.matchesId = matchesId
 
@@ -786,8 +825,19 @@ export default function Utils (Y) {
   }
   Y.utils.createSmallLookupBuffer = createSmallLookupBuffer
 
-  // Generates a unique id, for use as a user id.
-  // Thx to @jed for this script https://gist.github.com/jed/982883
-  function generateGuid(a){return a?(a^Math.random()*16>>a/4).toString(16):([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g,generateGuid)} // eslint-disable-line
-  Y.utils.generateGuid = generateGuid
+  function generateUserId () {
+    if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) {
+      // browser
+      let arr = new Uint32Array(1)
+      crypto.getRandomValues(arr)
+      return arr[0]
+    } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) {
+      // node
+      let buf = crypto.randomBytes(4)
+      return new Uint32Array(buf.buffer)[0]
+    } else {
+      return Math.ceil(Math.random() * 0xFFFFFFFF)
+    }
+  }
+  Y.utils.generateUserId = generateUserId
 }

src/y.js

@@ -1,9 +1,10 @@
-import debug from 'debug'
 import extendConnector from './Connector.js'
 import extendDatabase from './Database.js'
 import extendTransaction from './Transaction.js'
 import extendStruct from './Struct.js'
 import extendUtils from './Utils.js'
+import debug from 'debug'
+import { formatYjsMessage, formatYjsMessageType } from './MessageHandler.js'
 
 extendConnector(Y)
 extendDatabase(Y)
@@ -12,6 +13,8 @@ extendStruct(Y)
 extendUtils(Y)
 
 Y.debug = debug
+debug.formatters.Y = formatYjsMessage
+debug.formatters.y = formatYjsMessageType
 
 var requiringModules = {}
 
@@ -144,7 +147,7 @@ export default function Y (opts/* :YOptions */) /* :Promise<YConfig> */ {
   })
 }
 
-class YConfig {
+class YConfig extends Y.utils.NamedEventHandler {
   /* ::
   db: Y.AbstractDatabase;
   connector: Y.AbstractConnector;
@@ -152,6 +155,7 @@ class YConfig {
   options: Object;
   */
   constructor (opts, callback) {
+    super()
     this.options = opts
     this.db = new Y[opts.db.name](this, opts.db)
     this.connector = new Y[opts.connector.name](this, opts.connector)
@@ -168,7 +172,7 @@ class YConfig {
         var typeName = typeConstructor.splice(0, 1)
         var type = Y[typeName]
         var typedef = type.typeDefinition
-        var id = ['_', typedef.struct + '_' + typeName + '_' + propertyname + '_' + typeConstructor]
+        var id = [0xFFFFFF, typedef.struct + '_' + typeName + '_' + propertyname + '_' + typeConstructor]
         var args = []
         if (typeConstructor.length === 1) {
           try {
@@ -215,6 +219,9 @@ class YConfig {
       } else {
         return Promise.resolve()
       }
+    }).then(() => {
+      // remove existing event listener
+      super.destroy()
     })
   }
   close () {

test/encode-decode.js

@@ -0,0 +1,229 @@
+import { test } from 'cutest'
+import Chance from 'chance'
+import Y from '../src/y.js'
+import { BinaryEncoder, BinaryDecoder } from '../src/Encoding.js'
+
+function testEncoding (t, write, read, val) {
+  let encoder = new BinaryEncoder()
+  write(encoder, val)
+  let reader = new BinaryDecoder(encoder.createBuffer())
+  let result = read(reader)
+  t.log(`string encode: ${JSON.stringify(val).length} bytes / binary encode: ${encoder.data.length} bytes`)
+  t.compare(val, result, 'Compare results')
+}
+
+const writeVarUint = (encoder, val) => encoder.writeVarUint(val)
+const readVarUint = decoder => decoder.readVarUint()
+
+test('varUint 1 byte', async function varUint1 (t) {
+  testEncoding(t, writeVarUint, readVarUint, 42)
+})
+
+test('varUint 2 bytes', async function varUint2 (t) {
+  testEncoding(t, writeVarUint, readVarUint, 1 << 9 | 3)
+  testEncoding(t, writeVarUint, readVarUint, 1 << 9 | 3)
+})
+test('varUint 3 bytes', async function varUint3 (t) {
+  testEncoding(t, writeVarUint, readVarUint, 1 << 17 | 1 << 9 | 3)
+})
+
+test('varUint 4 bytes', async function varUint4 (t) {
+  testEncoding(t, writeVarUint, readVarUint, 1 << 25 | 1 << 17 | 1 << 9 | 3)
+})
+
+test('varUint of 2839012934', async function varUint2839012934 (t) {
+  testEncoding(t, writeVarUint, readVarUint, 2839012934)
+})
+
+test('varUint random', async function varUintRandom (t) {
+  const chance = new Chance(t.getSeed() * Math.pow(Number.MAX_SAFE_INTEGER))
+  testEncoding(t, writeVarUint, readVarUint, chance.integer({min: 0, max: (1 << 28) - 1}))
+})
+
+test('varUint random user id', async function varUintRandomUserId (t) {
+  t.getSeed() // enforces that this test is repeated
+  testEncoding(t, writeVarUint, readVarUint, Y.utils.generateUserId())
+})
+
+const writeVarString = (encoder, val) => encoder.writeVarString(val)
+const readVarString = decoder => decoder.readVarString()
+
+test('varString', async function varString (t) {
+  testEncoding(t, writeVarString, readVarString, 'hello')
+  testEncoding(t, writeVarString, readVarString, 'test!')
+  testEncoding(t, writeVarString, readVarString, '☺☺☺')
+  testEncoding(t, writeVarString, readVarString, '1234')
+})
+
+test('varString random', async function varStringRandom (t) {
+  const chance = new Chance(t.getSeed() * 1000000000)
+  testEncoding(t, writeVarString, readVarString, chance.string())
+})
+
+const writeDelete = Y.Struct.Delete.binaryEncode
+const readDelete = Y.Struct.Delete.binaryDecode
+
+test('encode/decode Delete operation', async function binDelete (t) {
+  let op = {
+    target: [10, 3000],
+    length: 40000,
+    struct: 'Delete'
+  }
+  testEncoding(t, writeDelete, readDelete, op)
+})
+
+const writeInsert = Y.Struct.Insert.binaryEncode
+const readInsert = Y.Struct.Insert.binaryDecode
+
+test('encode/decode Insert operations', async function binInsert (t) {
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: [5, 6],
+    left: [3, 4],
+    origin: [7, 8],
+    parent: [9, 10],
+    struct: 'Insert',
+    content: ['a']
+  })
+
+  t.log('left === origin')
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: [5, 6],
+    left: [3, 4],
+    origin: [3, 4],
+    parent: [9, 10],
+    struct: 'Insert',
+    content: ['a']
+  })
+
+  t.log('parentsub')
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: [5, 6],
+    left: [3, 4],
+    origin: [3, 4],
+    parent: [9, 10],
+    parentSub: 'sub',
+    struct: 'Insert',
+    content: ['a']
+  })
+
+  t.log('opContent')
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: [5, 6],
+    left: [3, 4],
+    origin: [3, 4],
+    parent: [9, 10],
+    struct: 'Insert',
+    opContent: [1000, 10000]
+  })
+
+  t.log('mixed content')
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: [5, 6],
+    left: [3, 4],
+    origin: [3, 4],
+    parent: [9, 10],
+    struct: 'Insert',
+    content: ['a', 1]
+  })
+
+  t.log('origin is null')
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: [5, 6],
+    left: [3, 4],
+    origin: null,
+    parent: [9, 10],
+    struct: 'Insert',
+    content: ['a']
+  })
+
+  t.log('left = origin = right = null')
+  testEncoding(t, writeInsert, readInsert, {
+    id: [1, 2],
+    right: null,
+    left: null,
+    origin: null,
+    parent: [9, 10],
+    struct: 'Insert',
+    content: ['a']
+  })
+})
+
+const writeList = Y.Struct.List.binaryEncode
+const readList = Y.Struct.List.binaryDecode
+
+test('encode/decode List operations', async function binList (t) {
+  testEncoding(t, writeList, readList, {
+    struct: 'List',
+    id: [100, 33],
+    type: 'Array'
+  })
+
+  t.log('info is an object')
+  testEncoding(t, writeList, readList, {
+    struct: 'List',
+    id: [100, 33],
+    type: 'Array',
+    info: { prop: 'yay' }
+  })
+
+  t.log('info is a string')
+  testEncoding(t, writeList, readList, {
+    struct: 'List',
+    id: [100, 33],
+    type: 'Array',
+    info: 'hi'
+  })
+
+  t.log('info is a number')
+  testEncoding(t, writeList, readList, {
+    struct: 'List',
+    id: [100, 33],
+    type: 'Array',
+    info: 400
+  })
+})
+
+const writeMap = Y.Struct.Map.binaryEncode
+const readMap = Y.Struct.Map.binaryDecode
+
+test('encode/decode Map operations', async function binMap (t) {
+  testEncoding(t, writeMap, readMap, {
+    struct: 'Map',
+    id: [100, 33],
+    type: 'Map',
+    map: {}
+  })
+
+  t.log('info is an object')
+  testEncoding(t, writeMap, readMap, {
+    struct: 'Map',
+    id: [100, 33],
+    type: 'Map',
+    info: { prop: 'yay' },
+    map: {}
+  })
+
+  t.log('info is a string')
+  testEncoding(t, writeMap, readMap, {
+    struct: 'Map',
+    id: [100, 33],
+    type: 'Map',
+    map: {},
+    info: 'hi'
+  })
+
+  t.log('info is a number')
+  testEncoding(t, writeMap, readMap, {
+    struct: 'Map',
+    id: [100, 33],
+    type: 'Map',
+    map: {},
+    info: 400
+  })
+})

tests-lib/helper.js

@@ -1,7 +1,7 @@
 
 import _Y from '../../yjs/src/y.js'
 
-import yMemory from '../../y-memory/src/Memory.js'
+import yMemory from '../../y-memory/src/y-memory.js'
 import yArray from '../../y-array/src/y-array.js'
 import yMap from '../../y-map/src/Map.js'
 import yTest from './test-connector.js'
@@ -12,6 +12,33 @@ export let Y = _Y
 
 Y.extend(yMemory, yArray, yMap, yTest)
 
+function * getStateSet () {
+  var ss = {}
+  yield * this.ss.iterate(this, null, null, function * (n) {
+    var user = n.id[0]
+    var clock = n.clock
+    ss[user] = clock
+  })
+  return ss
+}
+
+function * getDeleteSet () {
+  var ds = {}
+  yield * this.ds.iterate(this, null, null, function * (n) {
+    var user = n.id[0]
+    var counter = n.id[1]
+    var len = n.len
+    var gc = n.gc
+    var dv = ds[user]
+    if (dv === void 0) {
+      dv = []
+      ds[user] = dv
+    }
+    dv.push([counter, len, gc])
+  })
+  return ds
+}
+
 export async function garbageCollectUsers (t, users) {
   await flushAll(t, users)
   await Promise.all(users.map(u => u.db.emptyGarbageCollector()))
@@ -30,12 +57,14 @@ export async function compareUsers (t, users) {
     await wait(100)
   }
   await flushAll(t, users)
-
-  await users[0].db.garbageCollect()
-  await users[0].db.garbageCollect()
+  await wait()
+  await flushAll(t, users)
 
   var userTypeContents = users.map(u => u.share.array._content.map(c => c.val || JSON.stringify(c.type)))
 
+  await users[0].db.garbageCollect()
+  await users[0].db.garbageCollect()
+
   // disconnect all except user 0
   await Promise.all(users.slice(1).map(async u =>
     u.disconnect()
@@ -54,20 +83,35 @@ export async function compareUsers (t, users) {
       u.connector.whenSynced(resolve)
     })
   ))
-
+  let filterDeletedOps = users.every(u => u.db.gc === false)
   var data = await Promise.all(users.map(async (u) => {
     var data = {}
     u.db.requestTransaction(function * () {
-      var os = yield * this.getOperationsUntransformed()
+      let ops = []
+      yield * this.os.iterate(this, null, null, function * (op) {
+        ops.push(Y.Struct[op.struct].encode(op))
+      })
+
       data.os = {}
-      os.untransformed.forEach((op) => {
+      for (let i = 0; i < ops.length; i++) {
+        let op = ops[i]
         op = Y.Struct[op.struct].encode(op)
         delete op.origin
-        data.os[JSON.stringify(op.id)] = op
-        return op
-      })
-      data.ds = yield * this.getDeleteSet()
-      data.ss = yield * this.getStateSet()
+        /*
+          If gc = false, it is necessary to filter deleted ops
+          as they might have been split up differently..
+         */
+        if (filterDeletedOps) {
+          let opIsDeleted = yield * this.isDeleted(op.id)
+          if (!opIsDeleted) {
+            data.os[JSON.stringify(op.id)] = op
+          }
+        } else {
+          data.os[JSON.stringify(op.id)] = op
+        }
+      }
+      data.ds = yield * getDeleteSet.apply(this)
+      data.ss = yield * getStateSet.apply(this)
     })
     await u.db.whenTransactionsFinished()
     return data
@@ -89,9 +133,9 @@ export async function initArrays (t, opts) {
   var result = {
     users: []
   }
-  var share = Object.assign({ flushHelper: 'Map', array: 'Array' }, opts.share)
+  var share = Object.assign({ flushHelper: 'Map', array: 'Array', map: 'Map' }, opts.share)
   var chance = opts.chance || new Chance(t.getSeed() * 1000000000)
-  var connector = Object.assign({ room: 'debugging_' + t.name, testContext: t, chance }, opts.connector)
+  var connector = Object.assign({ room: 'debugging_' + t.name, generateUserId: false, testContext: t, chance }, opts.connector)
   for (let i = 0; i < opts.users; i++) {
     let dbOpts
     let connOpts

tests-lib/test-connector.js

@@ -1,51 +1,53 @@
 /* global Y */
 import { wait } from './helper.js'
+import { formatYjsMessage } from '../src/MessageHandler.js'
 
 var rooms = {}
 
 export class TestRoom {
   constructor (roomname) {
     this.room = roomname
-    this.users = {}
+    this.users = new Map()
     this.nextUserId = 0
   }
   join (connector) {
     if (connector.userId == null) {
-      connector.setUserId('' + (this.nextUserId++))
+      connector.setUserId(this.nextUserId++)
     }
-    Object.keys(this.users).forEach(uid => {
-      this.users[uid].userJoined(connector.userId, connector.role)
-      connector.userJoined(uid, this.users[uid].role)
+    this.users.forEach((user, uid) => {
+      if (user.role === 'master' || connector.role === 'master') {
+        this.users.get(uid).userJoined(connector.userId, connector.role)
+        connector.userJoined(uid, this.users.get(uid).role)
+      }
     })
-    this.users[connector.userId] = connector
+    this.users.set(connector.userId, connector)
   }
   leave (connector) {
-    delete this.users[connector.userId]
-    Object.keys(this.users).forEach(uid => {
-      this.users[uid].userLeft(connector.userId)
+    this.users.delete(connector.userId)
+    this.users.forEach(user => {
+      user.userLeft(connector.userId)
     })
   }
   send (sender, receiver, m) {
-    m = JSON.parse(JSON.stringify(m))
-    var user = this.users[receiver]
+    var user = this.users.get(receiver)
     if (user != null) {
       user.receiveMessage(sender, m)
     }
   }
   broadcast (sender, m) {
-    Object.keys(this.users).forEach(receiver => {
+    this.users.forEach((user, receiver) => {
       this.send(sender, receiver, m)
     })
   }
   async flushAll (users) {
     let flushing = true
-    let allUserIds = Object.keys(this.users)
+    let allUserIds = Array.from(this.users.keys())
     if (users == null) {
-      users = allUserIds.map(id => this.users[id].y)
+      users = allUserIds.map(id => this.users.get(id).y)
     }
     while (flushing) {
       await wait(10)
-      let res = await Promise.all(allUserIds.map(id => this.users[id]._flushAll(users)))
+      let res = await Promise.all(allUserIds.map(id => this.users.get(id)._flushAll(users)))
       flushing = res.some(status => status === 'flushing')
     }
   }
@@ -67,6 +69,7 @@ export default function extendTestConnector (Y) {
       if (options.room == null) {
         throw new Error('You must define a room name!')
       }
+      options.forwardAppliedOperations = options.role === 'master'
       super(y, options)
       this.options = options
       this.room = options.room
@@ -78,14 +81,25 @@ export default function extendTestConnector (Y) {
       this.testRoom.leave(this)
       return super.disconnect()
     }
+    logBufferParsed () {
+      console.log(' === Logging buffer of user ' + this.userId + ' === ')
+      for (let [user, conn] of this.connections) {
+        console.log(` ${user}:`)
+        for (let i = 0; i < conn.buffer.length; i++) {
+          console.log(formatYjsMessage(conn.buffer[i]))
+        }
+      }
+    }
     reconnect () {
       this.testRoom.join(this)
       return super.reconnect()
     }
     send (uid, message) {
+      super.send(uid, message)
       this.testRoom.send(this.userId, uid, message)
     }
     broadcast (message) {
+      super.broadcast(message)
       this.testRoom.broadcast(this.userId, message)
     }
     async whenSynced (f) {
@@ -105,10 +119,10 @@ export default function extendTestConnector (Y) {
       })
     }
     receiveMessage (sender, m) {
-      if (this.userId !== sender && this.connections[sender] != null) {
-        var buffer = this.connections[sender].buffer
+      if (this.userId !== sender && this.connections.has(sender)) {
+        var buffer = this.connections.get(sender).buffer
         if (buffer == null) {
-          buffer = this.connections[sender].buffer = []
+          buffer = this.connections.get(sender).buffer = []
         }
         buffer.push(m)
         if (this.chance.bool({likelihood: 30})) {
@@ -123,18 +137,19 @@ export default function extendTestConnector (Y) {
     async _flushAll (flushUsers) {
       if (flushUsers.some(u => u.connector.userId === this.userId)) {
         // this one needs to sync with every other user
-        flushUsers = Object.keys(this.connections).map(id => this.testRoom.users[id].y)
+        flushUsers = Array.from(this.connections.keys()).map(uid => this.testRoom.users.get(uid).y)
       }
       var finished = []
       for (let i = 0; i < flushUsers.length; i++) {
         let userId = flushUsers[i].connector.userId
-        if (userId === this.userId) continue
-        let buffer = this.connections[userId].buffer
-        if (buffer != null) {
-          var messages = buffer.splice(0)
-          for (let j = 0; j < messages.length; j++) {
-            let p = super.receiveMessage(userId, messages[j])
-            finished.push(p)
+        if (userId !== this.userId && this.connections.has(userId)) {
+          let buffer = this.connections.get(userId).buffer
+          if (buffer != null) {
+            var messages = buffer.splice(0)
+            for (let j = 0; j < messages.length; j++) {
+              let p = super.receiveMessage(userId, messages[j])
+              finished.push(p)
+            }
           }
         }
       }