Deploy 0.8.8

.gitignore

@@ -1,5 +1,5 @@
 node_modules
-Examples/bower_components
+bower_components
 .directory
 .codio
 .settings

Examples/Quill/index.html

@@ -139,7 +139,7 @@
 
   <!-- Include the Quill library -->
   <script src="../bower_components/quill/dist/quill.js"></script>
-  <script src="../bower_components/yjs/y.es6"></script>
+  <script src="../bower_components/yjs/y.js"></script>
   <script src="./index.js"></script>
 </body>
 </html>

Examples/Quill/index.js

@@ -4,13 +4,14 @@
 
 Y({
   db: {
-    name: 'memory'
+    name: 'memory',
+    namespace: 'richtext-example20'
   },
   connector: {
     name: 'websockets-client',
-    room: 'richtext-example18',
-    debug: true
-    //url: 'http://127.0.0.1:2345'
+    room: 'richtext-example20',
+    debug: true,
+    url: 'http://127.0.0.1:1234'
   },
   sourceDir: '/bower_components',
   share: {
@@ -30,3 +31,22 @@ Y({
   // bind quill to richtext type
   y.share.richtext.bind(window.quill)
 })
+
+Y({
+  db: {
+    name: 'memory',
+    namespace: 'richtext-example220'
+  },
+  connector: {
+    name: 'websockets-client',
+    room: 'richtext-example22'
+    // debug: true,
+    // url: 'http://127.0.0.1:1234'
+  },
+  sourceDir: '/bower_components',
+  share: {
+    richtext: 'Richtext' // y.share.richtext is of type Y.Richtext
+  }
+}).then(function (y) {
+  window.y = y
+})

Examples/bower.json

@@ -9,7 +9,15 @@
   "license": "MIT",
   "ignore": [],
   "dependencies": {
-    "yjs": "../",
-    "y-webrtc": "~0.6.4"
+    "yjs": "~0.7.6",
+    "y-array": "~0.7.5",
+    "y-map": "~0.7.2",
+    "y-memory": "~0.7.0",
+    "y-richtext": "~0.7.5",
+    "y-webrtc": "~0.7.1",
+    "y-websockets-client": "~0.7.10",
+    "y-text": "~0.7.1",
+    "y-indexeddb": "~0.7.1",
+    "quill": "~0.20.1"
   }
 }

README.md

@@ -1,17 +1,16 @@
 
 # ![Yjs](http://y-js.org/images/yjs.png)
 
-Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data types. The framework implements a new OT-like concurrency algorithm and provides similar functionality as [ShareJs] and [OpenCoweb]. Yjs was designed to handle concurrent actions on arbitrary complex data types like Text, Json, and XML. We provide a tutorial and some applications for this framework on our [homepage](http://y-js.org/).
+Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data. The framework provides similar functionality as [ShareJs] and [OpenCoweb], but it implements a new algorithm that also support peer-to-peer communication protocols. Yjs was designed to handle concurrent actions on arbitrary data types like Text, Json, and XML. We also provide support for storing and manipulating your shared data offline. For more information and demo applications visit our [homepage](http://y-js.org/).
 
 **NOTE** This project is currently migrating. So there may exist some information that is not true anymore..
 
-You can create you own shared types easily. Therefore, you can take matters into your own hand by defining the meaning of the shared types and ensure that it is valid, while Yjs ensures data consistency (everyone will eventually end up with the same data). We already provide data types for
+You can create you own shared types easily. Therefore, you can take matters into your own hand by defining the meaning of the shared types, and ensure that it is valid, while Yjs ensures data consistency (everyone will eventually end up with the same data). We already provide data types for
 
 | Name     | Description       |
 |----------|-------------------|
 |[map](https://github.com/y-js/y-map) | Add, update, and remove properties of an object. Included in Yjs|
 |[array](https://github.com/y-js/y-array) | A shared linked list implementation |
-|[selections](https://github.com/y-js/y-selections) | Manages selections on types that use linear structures (e.g. the y-array type). Select a range of elements, and assign meaning to them.|
 |[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects|
 |[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *h1*, or *p*)|
 |[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to several editors|
@@ -24,26 +23,30 @@ We support several communication protocols as so called *Connectors*. You can cr
 |----------------|-----------------------------------|
 |[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))|
 |[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC|
+|[websockets](https://github.com/y-js/y-websockets-client) | Exchange updates efficiently in the classical client-server model |
 |[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios|
 
+You are not limited to use a specific database to store the shared data. We provide the following database adapters:
+
+|Name            | Description               |
+|----------------|-----------------------------------|
+|[memory](https://github.com/y-js/y-memory) | In-memory storage. |
+|[IndexedDb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser |
 
 You can use Yjs client-, and server- side. You can get it as via npm, and bower. We even provide polymer elements for Yjs!
 
 The advantages over similar frameworks are support for
 * .. P2P message propagation and arbitrary communication protocols
 * .. arbitrary complex data types
-* .. offline editing: Changes are stored persistently and only relevant changes are propagated on rejoin
-* .. AnyUndo: Undo *any* action that was executed in constant time (coming..)
+* .. offline support: Changes are stored persistently and only relevant changes are propagated on rejoin
 * .. Intention Preservation: When working on Text, the intention of your changes are preserved. This is particularily important when working offline. Every type has a notion on how we define Intention Preservation on it.
 
 ## Use it!
-You can find a tutorial, and examples on the [website](http://y-js.org). Furthermore, the [github wiki](https://github.com/y-js/yjs/wiki) offers more information about how you can use Yjs in your application.
-
-Either clone this git repository, install it with [bower](http://bower.io/), or install it with [npm](https://www.npmjs.org/package/yjs).
+Install yjs and its modules with [bower](http://bower.io/), or with [npm](https://www.npmjs.org/package/yjs).
 
 ### Bower
 ```
-bower install y-js/yjs
+bower install yjs
 ```
 Then you include the libraries directly from the installation folder.
 ```
@@ -60,58 +63,30 @@ And use it like this with *npm*:
 Y = require("yjs");
 ```
 
-# Y()
-In order to create an instance of Y, you need to have a connection object (instance of a Connector). Then, you can create a shared data type like this:
+# Text editing example
 ```
-var y = new Y(connector);
+Y({
+  db: {
+    name: 'memory' // store in memory.
+    // name: 'indexeddb'
+  },
+  connector: {
+    name: 'websockets-client', // choose the websockets connector
+    // name: 'webrtc'
+    // name: 'xmpp'
+    room: 'Textarea-example-dev'
+  },
+  sourceDir: '/bower_components', // location of the y-* modules
+  share: {
+    textarea: 'Text' // y.share.textarea is of type Y.Text
+  }
+  // modules: ['Richtext', 'Array'] // optional list of modules you want to import
+}).then(function (y) {
+  // bind the textarea to a shared text element
+  y.share.textarea.bind(document.getElementById('textfield'))
+}
 ```
 
-
-# Y.Map
-Yjs includes only one type by default - the Y.Map type. It mimics the behaviour of a javascript Object. You can create, update, and remove properies on the Y.Map type. Furthermore, you can observe changes on this type as you can observe changes on Javascript Objects with [Object.observe](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/observe) - an ECMAScript 7 proposal which is likely to become accepted by the committee. Until then, we have our own implementation.
-
-
-##### Reference
-* Create
-```
-var map = y.set("new_map", Y.Map).then(function(map){
-  map // is my map type
-});
-```
-* Every instance of Y is an Y.Map
-```
-var y = new Y(options);
-```
-* .get(name)
-  * Retrieve the value of a property. If the value is a type, `.get(name)` returns a promise
-* .set(name, value)
-  * Set/update a property. `value` may be a primitive type, or a custom type definition (e.g. `Y.Map`)
-* .delete(name)
-  * Delete a property
-* .observe(observer)
-  * The `observer` is called whenever something on this object changes. Throws *add*, *update*, and *delete* events
-* .observePath(path, observer)
-  * `path` is an array of property names. `observer` is called when the property under `path` is set, deleted, or updated
-* .unobserve(f)
-  * Delete an observer
-
-# A note on intention preservation
-When users create/update/delete the same property concurrently, only one change will prevail. Changes on different properties do not conflict with each other.
-
-# A note on time complexities
-* .get(name)
-  * O(1)
-* .set(name, value)
-  * O(1)
-* .delete(name)
-  * O(1)
-* Apply a delete operation from another user
-  * O(1)
-* Apply an update operation from another user (set/update a property)
-  * Yjs does not transform against operations that do not conflict with each other.
-  * An operation conflicts with another operation if it changes the same property.
-  * Overall worst case complexety: O(|conflicts|!)
-
 # Status
 Yjs is a work in progress. Different versions of the *y-* repositories may not work together. Just drop me a line if you run into troubles.
 

bower.json

@@ -1,6 +1,6 @@
 {
   "name": "yjs",
-  "version": "0.7.7",
+  "version": "0.8.9",
   "homepage": "y-js.org",
   "authors": [
     "Kevin Jahns <kevin.jahns@rwth-aachen.de>"

y.es6

@@ -51,6 +51,7 @@ module.exports = function (Y/* :any */) {
       this.debug = opts.debug === true
       this.broadcastedHB = false
       this.syncStep2 = Promise.resolve()
+      this.broadcastOpBuffer = []
     }
     reconnect () {
     }
@@ -75,19 +76,21 @@ module.exports = function (Y/* :any */) {
       this.userEventListeners.push(f)
     }
     userLeft (user) {
-      delete this.connections[user]
-      if (user === this.currentSyncTarget) {
-        this.currentSyncTarget = null
-        this.findNextSyncTarget()
-      }
-      this.syncingClients = this.syncingClients.filter(function (cli) {
-        return cli !== user
-      })
-      for (var f of this.userEventListeners) {
-        f({
-          action: 'userLeft',
-          user: user
+      if (this.connections[user] != null) {
+        delete this.connections[user]
+        if (user === this.currentSyncTarget) {
+          this.currentSyncTarget = null
+          this.findNextSyncTarget()
+        }
+        this.syncingClients = this.syncingClients.filter(function (cli) {
+          return cli !== user
         })
+        for (var f of this.userEventListeners) {
+          f({
+            action: 'userLeft',
+            user: user
+          })
+        }
       }
     }
     userJoined (user, role) {
@@ -167,6 +170,34 @@ module.exports = function (Y/* :any */) {
         console.log(`send ${this.userId} -> ${uid}: ${message.type}`, message) // eslint-disable-line
       }
     }
+    /*
+      Buffer operations, and broadcast them when ready.
+    */
+    broadcastOps (ops) {
+      ops = ops.map(function (op) {
+        return Y.Struct[op.struct].encode(op)
+      })
+      var self = this
+      function broadcastOperations () {
+        if (self.broadcastOpBuffer.length > 0) {
+          self.broadcast({
+            type: 'update',
+            ops: self.broadcastOpBuffer
+          })
+          self.broadcastOpBuffer = []
+        }
+      }
+      if (this.broadcastOpBuffer.length === 0) {
+        this.broadcastOpBuffer = ops
+        if (this.y.db.transactionInProgress) {
+          this.y.db.whenTransactionsFinished().then(broadcastOperations)
+        } else {
+          setTimeout(broadcastOperations, 0)
+        }
+      } else {
+        this.broadcastOpBuffer = this.broadcastOpBuffer.concat(ops)
+      }
+    }
     /*
       You received a raw message, and you know that it is intended for Yjs. Then call this function.
     */
@@ -227,15 +258,14 @@ module.exports = function (Y/* :any */) {
           db.requestTransaction(function * () {
             var ops = yield* this.getOperations(m.stateSet)
             if (ops.length > 0) {
-              var update /* :MessageUpdate */ = {
-                type: 'update',
-                ops: ops
-              }
               if (!broadcastHB) { // TODO: consider to broadcast here..
-                conn.send(sender, update)
+                conn.send(sender, {
+                  type: 'update',
+                  ops: ops
+                })
               } else {
                 // broadcast only once!
-                conn.broadcast(update)
+                conn.broadcastOps(ops)
               }
             }
             defer.resolve()
@@ -257,10 +287,7 @@ module.exports = function (Y/* :any */) {
             return o.struct === 'Delete'
           })
           if (delops.length > 0) {
-            this.broadcast({
-              type: 'update',
-              ops: delops
-            })
+            this.broadcastOps(delops)
           }
         }
         this.y.db.apply(message.ops)
@@ -1406,10 +1433,7 @@ module.exports = function (Y/* :any */) {
       }
       if (!this.store.y.connector.isDisconnected() && send.length > 0) { // TODO: && !this.store.forwardAppliedOperations (but then i don't send delete ops)
         // is connected, and this is not going to be send in addOperation
-        this.store.y.connector.broadcast({
-          type: 'update',
-          ops: send
-        })
+        this.store.y.connector.broadcastOps(send)
       }
     }
 
@@ -1805,10 +1829,7 @@ module.exports = function (Y/* :any */) {
         var ops = deletions.map(function (d) {
           return {struct: 'Delete', target: [d[0], d[1]]}
         })
-        this.store.y.connector.broadcast({
-          type: 'update',
-          ops: ops
-        })
+        this.store.y.connector.broadcastOps(ops)
       }
     }
     * isGarbageCollected (id) {
@@ -1846,10 +1867,7 @@ module.exports = function (Y/* :any */) {
       yield* this.os.put(op)
       if (!this.store.y.connector.isDisconnected() && this.store.forwardAppliedOperations && op.id[0] !== '_') {
         // is connected, and this is not going to be send in addOperation
-        this.store.y.connector.broadcast({
-          type: 'update',
-          ops: [op]
-        })
+        this.store.y.connector.broadcastOps([op])
       }
     }
     * getOperation (id/* :any */)/* :Transaction<any> */ {
@@ -1909,34 +1927,8 @@ module.exports = function (Y/* :any */) {
       })
       return ss
     }
-    * getOperations (startSS) {
-      // TODO: use bounds here!
-      if (startSS == null) {
-        startSS = {}
-      }
-      var ops = []
-
-      var endSV = yield* this.getStateVector()
-      for (var endState of endSV) {
-        var user = endState.user
-        if (user === '_') {
-          continue
-        }
-        var startPos = startSS[user] || 0
-
-        yield* this.os.iterate(this, [user, startPos], [user, Number.MAX_VALUE], function * (op) {
-          ops.push(op)
-        })
-      }
-      var res = []
-      for (var op of ops) {
-        var o = yield* this.makeOperationReady(startSS, op)
-        res.push(o)
-      }
-      return res
-    }
     /*
-      Here, we make op executable for the receiving user.
+      Here, we make all missing operations executable for the receiving user.
 
       Notes:
         startSS: denotes to the SV that the remote user sent
@@ -1971,7 +1963,92 @@ module.exports = function (Y/* :any */) {
             (startSS or currSS.. ?)
             -> Could be necessary when I turn GC again.
             -> Is a bad(ish) idea because it requires more computation
+
+      What we do:
+      * Iterate over all missing operations.
+      * When there is an operation, where the right op is known, send this op all missing ops to the left to the user
+      * I explained above what we have to do with each operation. Here is how we do it efficiently:
+        1. Go to the left until you find either op.origin, or a known operation (let o denote current operation in the iteration)
+        2. Found a known operation -> set op.left = o, and send it to the user. stop
+        3. Found o = op.origin -> set op.left = op.origin, and send it to the user. start again from 1. (set op = o)
+        4. Found some o -> set o.right = op, o.left = o.origin, send it to the user, continue
     */
+    * getOperations (startSS) {
+      // TODO: use bounds here!
+      if (startSS == null) {
+        startSS = {}
+      }
+      var send = []
+
+      var endSV = yield* this.getStateVector()
+      for (var endState of endSV) {
+        var user = endState.user
+        if (user === '_') {
+          continue
+        }
+        var startPos = startSS[user] || 0
+
+        yield* this.os.iterate(this, [user, startPos], [user, Number.MAX_VALUE], function * (op) {
+          op = Y.Struct[op.struct].encode(op)
+          if (op.struct !== 'Insert') {
+            send.push(op)
+          } else if (op.right == null || op.right[1] < (startSS[op.right[0]] || 0)) {
+            // case 1. op.right is known
+            var o = op
+            // Remember: ?
+            // -> set op.right
+            //    1. to the first operation that is known (according to startSS)
+            //    2. or to the first operation that has an origin that is not to the
+            //      right of op.
+            // For this we maintain a list of ops which origins are not found yet.
+            var missing_origins = [op]
+            var newright = op.right
+            while (true) {
+              if (o.left == null) {
+                op.left = null
+                send.push(op)
+                if (!Y.utils.compareIds(o.id, op.id)) {
+                  o = Y.Struct[op.struct].encode(o)
+                  o.right = missing_origins[missing_origins.length - 1].id
+                  send.push(o)
+                }
+                break
+              }
+              o = yield* this.getOperation(o.left)
+              // we set another o, check if we can reduce $missing_origins
+              while (missing_origins.length > 0 && Y.utils.compareIds(missing_origins[missing_origins.length - 1].origin, o.id)) {
+                missing_origins.pop()
+              }
+              if (o.id[1] < (startSS[o.id[0]] || 0)) {
+                // case 2. o is known
+                op.left = o.id
+                send.push(op)
+                break
+              } else if (Y.utils.compareIds(o.id, op.origin)) {
+                // case 3. o is op.origin
+                op.left = op.origin
+                send.push(op)
+                op = Y.Struct[op.struct].encode(o)
+                op.right = newright
+                if (missing_origins.length > 0) {
+                  console.log('This should not happen .. :( please report this')
+                }
+                missing_origins = [op]
+              } else {
+                // case 4. send o, continue to find op.origin
+                var s = Y.Struct[op.struct].encode(o)
+                s.right = missing_origins[missing_origins.length - 1].id
+                s.left = s.origin
+                send.push(s)
+                missing_origins.push(o)
+              }
+            }
+          }
+        })
+      }
+      return send.reverse()
+    }
+    /* this is what we used before.. use this as a reference..
     * makeOperationReady (startSS, op) {
       op = Y.Struct[op.struct].encode(op)
       op = Y.utils.copyObject(op)
@@ -1995,6 +2072,7 @@ module.exports = function (Y/* :any */) {
       op.left = op.origin
       return op
     }
+    */
   }
   Y.Transaction = TransactionInterface
 }
@@ -2325,8 +2403,9 @@ function Y (opts/* :YOptions */) /* :Promise<YConfig> */ {
   Y.sourceDir = opts.sourceDir
   return Y.requestModules(modules).then(function () {
     return new Promise(function (resolve) {
-      var yconfig = new YConfig(opts, function () {
-        yconfig.db.whenUserIdSet(function () {
+      var yconfig = new YConfig(opts)
+      yconfig.db.whenUserIdSet(function () {
+        yconfig.init(function () {
           resolve(yconfig)
         })
       })
@@ -2343,6 +2422,10 @@ class YConfig {
   constructor (opts, callback) {
     this.db = new Y[opts.db.name](this, opts.db)
     this.connector = new Y[opts.connector.name](this, opts.connector)
+    this.options = opts
+  }
+  init (callback) {
+    var opts = this.options
     var share = {}
     this.share = share
     this.db.requestTransaction(function * requestTransaction () {