Merge branch 'master' of https://github.com/y-js/yjs

README.md

@@ -177,8 +177,16 @@ If you want to see an issue fixed, please subscribe to the thread (or remind me
 
 ## Changelog
 
+### 10.0.0
+
+* Support for more complex types (a type can be a composition of several types)
+* Fixes several memory leaks
+
 ### 9.0.0
-There were several rolling updates from 0.6 to 0.8. We'll now follow the semver versioning scheme. This is all what this jump from 0.8 to 9.0 is about.
+There were several rolling updates from 0.6 to 0.8. We consider Yjs stable since a long time, 
+and intend to continue stable releases. From this release forward y-* modules will implement peer-dependencies for npm, and dependencies for bower.
+Furthermore, incompatible yjs instances will now throw errors when syncing - this feature was influenced by #48. The versioning jump was influenced by react (see [here](https://facebook.github.io/react/blog/2016/02/19/new-versioning-scheme.html))
+
 
 ### 0.6.0
 This is a complete rewrite of the 0.5 version of Yjs. Since Yjs 0.6.0 it is possible to work asynchronously on a persistent database, which enables offline support.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yjs",
-  "version": "9.1.0",
+  "version": "10.0.2",
   "description": "A framework for real-time p2p shared editing on arbitrary complex data types",
   "main": "./src/y.js",
   "scripts": {

src/Connector.js

@@ -19,6 +19,8 @@ module.exports = function (Y/* :any */) {
     userId: UserId;
     send: Function;
     broadcast: Function;
+    broadcastOpBuffer: Array<Operation>;
+    protocolVersion: number;
     */
     /*
       opts contains the following information:
@@ -51,7 +53,7 @@ module.exports = function (Y/* :any */) {
       this.broadcastedHB = false
       this.syncStep2 = Promise.resolve()
       this.broadcastOpBuffer = []
-      this.protocolVersion = 8
+      this.protocolVersion = 10
     }
     reconnect () {
     }
@@ -150,7 +152,8 @@ module.exports = function (Y/* :any */) {
           conn.send(syncUser, {
             type: 'sync step 1',
             stateSet: stateSet,
-            deleteSet: deleteSet
+            deleteSet: deleteSet,
+            protocolVersion: conn.protocolVersion
           })
         })
       } else {
@@ -234,7 +237,8 @@ module.exports = function (Y/* :any */) {
             type: 'sync step 2',
             os: ops,
             stateSet: currentStateSet,
-            deleteSet: ds
+            deleteSet: ds,
+            protocolVersion: this.protocolVersion
           })
           if (this.forwardToSyncingClients) {
             conn.syncingClients.push(sender)

src/Database.js

@@ -110,6 +110,18 @@ module.exports = function (Y /* :any */) {
         garbageCollect()
       }
     }
+    emptyGarbageCollector () {
+      return new Promise(resolve => {
+        var check = () => {
+          if (this.gc1.length > 0 || this.gc2.length > 0) {
+            this.garbageCollect().then(check)
+          } else {
+            resolve()
+          }
+        }
+        setTimeout(check, 0)
+      })
+    }
     addToDebug () {
       if (typeof YConcurrency_TestingMode !== 'undefined') {
         var command /* :string */ = Array.prototype.map.call(arguments, function (s) {
@@ -339,15 +351,7 @@ module.exports = function (Y /* :any */) {
         }
       } else {
         // increase SS
-        var o = op
+        yield* transaction.updateState(op.id[0])
-        var state = yield* transaction.getState(op.id[0])
-        while (o != null && o.id[1] === state.clock && op.id[0] === o.id[0]) {
-          // either its a new operation (1. case), or it is an operation that was deleted, but is not yet in the OS
-          state.clock++
-          yield* transaction.checkDeleteStoreForState(state)
-          o = yield* transaction.os.findNext(o.id)
-        }
-        yield* transaction.setState(state)
 
         // notify whenOperation listeners (by id)
         var sid = JSON.stringify(op.id)
@@ -364,6 +368,15 @@ module.exports = function (Y /* :any */) {
         }
         var t = this.initializedTypes[JSON.stringify(op.parent)]
 
+        // if parent is deleted, mark as gc'd and return
+        if (op.parent != null) {
+          var parentIsDeleted = yield* transaction.isDeleted(op.parent)
+          if (parentIsDeleted) {
+            yield* transaction.deleteList(op.id)
+            return
+          }
+        }
+
         // Delete if DS says this is actually deleted
         var opIsDeleted = yield* transaction.isDeleted(op.id)
         if (!op.deleted && opIsDeleted) {
@@ -375,8 +388,13 @@ module.exports = function (Y /* :any */) {
         }
 
         // notify parent, if it was instanciated as a custom type
-        if (t != null && !opIsDeleted) {
+        if (t != null) {
-          yield* t._changed(transaction, Y.utils.copyObject(op))
+          let o = Y.utils.copyObject(op)
+          if (opIsDeleted && !o.deleted) {
+            // op did not reflect the created delete op (happens when not using y-memory)
+            o.deleted = true
+          }
+          yield* t._changed(transaction, o)
         }
       }
     }

src/SpecHelper.js

@@ -139,9 +139,9 @@ g.applyRandomTransactionsWithGC = async(function * applyRandomTransactions (user
 
 g.garbageCollectAllUsers = async(function * garbageCollectAllUsers (users) {
   // gc two times because of the two gc phases (really collect everything)
+  yield wait(100)
   for (var i in users) {
-    yield users[i].db.garbageCollect()
+    yield users[i].db.emptyGarbageCollector()
-    yield users[i].db.garbageCollect()
   }
 })
 
@@ -182,6 +182,10 @@ g.compareAllUsers = async(function * compareAllUsers (users) {
   for (var uid = 0; uid < users.length; uid++) {
     var u = users[uid]
     u.db.requestTransaction(function * () {
+      var sv = yield* this.getStateVector()
+      for (var s of sv) {
+        yield* this.updateState(s.user)
+      }
       // compare deleted ops against deleteStore
       yield* this.os.iterate(this, null, null, function * (o) {
         if (o.deleted === true) {

src/Transaction.js

@@ -154,17 +154,21 @@ module.exports = function (Y/* :any */) {
     }
 
     * deleteList (start) {
-      if (this.store.y.connector.isSynced) {
+      while (start != null) {
-        while (start != null && this.store.y.connector.isSynced) {
+        start = yield* this.getOperation(start)
-          start = yield* this.getOperation(start)
+        if (!start.gc) {
           start.gc = true
+          start.deleted = true
           yield* this.setOperation(start)
-          // TODO: will always reset the parent..
+          yield* this.markDeleted(start.id, 1)
-          this.store.gc1.push(start.id)
+          if (start.opContent != null) {
-          start = start.right
+            yield* this.deleteOperation(start.opContent)
+          }
+          if (this.store.y.connector.isSynced) {
+            this.store.gc1.push(start.id)
+          }
         }
-      } else {
+        start = start.right
-        // TODO: when not possible??? do later in (gcWhenSynced)
       }
     }
 
@@ -199,19 +203,24 @@ module.exports = function (Y/* :any */) {
           if (target.start != null) {
             // TODO: don't do it like this .. -.-
             yield* this.deleteList(target.start)
-            yield* this.deleteList(target.id)
+            // yield* this.deleteList(target.id) -- do not gc itself because this may still get referenced
           }
           if (target.map != null) {
             for (var name in target.map) {
               yield* this.deleteList(target.map[name])
             }
             // TODO: here to..  (see above)
-            yield* this.deleteList(target.id)
+            // yield* this.deleteList(target.id) -- see above
           }
           if (target.opContent != null) {
             yield* this.deleteOperation(target.opContent)
             // target.opContent = null
           }
+          if (target.requires != null) {
+            for (var i = 0; i < target.requires.length; i++) {
+              yield* this.deleteOperation(target.requires[i])
+            }
+          }
         }
         var left
         if (target.left != null) {
@@ -377,9 +386,40 @@ module.exports = function (Y/* :any */) {
     */
     * garbageCollectAfterSync () {
       yield* this.os.iterate(this, null, null, function * (op) {
-        if (op.deleted && op.left != null) {
+        if (op.gc) {
-          var left = yield* this.getOperation(op.left)
+          this.store.gc1.push(op.id)
-          this.store.addToGarbageCollector(op, left)
+        } else {
+          if (op.parent != null) {
+            var parentDeleted = yield* this.isDeleted(op.parent)
+            if (parentDeleted) {
+              op.gc = true
+              if (!op.deleted) {
+                yield* this.markDeleted(op.id, 1)
+                op.deleted = true
+                if (op.opContent != null) {
+                  yield* this.deleteOperation(op.opContent)
+                  /*
+                  var opContent = yield* this.getOperation(op.opContent)
+                  opContent.gc = true
+                  yield* this.setOperation(opContent)
+                  this.store.gc1.push(opContent.id)
+                  */
+                }
+                if (op.requires != null) {
+                  for (var i = 0; i < op.requires.length; i++) {
+                    yield* this.deleteOperation(op.requires[i])
+                  }
+                }
+              }
+              yield* this.setOperation(op)
+              this.store.gc1.push(op.id)
+              return
+            }
+          }
+          if (op.deleted && op.left != null) {
+            var left = yield* this.getOperation(op.left)
+            this.store.addToGarbageCollector(op, left)
+          }
         }
       })
     }
@@ -405,6 +445,29 @@ module.exports = function (Y/* :any */) {
         }
         */
 
+        var deps = []
+        if (o.opContent != null) {
+          deps.push(o.opContent)
+        }
+        if (o.requires != null) {
+          deps = deps.concat(o.requires)
+        }
+        for (var i = 0; i < deps.length; i++) {
+          var dep = yield* this.getOperation(deps[i])
+          if (dep != null) {
+            if (!dep.deleted) {
+              yield* this.deleteOperation(dep.id)
+              dep = yield* this.getOperation(dep.id)
+            }
+            dep.gc = true
+            yield* this.setOperation(dep)
+            this.store.gc1.push(dep.id)
+          } else {
+            yield* this.markGarbageCollected(deps[i], 1)
+            yield* this.updateState(deps[i][0]) // TODO: unneccessary?
+          }
+        }
+
         // remove gc'd op from the left op, if it exists
         if (o.left != null) {
           var left = yield* this.getOperation(o.left)
@@ -487,18 +550,18 @@ module.exports = function (Y/* :any */) {
             // so we have to set right here
             yield* this.setOperation(right)
           }
-          // o may originate in another operation.
+        }
-          // Since o is deleted, we have to reset o.origin's `originOf` property
+        // o may originate in another operation.
-          if (o.origin != null) {
+        // Since o is deleted, we have to reset o.origin's `originOf` property
-            var origin = yield* this.getOperation(o.origin)
+        if (o.origin != null) {
-            origin.originOf = origin.originOf.filter(function (_id) {
+          var origin = yield* this.getOperation(o.origin)
-              return !Y.utils.compareIds(id, _id)
+          origin.originOf = origin.originOf.filter(function (_id) {
-            })
+            return !Y.utils.compareIds(id, _id)
-            yield* this.setOperation(origin)
+          })
-          }
+          yield* this.setOperation(origin)
         }
         var parent
-        if (o.parent != null){ 
+        if (o.parent != null) {
           parent = yield* this.getOperation(o.parent)
         }
         // remove gc'd op from parent, if it exists
@@ -535,6 +598,18 @@ module.exports = function (Y/* :any */) {
         state.clock = Math.max(state.clock, n.id[1] + n.len)
       }
     }
+    * updateState (user) {
+      var state = yield* this.getState(user)
+      yield* this.checkDeleteStoreForState(state)
+      var o = yield* this.getOperation([user, state.clock])
+      while (o != null && o.id[1] === state.clock && user === o.id[0]) {
+        // either its a new operation (1. case), or it is an operation that was deleted, but is not yet in the OS
+        state.clock++
+        yield* this.checkDeleteStoreForState(state)
+        o = yield* this.os.findNext(o.id)
+      }
+      yield* this.setState(state)
+    }
     /*
       apply a delete set in order to get
       the state of the supplied ds