improved new sync idea (save gcs in DS)

src/Connector.js

@@ -148,13 +148,10 @@ class AbstractConnector {
       let conn = this
       this.y.db.requestTransaction(function *() {
         var currentStateSet = yield* this.getStateSet()
-        var dels = yield* this.getOpsFromDeleteSet(m.deleteSet)
+        yield* this.applyDeleteSet(m.deleteSet)
-        for (var i in dels) {
-          // TODO: no longer get delete ops (just get the ids..)!
-          yield* Y.Struct.Delete.delete.call(this, dels[i].target)
-        }
 
         var ops = yield* this.getOperations(m.stateSet)
+        ops = JSON.parse(JSON.stringify(ops)) // TODO: don't do something like that!!
         conn.send(sender, {
           type: 'sync step 2',
           os: ops,
@@ -178,17 +175,15 @@ class AbstractConnector {
         }
       })
     } else if (m.type === 'sync step 2') {
-      this.y.db.apply(m.os)
       let conn = this
       var broadcastHB = !this.broadcastedHB
       this.broadcastedHB = true
       this.y.db.requestTransaction(function * () {
-        var dels = yield* this.getOpsFromDeleteSet(m.deleteSet)
+        yield* this.applyDeleteSet(m.deleteSet)
-        for (var i in dels) {
-          yield* Y.Struct.Delete.delete.call(this, dels[i].target)
-        }
-        var ops = yield* this.getOperations(m.stateSet)
         this.store.apply(m.os)
+      })
+      this.y.db.requestTransaction(function * () {
+        var ops = yield* this.getOperations(m.stateSet)
         if (ops.length > 0) {
           m = {
             type: 'update',

src/OperationStore.js

@@ -23,7 +23,7 @@
         "userY": ...
       }
   * isDeleted(id)
-  * getOpsFromDeleteSet(ds) -- TODO: just call Struct.Delete.delete(id) here
+  * getOpsFromDeleteSet(ds) -- TODO: just call this.deleteOperation(id) here
     - get a set of deletions that need to be applied in order to get to
       achieve the state of the supplied ds
   * setOperation(op)
@@ -107,6 +107,97 @@ class AbstractTransaction {
       })
     }
   }
+  /*
+    Delete an operation from the OS, and add it to the GC, if necessary.
+
+    Rulez:
+    * The most left element in a list must not be deleted.
+      => There is at least one element in the list
+    * When an operation o is deleted, then it checks if its right operation
+      can be gc'd (iff it's deleted)
+  */
+  * deleteOperation (targetId) {
+    var target = yield* this.getOperation(targetId)
+
+    if (target == null || !target.deleted) {
+      this.ds.markDeleted(targetId)
+      var state = yield* this.getState(targetId[0])
+      if (state.clock === targetId[1]) {
+        yield* this.checkDeleteStoreForState(state)
+        yield* this.setState(state)
+      }
+    }
+
+    if (target != null && target.gc == null) {
+      if (!target.deleted) {
+        // set deleted & notify type
+        target.deleted = true
+        var type = this.store.initializedTypes[JSON.stringify(target.parent)]
+        if (type != null) {
+          yield* type._changed(this, {
+            struct: 'Delete',
+            target: targetId
+          })
+        }
+      }
+      var left = target.left != null ? yield* this.getOperation(target.left) : null
+      var right = target.right != null ? yield* this.getOperation(target.right) : null
+
+      this.store.addToGarbageCollector(target, left, right)
+
+      // set here because it was deleted and/or gc'd
+      yield* this.setOperation(target)
+
+      if (
+        left != null &&
+        left.left != null &&
+        this.store.addToGarbageCollector(left, yield* this.getOperation(left.left), target)
+      ) {
+        yield* this.setOperation(left)
+      }
+
+      if (
+        right != null &&
+        right.right != null &&
+        this.store.addToGarbageCollector(right, target, yield* this.getOperation(right.right))
+      ) {
+        yield* this.setOperation(right)
+      }
+    }
+  }
+  * garbageCollectOperation (id) {
+    var o = yield* this.getOperation(id)
+    if (!o.deleted) {
+      yield* this.deleteOperation(id)
+      o = yield* this.getOperation(id)
+    }
+
+    if (o.left != null) {
+      var left = yield* this.getOperation(o.left)
+      left.right = o.right
+      yield* this.setOperation(left)
+    }
+    if (o.right != null) {
+      var right = yield* this.getOperation(o.right)
+      right.left = o.left
+      yield* this.setOperation(right)
+    }
+    var parent = yield* this.getOperation(o.parent)
+    var setParent = false
+    if (Y.utils.compareIds(parent.start, o.id)) {
+      setParent = true
+      parent.start = o.right
+    }
+    if (Y.utils.compareIds(parent.end, o.id)) {
+      setParent = true
+      parent.end = o.left
+    }
+    if (setParent) {
+      yield* this.setOperation(parent)
+    }
+    yield* this.removeOperation(o.id)
+    yield* this.ds.markGarbageCollected(o.id)
+  }
 }
 Y.AbstractTransaction = AbstractTransaction
 
@@ -156,32 +247,7 @@ class AbstractOperationStore {
         os.requestTransaction(function * () {
           for (var i in os.gc2) {
             var oid = os.gc2[i]
-            var o = yield* this.getOperation(oid)
+            yield* this.garbageCollectOperation(oid)
-
-            if (o.left != null) {
-              var left = yield* this.getOperation(o.left)
-              left.right = o.right
-              yield* this.setOperation(left)
-            }
-            if (o.right != null) {
-              var right = yield* this.getOperation(o.right)
-              right.left = o.left
-              yield* this.setOperation(right)
-            }
-            var parent = yield* this.getOperation(o.parent)
-            var setParent = false
-            if (Y.utils.compareIds(parent.start, o.id)) {
-              setParent = true
-              parent.start = o.right
-            }
-            if (Y.utils.compareIds(parent.end, o.id)) {
-              setParent = true
-              parent.end = o.left
-            }
-            if (setParent) {
-              yield* this.setOperation(parent)
-            }
-            yield* this.removeOperation(o.id)
           }
           os.gc2 = os.gc1
           os.gc1 = []
@@ -276,8 +342,12 @@ class AbstractOperationStore {
     for (var key in ops) {
       var o = ops[key]
       if (o.gc == null) { // TODO: why do i get the same op twice?
+        if (o.deleted == null) {
           var required = Y.Struct[o.struct].requiredOps(o)
           this.whenOperationsExist(required, o)
+        } else {
+          throw new Error('Ops must not contain deleted field!')
+        }
       } else {
         throw new Error("Must not receive gc'd ops!")
       }

src/OperationStores/Memory.js

@@ -9,8 +9,13 @@ class DeleteStore extends Y.utils.RBTree {
     var n = this.findNodeWithUpperBound(id)
     return n !== null && n.val.id[0] === id[0] && id[1] < n.val.id[1] + n.val.len
   }
-  garbageCollect (id) {
+  /*
-    var n = this.delete(id)
+    Mark an operation as deleted&gc'd
+
+    returns the delete node
+  */
+  * markGarbageCollected (id) {
+    var n = this.markDeleted(id)
     if (!n.val.gc) {
       if (n.val.id[1] < id[1]) {
         // un-extend left
@@ -20,7 +25,7 @@ class DeleteStore extends Y.utils.RBTree {
       }
       if (id[1] < n.val.id[1] + n.val.len - 1) {
         // un-extend right
-        this.add({id: id, len: n.val.len - 1, gc: false})
+        this.add({id: [id[0], id[1] + 1], len: n.val.len - 1, gc: false})
         n.val.len = 1
       }
       // set gc'd
@@ -39,13 +44,14 @@ class DeleteStore extends Y.utils.RBTree {
         super.delete(next.val.id)
       }
     }
+    return n
   }
   /*
     Mark an operation as deleted.
 
     returns the delete node
   */
-  delete (id) {
+  markDeleted (id) {
     var n = this.findNodeWithUpperBound(id)
     if (n != null && n.val.id[0] === id[0]) {
       if (n.val.id[1] <= id[1] && id[1] < n.val.id[1] + n.val.len) {
@@ -70,80 +76,25 @@ class DeleteStore extends Y.utils.RBTree {
     }
     return n
   }
-  // a DeleteSet (ds) describes all the deleted ops in the OS
+  /*
+    A DeleteSet (ds) describes all the deleted ops in the OS
+  */
   toDeleteSet () {
     var ds = {}
     this.iterate(null, null, function (n) {
       var user = n.id[0]
       var counter = n.id[1]
       var len = n.len
+      var gc = n.gc
       var dv = ds[user]
       if (dv === void 0) {
         dv = []
         ds[user] = dv
       }
-      dv.push([counter, len])
+      dv.push([counter, len, gc])
     })
     return ds
   }
-  // returns a set of deletions that need to be applied in order to get to
-  // the state of the supplied ds
-  getDeletions (ds) {
-    var deletions = []
-    function createDeletions (user, start, len) {
-      for (var c = start; c < start + len; c++) {
-        deletions.push({
-          target: [user, c],
-          struct: 'Delete'
-        })
-      }
-    }
-    for (var user in ds) {
-      var dv = ds[user]
-      var pos = 0
-      var d = dv[pos]
-      this.iterate([user, 0], [user, Number.MAX_VALUE], function (n) {
-        // cases:
-        // 1. d deletes something to the right of n
-        //  => go to next n (break)
-        // 2. d deletes something to the left of n
-        //  => create deletions
-        //  => reset d accordingly
-        //  *)=> if d doesn't delete anything anymore, go to next d (continue)
-        // 3. not 2) and d deletes something that also n deletes
-        //  => reset d so that it doesn't contain n's deletion
-        //  *)=> if d does not delete anything anymore, go to next d (continue)
-        while (d != null) {
-          var diff // describe the diff of length in 1) and 2)
-          if (n.id[1] + n.len <= d[0]) {
-            // 1)
-            break
-          } else if (d[0] < n.id[1]) {
-            // 2)
-            // delete maximum the len of d
-            // else delete as much as possible
-            diff = Math.min(n.id[1] - d[0], d[1])
-            createDeletions(user, d[0], diff)
-          } else {
-            // 3)
-            diff = n.id[1] + n.len - d[0] // never null (see 1)
-          }
-          if (d[1] <= diff) {
-            // d doesn't delete anything anymore
-            d = dv[++pos]
-          } else {
-            d[0] = d[0] + diff // reset pos
-            d[1] = d[1] - diff // reset length
-          }
-        }
-      })
-      for (; pos < dv.length; pos++) {
-        d = dv[pos]
-        createDeletions(user, d[0], d[1])
-      }
-    }
-    return deletions
-  }
 }
 
 Y.utils.DeleteStore = DeleteStore
@@ -166,12 +117,82 @@ Y.Memory = (function () {
     * getDeleteSet (id) {
       return this.ds.toDeleteSet(id)
     }
+    /*
+      apply a delete set in order to get
+      the state of the supplied ds
+    */
+    * applyDeleteSet (ds) {
+      var deletions = []
+      function createDeletions (user, start, len, gc) {
+        for (var c = start; c < start + len; c++) {
+          deletions.push([user, c, gc])
+        }
+      }
+      for (var user in ds) {
+        var dv = ds[user]
+        var pos = 0
+        var d = dv[pos]
+        this.ds.iterate([user, 0], [user, Number.MAX_VALUE], function (n) {
+          // cases:
+          // 1. d deletes something to the right of n
+          //  => go to next n (break)
+          // 2. d deletes something to the left of n
+          //  => create deletions
+          //  => reset d accordingly
+          //  *)=> if d doesn't delete anything anymore, go to next d (continue)
+          // 3. not 2) and d deletes something that also n deletes
+          //  => reset d so that it doesn't contain n's deletion
+          //  *)=> if d does not delete anything anymore, go to next d (continue)
+          while (d != null) {
+            var diff = 0 // describe the diff of length in 1) and 2)
+            if (n.id[1] + n.len <= d[0]) {
+              // 1)
+              break
+            } else if (d[0] < n.id[1]) {
+              // 2)
+              // delete maximum the len of d
+              // else delete as much as possible
+              diff = Math.min(n.id[1] - d[0], d[1])
+              createDeletions(user, d[0], diff, d[2])
+            } else {
+              // 3)
+              diff = n.id[1] + n.len - d[0] // never null (see 1)
+              if (d[2] && !n.gc) {
+                // d marks as gc'd but n does not
+                // then delete either way
+                createDeletions(user, d[0], diff, d[2])
+              }
+            }
+            if (d[1] <= diff) {
+              // d doesn't delete anything anymore
+              d = dv[++pos]
+            } else {
+              d[0] = d[0] + diff // reset pos
+              d[1] = d[1] - diff // reset length
+            }
+          }
+        })
+        // for the rest.. just apply it
+        for (; pos < dv.length; pos++) {
+          d = dv[pos]
+          createDeletions(user, d[0], d[1], d[2])
+        }
+      }
+      for (var i in deletions) {
+        var del = deletions[i]
+        var id = [del[0], del[1]]
+        if (del[2]) {
+          // gc
+          yield* this.garbageCollectOperation(id)
+        } else {
+          // delete
+          yield* this.deleteOperation(id)
+        }
+      }
+    }
     * isDeleted (id) {
       return this.ds.isDeleted(id)
     }
-    * getOpsFromDeleteSet (ds) {
-      return this.ds.getDeletions(ds)
-    }
     * setOperation (op) {
       // TODO: you can remove this step! probs..
       var n = this.os.findNode(op.id)

src/OperationStores/Memory.spec.js

@@ -8,30 +8,22 @@ describe('Memory', function () {
       ds = new Y.utils.DeleteStore()
     })
     it('Deleted operation is deleted', function () {
-      ds.delete(['u1', 10])
+      ds.markDeleted(['u1', 10])
       expect(ds.isDeleted(['u1', 10])).toBeTruthy()
-      expect(ds.toDeleteSet()).toEqual({'u1': [[10, 1]]})
+      expect(ds.toDeleteSet()).toEqual({'u1': [[10, 1, false]]})
     })
     it('Deleted operation extends other deleted operation', function () {
-      ds.delete(['u1', 10])
+      ds.markDeleted(['u1', 10])
-      ds.delete(['u1', 11])
+      ds.markDeleted(['u1', 11])
       expect(ds.isDeleted(['u1', 10])).toBeTruthy()
       expect(ds.isDeleted(['u1', 11])).toBeTruthy()
-      expect(ds.toDeleteSet()).toEqual({'u1': [[10, 2]]})
+      expect(ds.toDeleteSet()).toEqual({'u1': [[10, 2, false]]})
     })
     it('Deleted operation extends other deleted operation', function () {
-      ds.delete(['0', 3])
+      ds.markDeleted(['0', 3])
-      ds.delete(['0', 4])
+      ds.markDeleted(['0', 4])
-      ds.delete(['0', 2])
+      ds.markDeleted(['0', 2])
-      expect(ds.toDeleteSet()).toEqual({'0': [[2, 3]]})
+      expect(ds.toDeleteSet()).toEqual({'0': [[2, 3, false]]})
-    })
-    it('Creates operations', function () {
-      var dels = ds.getDeletions({5: [[4, 1]]})
-      expect(dels.length === 1).toBeTruthy()
-      expect(dels[0]).toEqual({
-        struct: 'Delete',
-        target: ['5', 4]
-      })
     })
   })
 })

src/Struct.js

@@ -35,66 +35,8 @@ var Struct = {
     requiredOps: function (op) {
       return [] // [op.target]
     },
-    /*
-      Delete an operation from the OS, and add it to the GC, if necessary.
-
-      Rulez:
-      * The most left element in a list must not be deleted.
-        => There is at least one element in the list
-      * When an operation o is deleted, then it checks if its right operation
-        can be gc'd (iff it's deleted)
-    */
-    delete: function * (targetId) {
-      var target = yield* this.getOperation(targetId)
-
-      if (target == null || !target.deleted) {
-        this.ds.delete(targetId)
-        var state = yield* this.getState(targetId[0])
-        if (state.clock === targetId[1]) {
-          yield* this.checkDeleteStoreForState(state)
-          yield* this.setState(state)
-        }
-      }
-
-      if (target != null && target.gc == null) {
-        if (!target.deleted) {
-          // set deleted & notify type
-          target.deleted = true
-          var type = this.store.initializedTypes[JSON.stringify(target.parent)]
-          if (type != null) {
-            yield* type._changed(this, {
-              struct: 'Delete',
-              target: targetId
-            })
-          }
-        }
-        var left = target.left != null ? yield* this.getOperation(target.left) : null
-        var right = target.right != null ? yield* this.getOperation(target.right) : null
-
-        this.store.addToGarbageCollector(target, left, right)
-
-        // set here because it was deleted and/or gc'd
-        yield* this.setOperation(target)
-
-        if (
-          left != null &&
-          left.left != null &&
-          this.store.addToGarbageCollector(left, yield* this.getOperation(left.left), target)
-        ) {
-          yield* this.setOperation(left)
-        }
-
-        if (
-          right != null &&
-          right.right != null &&
-          this.store.addToGarbageCollector(right, target, yield* this.getOperation(right.right))
-        ) {
-          yield* this.setOperation(right)
-        }
-      }
-    },
     execute: function * (op) {
-      yield* Struct.Delete.delete.call(this, op.target)
+      yield* this.deleteOperation(op.target)
     }
   },
   Insert: {

src/Types/Array.spec.js

@@ -244,7 +244,6 @@ describe('Array Type', function () {
       }
       yield applyRandomTransactions(this.users, this.arrays, randomArrayTransactions, numberOfYArrayTests)
       yield flushAll()
-      yield garbageCollectAllUsers(this.users)
       yield compareArrayValues(this.arrays)
       yield compareAllUsers(this.users)
       done()