From 72bd0d9c3a0d7e12ed207def48b970dd8ac6031e Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 8 Dec 2015 16:26:55 +0100 Subject: [PATCH 01/43] update map type --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba105cbf..86620de0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ You can create you own shared types easily. Therefore, you can take matters into | Name | Description | |----------|-------------------| -| map | Add, update, and remove properties of an object. Included in Yjs| +|[map](https://github.com/y-js/y-map) | Add, update, and remove properties of an object. Included in Yjs| |[array](https://github.com/y-js/y-array) | A shared linked list implementation | |[selections](https://github.com/y-js/y-selections) | Manages selections on types that use linear structures (e.g. the y-array type). Select a range of elements, and assign meaning to them.| |[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects| From fbbfa9fd476bf917e01ee1c94158af973bef53d0 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 9 Dec 2015 18:40:10 +0100 Subject: [PATCH 02/43] added example --- dist | 2 +- package.json | 2 +- src/y.js | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dist b/dist index 96f8f77d..625a6463 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 96f8f77dc4d90cd7fdc09ecb354b8ce7ae270796 +Subproject commit 625a64635aa47b18e1533774cf707899fb032051 diff --git a/package.json b/package.json index ecaadbe3..f947b6e3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.6.42", + "version": "0.7.1", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "y.js", "scripts": { diff --git a/src/y.js b/src/y.js index 90c6ed18..36a813da 100644 --- a/src/y.js +++ b/src/y.js @@ -11,6 +11,7 @@ require('./Connectors/Test.js')(Y) var requiringModules = {} module.exports = Y +Y.requiringModules = requiringModules Y.extend = function (name, value) { Y[name] = value From 98d87cb26d3e8b6751ce9aa0b7983aba91b2f1b9 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 18 Dec 2015 16:34:21 +0100 Subject: [PATCH 03/43] update --- dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist b/dist index 625a6463..8ba1e4ce 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 625a64635aa47b18e1533774cf707899fb032051 +Subproject commit 8ba1e4ce27a75bb025a5478723ea57f7f437ccf0 From bdf290adb229bd1fbf8d2bbccca2bbc31d96e427 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 30 Dec 2015 16:37:35 +0100 Subject: [PATCH 04/43] added safety to setUserId (when called twice) --- README.md | 1 + dist | 2 +- src/Connector.js | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 86620de0..a804697f 100644 --- a/README.md +++ b/README.md @@ -144,3 +144,4 @@ Yjs is licensed under the [MIT License](./LICENSE.txt). [ShareJs]: https://github.com/share/ShareJS [OpenCoweb]: https://github.com/opencoweb/coweb/wiki + diff --git a/dist b/dist index 8ba1e4ce..ab6a193e 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 8ba1e4ce27a75bb025a5478723ea57f7f437ccf0 +Subproject commit ab6a193ec6ec799bdb77fa23eeec1f05d5b910d6 diff --git a/src/Connector.js b/src/Connector.js index 0a1bb4e1..bc55ccf2 100644 --- a/src/Connector.js +++ b/src/Connector.js @@ -63,8 +63,12 @@ module.exports = function (Y/* :any */) { return this.y.db.stopGarbageCollector() } setUserId (userId) { - this.userId = userId - return this.y.db.setUserId(userId) + if (this.userId == null) { + this.userId = userId + return this.y.db.setUserId(userId) + } else { + return null + } } onUserEvent (f) { this.userEventListeners.push(f) From 8cfc9d41c3b3b9ca3d3d4594191cdcf9ba75d139 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 9 Jan 2016 04:17:23 +0100 Subject: [PATCH 05/43] Made compatible with windows --- gulpfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gulpfile.js b/gulpfile.js index 7ea185a7..b4eee461 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -65,7 +65,7 @@ gulp.task('dev:examples', ['watch:dist'], function () { gulp.src(distfiles) .pipe($.watch(distfiles)) .pipe($.rename(function (path) { - var dir = path.dirname.split('/')[0] + var dir = path.dirname.split(/[\\\/]/)[0] console.log(JSON.stringify(path)) path.dirname = dir === '.' ? 'yjs' : dir })) From 579fd524550516e435c82a6d45f3f2c762109492 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 9 Jan 2016 21:08:02 +0100 Subject: [PATCH 06/43] publish v0.7.3 --- .vscode/launch.json | 2 +- dist | 2 +- package.json | 4 ++-- src/Transaction.js | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index f5f0d3e1..ad0153dd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ "request": "launch", "program": "node_modules/gulp/bin/gulp.js", "stopOnEntry": false, - "args": ["test"], + "args": ["dev:examples"], "cwd": ".", "runtimeExecutable": null, "runtimeArgs": [ diff --git a/dist b/dist index ab6a193e..4cb0f2b5 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit ab6a193ec6ec799bdb77fa23eeec1f05d5b910d6 +Subproject commit 4cb0f2b5b9293b521db173f01263e6f9d1a8e6d1 diff --git a/package.json b/package.json index f947b6e3..e9265656 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "yjs", - "version": "0.7.1", + "version": "0.7.3", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", - "main": "y.js", + "main": "./src/y.js", "scripts": { "test": "node --harmony ./node_modules/.bin/gulp test", "lint": "./node_modules/.bin/standard" diff --git a/src/Transaction.js b/src/Transaction.js index 68533c1e..b3837fbe 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -397,6 +397,7 @@ module.exports = function (Y/* :any */) { if (i.right == null) { break } else { + ids.push(i.id) i = yield* this.getOperation(i.right) } } From fd6a28eb2573dceb6090b7e7da9579c1bd0a4bf6 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 11 Jan 2016 15:47:24 +0100 Subject: [PATCH 07/43] Release 0.7.5 --- dist | 2 +- gulpfile.helper.js | 10 ++++++---- package.json | 5 ++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dist b/dist index 4cb0f2b5..15a472df 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 4cb0f2b5b9293b521db173f01263e6f9d1a8e6d1 +Subproject commit 15a472df44c95844df894006919193186b80004d diff --git a/gulpfile.helper.js b/gulpfile.helper.js index f4f15243..2e1ea71f 100644 --- a/gulpfile.helper.js +++ b/gulpfile.helper.js @@ -121,13 +121,14 @@ module.exports = function (gulp, helperOptions) { gulp.task('updateSubmodule', function () { return gulp.src('./package.json', {read: false}) .pipe($.shell([ - 'git submodule update --init' + 'git submodule update --init', + 'cd dist && git pull origin dist' ])) }) gulp.task('bump', function () { var bumptype - return gulp.src(['./package.json', './dist/package.json', './dist/bower.json'], {base: '.'}) + return gulp.src(['./package.json', './bower.json', './dist/bower.json'], {base: '.'}) .pipe($.prompt.prompt({ type: 'checkbox', name: 'bump', @@ -145,14 +146,14 @@ module.exports = function (gulp, helperOptions) { gulp.task('publish', function (cb) { /* TODO: include 'test',*/ - runSequence(['updateSubmodule', 'dist'], 'bump', function () { + runSequence('updateSubmodule', 'dist', 'bump', function () { return gulp.src('./package.json', {read: false}) .pipe($.prompt.confirm({ message: 'Are you sure you want to publish this release?', default: false })) .pipe($.shell([ - 'cp ./README.md ./dist/', + // 'cp README.md dist', 'standard', 'echo "Deploying version <%= getVersion(file.path) %>"', 'git pull', @@ -163,6 +164,7 @@ module.exports = function (gulp, helperOptions) { 'cd ./dist/ && git push origin --tags', 'git commit -am "Release <%= getVersion(file.path) %>" -n', 'git push', + 'npm publish', 'echo Finished <%= callback() %>' ], { templateData: { diff --git a/package.json b/package.json index e9265656..25182b31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.7.3", + "version": "0.7.6", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { @@ -72,5 +72,8 @@ "standard": "^5.2.2", "vinyl-buffer": "^1.0.0", "vinyl-source-stream": "^1.1.0" + }, + "dependencies": { + "babel-eslint": "^5.0.0-beta6" } } From a33d0bf7bc36a57da1b425f4b4b3cd865694cb1b Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 11 Jan 2016 15:48:10 +0100 Subject: [PATCH 08/43] Release 0.7.6 --- dist | 2 +- package.json | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dist b/dist index 15a472df..ff006c92 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 15a472df44c95844df894006919193186b80004d +Subproject commit ff006c92d70c0afca6c7dde3aa72b176b1b3ec0b diff --git a/package.json b/package.json index 25182b31..a1152759 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.7.6", + "version": "0.7.7", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { @@ -71,9 +71,8 @@ "run-sequence": "^1.1.4", "standard": "^5.2.2", "vinyl-buffer": "^1.0.0", - "vinyl-source-stream": "^1.1.0" - }, - "dependencies": { + "vinyl-source-stream": "^1.1.0", "babel-eslint": "^5.0.0-beta6" - } + }, + "dependencies": {} } From 51a834d6c95c68f9fe5730f935c6d8485e4c6fe3 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 00:00:41 +0100 Subject: [PATCH 09/43] Implemente a new sync procedure that is optimal with respect to big O notation (there is probably a way to reduce it by a factor of 1/2) --- package.json | 2 +- src/Transaction.js | 114 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 88 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index a1152759..1a4e4282 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.7.7", + "version": "0.8.0", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { diff --git a/src/Transaction.js b/src/Transaction.js index b3837fbe..cd078e70 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -626,34 +626,8 @@ module.exports = function (Y/* :any */) { }) return ss } - * getOperations (startSS) { - // TODO: use bounds here! - if (startSS == null) { - startSS = {} - } - var ops = [] - - var endSV = yield* this.getStateVector() - for (var endState of endSV) { - var user = endState.user - if (user === '_') { - continue - } - var startPos = startSS[user] || 0 - - yield* this.os.iterate(this, [user, startPos], [user, Number.MAX_VALUE], function * (op) { - ops.push(op) - }) - } - var res = [] - for (var op of ops) { - var o = yield* this.makeOperationReady(startSS, op) - res.push(o) - } - return res - } /* - Here, we make op executable for the receiving user. + Here, we make all missing operations executable for the receiving user. Notes: startSS: denotes to the SV that the remote user sent @@ -688,7 +662,92 @@ module.exports = function (Y/* :any */) { (startSS or currSS.. ?) -> Could be necessary when I turn GC again. -> Is a bad(ish) idea because it requires more computation + + What we do: + * Iterate over all missing operations. + * When there is an operation, where the right op is known, send this op all missing ops to the left to the user + * I explained above what we have to do with each operation. Here is how we do it efficiently: + 1. Go to the left until you find either op.origin, or a known operation (let o denote current operation in the iteration) + 2. Found a known operation -> set op.left = o, and send it to the user. stop + 3. Found o = op.origin -> set op.left = op.origin, and send it to the user. start again from 1. (set op = o) + 4. Found some o -> set o.right = op, o.left = o.origin, send it to the user, continue */ + * getOperations (startSS) { + // TODO: use bounds here! + if (startSS == null) { + startSS = {} + } + var send = [] + + var endSV = yield* this.getStateVector() + for (var endState of endSV) { + var user = endState.user + if (user === '_') { + continue + } + var startPos = startSS[user] || 0 + + yield* this.os.iterate(this, [user, startPos], [user, Number.MAX_VALUE], function * (op) { + op = Y.Struct[op.struct].encode(op) + if (op.struct !== 'Insert') { + send.push(op) + } else if (op.right == null || op.right[1] < (startSS[op.right[0]] || 0)) { + // case 1. op.right is known + var o = op + // Remember: ? + // -> set op.right + // 1. to the first operation that is known (according to startSS) + // 2. or to the first operation that has an origin that is not to the + // right of op. + // For this we maintain a list of ops which origins are not found yet. + var missing_origins = [op] + var newright = op.right + while (true) { + if (o.left == null) { + op.left = null + send.push(op) + if (!Y.utils.compareIds(o.id, op.id)) { + o = Y.Struct[op.struct].encode(o) + o.right = missing_origins[missing_origins.length - 1].id + send.push(o) + } + break + } + o = yield* this.getOperation(o.left) + // we set another o, check if we can reduce $missing_origins + while (missing_origins.length > 0 && Y.utils.compareIds(missing_origins[missing_origins.length - 1].origin, o.id)) { + missing_origins.pop() + } + if (o.id[1] < (startSS[o.id[0]] || 0)) { + // case 2. o is known + op.left = o.id + send.push(op) + break + } else if (Y.utils.compareIds(o.id, op.origin)) { + // case 3. o is op.origin + op.left = op.origin + send.push(op) + op = Y.Struct[op.struct].encode(o) + op.right = newright + if (missing_origins.length > 0) { + console.log('This should not happen .. :( please report this') + } + missing_origins = [op] + } else { + // case 4. send o, continue to find op.origin + var s = Y.Struct[op.struct].encode(o) + s.right = missing_origins[missing_origins.length - 1].id + s.left = s.origin + send.push(s) + missing_origins.push(o) + } + } + } + }) + } + return send + } + /* this is what we used before.. use this as a reference.. * makeOperationReady (startSS, op) { op = Y.Struct[op.struct].encode(op) op = Y.utils.copyObject(op) @@ -712,6 +771,7 @@ module.exports = function (Y/* :any */) { op.left = op.origin return op } + */ } Y.Transaction = TransactionInterface } From 7e046e075377b0229ab97439d8e786e12902133f Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 00:02:12 +0100 Subject: [PATCH 10/43] Release 0.8.0 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index ff006c92..0ec83aa4 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit ff006c92d70c0afca6c7dde3aa72b176b1b3ec0b +Subproject commit 0ec83aa4310ae376d1f08e58cb838848f6aae335 diff --git a/package.json b/package.json index 1a4e4282..aa5cf327 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.0", + "version": "0.8.1", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From dd279bccf7c5e1131e8ca58ce03d64474e4951aa Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 00:03:43 +0100 Subject: [PATCH 11/43] Release 0.8.1 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index 0ec83aa4..9902da47 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 0ec83aa4310ae376d1f08e58cb838848f6aae335 +Subproject commit 9902da470b472ba87ef09432e599a721f92c9531 diff --git a/package.json b/package.json index aa5cf327..8b7b88fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.1", + "version": "0.8.2", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From ece8268e44ec937bf35233859a44f262cd8646f7 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 03:10:58 +0100 Subject: [PATCH 12/43] Release 0.8.2 --- dist | 2 +- package.json | 2 +- src/Transaction.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dist b/dist index 9902da47..a77eb392 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 9902da470b472ba87ef09432e599a721f92c9531 +Subproject commit a77eb39218a5815854daa07373f7f4597c3b9cb9 diff --git a/package.json b/package.json index 8b7b88fc..e9aaf134 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.2", + "version": "0.8.3", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { diff --git a/src/Transaction.js b/src/Transaction.js index cd078e70..0d0b609d 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -745,7 +745,7 @@ module.exports = function (Y/* :any */) { } }) } - return send + return send.reverse() } /* this is what we used before.. use this as a reference.. * makeOperationReady (startSS, op) { From 102555a3b085e9e4f702b7ff722a4c969ad26d23 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 03:46:55 +0100 Subject: [PATCH 13/43] Release 0.8.3 --- dist | 2 +- package.json | 2 +- src/y.js | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dist b/dist index a77eb392..3b8e148d 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit a77eb39218a5815854daa07373f7f4597c3b9cb9 +Subproject commit 3b8e148d8fee752b6318df2b940316c77936b812 diff --git a/package.json b/package.json index e9aaf134..f5583f3c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.3", + "version": "0.8.4", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { diff --git a/src/y.js b/src/y.js index 36a813da..7f7d2485 100644 --- a/src/y.js +++ b/src/y.js @@ -93,8 +93,9 @@ function Y (opts/* :YOptions */) /* :Promise */ { Y.sourceDir = opts.sourceDir return Y.requestModules(modules).then(function () { return new Promise(function (resolve) { - var yconfig = new YConfig(opts, function () { - yconfig.db.whenUserIdSet(function () { + var yconfig = new YConfig(opts) + yconfig.db.whenUserIdSet(function () { + yconfig.init(function () { resolve(yconfig) }) }) @@ -111,6 +112,10 @@ class YConfig { constructor (opts, callback) { this.db = new Y[opts.db.name](this, opts.db) this.connector = new Y[opts.connector.name](this, opts.connector) + this.options = opts + } + init (callback) { + var opts = this.options var share = {} this.share = share this.db.requestTransaction(function * requestTransaction () { From a5f55359c32c62d4ed69c0147bfcb04a8061a019 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 17:57:06 +0100 Subject: [PATCH 14/43] improve data exchange performance --- src/Connector.js | 42 ++++++++++++++++++++++++++++++++---------- src/Transaction.js | 15 +++------------ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/Connector.js b/src/Connector.js index bc55ccf2..4d537ede 100644 --- a/src/Connector.js +++ b/src/Connector.js @@ -50,6 +50,7 @@ module.exports = function (Y/* :any */) { this.debug = opts.debug === true this.broadcastedHB = false this.syncStep2 = Promise.resolve() + this.broadcastOpBuffer = [] } reconnect () { } @@ -166,6 +167,31 @@ module.exports = function (Y/* :any */) { console.log(`send ${this.userId} -> ${uid}: ${message.type}`, message) // eslint-disable-line } } + /* + Buffer operations, and broadcast them when ready. + */ + broadcastOps (ops) { + var self = this + function broadcastOperations () { + if (self.broadcastOpBuffer.length > 0) { + self.broadcast({ + type: 'update', + ops: self.broadcastOpBuffer + }) + self.broadcastOpBuffer = [] + } + } + if (this.broadcastOpBuffer.length === 0) { + this.broadcastOpBuffer = ops + if (this.y.db.transactionInProgress) { + this.y.db.whenTransactionsFinished().then(broadcastOperations) + } else { + setTimeout(broadcastOperations, 0) + } + } else { + this.broadcastOpBuffer = this.broadcastOpBuffer.concat(ops) + } + } /* You received a raw message, and you know that it is intended for Yjs. Then call this function. */ @@ -226,15 +252,14 @@ module.exports = function (Y/* :any */) { db.requestTransaction(function * () { var ops = yield* this.getOperations(m.stateSet) if (ops.length > 0) { - var update /* :MessageUpdate */ = { - type: 'update', - ops: ops - } if (!broadcastHB) { // TODO: consider to broadcast here.. - conn.send(sender, update) + conn.send(sender, { + type: 'update', + ops: ops + }) } else { // broadcast only once! - conn.broadcast(update) + conn.broadcastOps(ops) } } defer.resolve() @@ -256,10 +281,7 @@ module.exports = function (Y/* :any */) { return o.struct === 'Delete' }) if (delops.length > 0) { - this.broadcast({ - type: 'update', - ops: delops - }) + this.broadcastOps(delops) } } this.y.db.apply(message.ops) diff --git a/src/Transaction.js b/src/Transaction.js index 0d0b609d..da82223c 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -123,10 +123,7 @@ module.exports = function (Y/* :any */) { } if (!this.store.y.connector.isDisconnected() && send.length > 0) { // TODO: && !this.store.forwardAppliedOperations (but then i don't send delete ops) // is connected, and this is not going to be send in addOperation - this.store.y.connector.broadcast({ - type: 'update', - ops: send - }) + this.store.y.connector.broadcastOps(send) } } @@ -522,10 +519,7 @@ module.exports = function (Y/* :any */) { var ops = deletions.map(function (d) { return {struct: 'Delete', target: [d[0], d[1]]} }) - this.store.y.connector.broadcast({ - type: 'update', - ops: ops - }) + this.store.y.connector.broadcastOps(ops) } } * isGarbageCollected (id) { @@ -563,10 +557,7 @@ module.exports = function (Y/* :any */) { yield* this.os.put(op) if (!this.store.y.connector.isDisconnected() && this.store.forwardAppliedOperations && op.id[0] !== '_') { // is connected, and this is not going to be send in addOperation - this.store.y.connector.broadcast({ - type: 'update', - ops: [op] - }) + this.store.y.connector.broadcastOps([op]) } } * getOperation (id/* :any */)/* :Transaction */ { From 8544c167714b74de2ecad6d9a429602685a73064 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 17:58:08 +0100 Subject: [PATCH 15/43] Release 0.8.4 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index 3b8e148d..919dfb5e 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 3b8e148d8fee752b6318df2b940316c77936b812 +Subproject commit 919dfb5e167408b087b9a33cf36c347e24a193b6 diff --git a/package.json b/package.json index f5583f3c..74651302 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.4", + "version": "0.8.5", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 0398b5260a35379570b08c3c195eafa57a5975fb Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 15 Jan 2016 18:09:46 +0100 Subject: [PATCH 16/43] Release 0.8.5 --- dist | 2 +- package.json | 2 +- src/Connector.js | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dist b/dist index 919dfb5e..2650338b 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 919dfb5e167408b087b9a33cf36c347e24a193b6 +Subproject commit 2650338b4b188d2e4e7fde9edcd82b93591ec03e diff --git a/package.json b/package.json index 74651302..3ba3274e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.5", + "version": "0.8.6", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { diff --git a/src/Connector.js b/src/Connector.js index 4d537ede..9607b536 100644 --- a/src/Connector.js +++ b/src/Connector.js @@ -171,6 +171,9 @@ module.exports = function (Y/* :any */) { Buffer operations, and broadcast them when ready. */ broadcastOps (ops) { + ops = ops.map(function (op) { + return Y.Struct[op.struct].encode(op) + }) var self = this function broadcastOperations () { if (self.broadcastOpBuffer.length > 0) { From 190442a58d53e62a8d47f9f6fb4ae09d1c3cab68 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 16 Jan 2016 01:40:26 +0100 Subject: [PATCH 17/43] update documentation --- README.md | 91 +++++++++++++++++------------------------------ src/SpecHelper.js | 6 ++++ 2 files changed, 39 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index a804697f..17775e63 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ # ![Yjs](http://y-js.org/images/yjs.png) -Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data types. The framework implements a new OT-like concurrency algorithm and provides similar functionality as [ShareJs] and [OpenCoweb]. Yjs was designed to handle concurrent actions on arbitrary complex data types like Text, Json, and XML. We provide a tutorial and some applications for this framework on our [homepage](http://y-js.org/). +Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data. The framework provides similar functionality as [ShareJs] and [OpenCoweb], but it implements a new algorithm that also support peer-to-peer communication protocols. Yjs was designed to handle concurrent actions on arbitrary data types like Text, Json, and XML. We also provide support for storing and manipulating your shared data offline. For more information and demo applications visit our [homepage](http://y-js.org/). **NOTE** This project is currently migrating. So there may exist some information that is not true anymore.. -You can create you own shared types easily. Therefore, you can take matters into your own hand by defining the meaning of the shared types and ensure that it is valid, while Yjs ensures data consistency (everyone will eventually end up with the same data). We already provide data types for +You can create you own shared types easily. Therefore, you can take matters into your own hand by defining the meaning of the shared types, and ensure that it is valid, while Yjs ensures data consistency (everyone will eventually end up with the same data). We already provide data types for | Name | Description | |----------|-------------------| |[map](https://github.com/y-js/y-map) | Add, update, and remove properties of an object. Included in Yjs| |[array](https://github.com/y-js/y-array) | A shared linked list implementation | -|[selections](https://github.com/y-js/y-selections) | Manages selections on types that use linear structures (e.g. the y-array type). Select a range of elements, and assign meaning to them.| |[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects| |[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *h1*, or *p*)| |[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to several editors| @@ -24,26 +23,30 @@ We support several communication protocols as so called *Connectors*. You can cr |----------------|-----------------------------------| |[xmpp](https://github.com/y-js/y-xmpp) | Propagate updates in a XMPP multi-user-chat room ([XEP-0045](http://xmpp.org/extensions/xep-0045.html))| |[webrtc](https://github.com/y-js/y-webrtc) | Propagate updates Browser2Browser via WebRTC| +|[websockets](https://github.com/y-js/y-websockets-client) | Exchange updates efficiently in the classical client-server model | |[test](https://github.com/y-js/y-test) | A Connector for testing purposes. It is designed to simulate delays that happen in worst case scenarios| +You are not limited to use a specific database to store the shared data. We provide the following database adapters: + +|Name | Description | +|----------------|-----------------------------------| +|[memory](https://github.com/y-js/y-memory) | In-memory storage. | +|[IndexedDb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | You can use Yjs client-, and server- side. You can get it as via npm, and bower. We even provide polymer elements for Yjs! The advantages over similar frameworks are support for * .. P2P message propagation and arbitrary communication protocols * .. arbitrary complex data types -* .. offline editing: Changes are stored persistently and only relevant changes are propagated on rejoin -* .. AnyUndo: Undo *any* action that was executed in constant time (coming..) +* .. offline support: Changes are stored persistently and only relevant changes are propagated on rejoin * .. Intention Preservation: When working on Text, the intention of your changes are preserved. This is particularily important when working offline. Every type has a notion on how we define Intention Preservation on it. ## Use it! -You can find a tutorial, and examples on the [website](http://y-js.org). Furthermore, the [github wiki](https://github.com/y-js/yjs/wiki) offers more information about how you can use Yjs in your application. - -Either clone this git repository, install it with [bower](http://bower.io/), or install it with [npm](https://www.npmjs.org/package/yjs). +Install yjs and its modules with [bower](http://bower.io/), or with [npm](https://www.npmjs.org/package/yjs). ### Bower ``` -bower install y-js/yjs +bower install yjs ``` Then you include the libraries directly from the installation folder. ``` @@ -60,58 +63,30 @@ And use it like this with *npm*: Y = require("yjs"); ``` -# Y() -In order to create an instance of Y, you need to have a connection object (instance of a Connector). Then, you can create a shared data type like this: +# Text editing example ``` -var y = new Y(connector); +Y({ + db: { + name: 'memory' // store in memory. + // name: 'indexeddb' + }, + connector: { + name: 'websockets-client', // choose the websockets connector + // name: 'webrtc' + // name: 'xmpp' + room: 'Textarea-example-dev' + }, + sourceDir: '/bower_components', // location of the y-* modules + share: { + textarea: 'Text' // y.share.textarea is of type Y.Text + } + // modules: ['Richtext', 'Array'] // optional list of modules you want to import +}).then(function (y) { + // bind the textarea to a shared text element + y.share.textarea.bind(document.getElementById('textfield')) +} ``` - -# Y.Map -Yjs includes only one type by default - the Y.Map type. It mimics the behaviour of a javascript Object. You can create, update, and remove properies on the Y.Map type. Furthermore, you can observe changes on this type as you can observe changes on Javascript Objects with [Object.observe](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/observe) - an ECMAScript 7 proposal which is likely to become accepted by the committee. Until then, we have our own implementation. - - -##### Reference -* Create -``` -var map = y.set("new_map", Y.Map).then(function(map){ - map // is my map type -}); -``` -* Every instance of Y is an Y.Map -``` -var y = new Y(options); -``` -* .get(name) - * Retrieve the value of a property. If the value is a type, `.get(name)` returns a promise -* .set(name, value) - * Set/update a property. `value` may be a primitive type, or a custom type definition (e.g. `Y.Map`) -* .delete(name) - * Delete a property -* .observe(observer) - * The `observer` is called whenever something on this object changes. Throws *add*, *update*, and *delete* events -* .observePath(path, observer) - * `path` is an array of property names. `observer` is called when the property under `path` is set, deleted, or updated -* .unobserve(f) - * Delete an observer - -# A note on intention preservation -When users create/update/delete the same property concurrently, only one change will prevail. Changes on different properties do not conflict with each other. - -# A note on time complexities -* .get(name) - * O(1) -* .set(name, value) - * O(1) -* .delete(name) - * O(1) -* Apply a delete operation from another user - * O(1) -* Apply an update operation from another user (set/update a property) - * Yjs does not transform against operations that do not conflict with each other. - * An operation conflicts with another operation if it changes the same property. - * Overall worst case complexety: O(|conflicts|!) - # Status Yjs is a work in progress. Different versions of the *y-* repositories may not work together. Just drop me a line if you run into troubles. diff --git a/src/SpecHelper.js b/src/SpecHelper.js index 484b29fd..baa9d586 100644 --- a/src/SpecHelper.js +++ b/src/SpecHelper.js @@ -77,6 +77,12 @@ function getRandomNumber (n) { } g.getRandomNumber = getRandomNumber +function getRandomString () { + var tokens = 'abcdefäö' // ü\n\n\n\n\n\n\n' + return tokens[getRandomNumber(tokens.length - 1)] +} +g.getRandomString = getRandomString + function * applyTransactions (relAmount, numberOfTransactions, objects, users, transactions) { function randomTransaction (root) { var f = getRandom(transactions) From cfa089f7cfb2833e2006149f52954a3ffbabc275 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 16 Jan 2016 01:42:00 +0100 Subject: [PATCH 18/43] Release 0.8.6 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index 2650338b..a220b06a 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 2650338b4b188d2e4e7fde9edcd82b93591ec03e +Subproject commit a220b06ad939a04d0307d9763bcc97bd39ca878d diff --git a/package.json b/package.json index 3ba3274e..4d348724 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.6", + "version": "0.8.7", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 5b7a4482cf00bf6645591316ebd8015ee96dd97e Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 16 Jan 2016 01:45:58 +0100 Subject: [PATCH 19/43] Release 0.8.7 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index a220b06a..65ea4248 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit a220b06ad939a04d0307d9763bcc97bd39ca878d +Subproject commit 65ea42481e33ae80064e2b36cdfead764d18123c diff --git a/package.json b/package.json index 4d348724..58b52f51 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.7", + "version": "0.8.8", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 65dc716936502e9d72db2e03b4d93110aa21fcab Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 18 Jan 2016 15:40:38 +0100 Subject: [PATCH 20/43] Release 0.8.8 --- dist | 2 +- package.json | 2 +- src/Connector.js | 26 ++++++++++++++------------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/dist b/dist index 65ea4248..e58f6331 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 65ea42481e33ae80064e2b36cdfead764d18123c +Subproject commit e58f6331158882d15b0ac25e8b77a0e049d5d2f2 diff --git a/package.json b/package.json index 58b52f51..6d7791f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.8", + "version": "0.8.9", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { diff --git a/src/Connector.js b/src/Connector.js index 9607b536..74af6f1d 100644 --- a/src/Connector.js +++ b/src/Connector.js @@ -75,19 +75,21 @@ module.exports = function (Y/* :any */) { this.userEventListeners.push(f) } userLeft (user) { - delete this.connections[user] - if (user === this.currentSyncTarget) { - this.currentSyncTarget = null - this.findNextSyncTarget() - } - this.syncingClients = this.syncingClients.filter(function (cli) { - return cli !== user - }) - for (var f of this.userEventListeners) { - f({ - action: 'userLeft', - user: user + if (this.connections[user] != null) { + delete this.connections[user] + if (user === this.currentSyncTarget) { + this.currentSyncTarget = null + this.findNextSyncTarget() + } + this.syncingClients = this.syncingClients.filter(function (cli) { + return cli !== user }) + for (var f of this.userEventListeners) { + f({ + action: 'userLeft', + user: user + }) + } } } userJoined (user, role) { From 5524ab9c20e0565d9e09a72b73c73b8ddb6a292e Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 18 Jan 2016 16:45:46 +0100 Subject: [PATCH 21/43] Release 0.8.9 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index e58f6331..5e295b80 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit e58f6331158882d15b0ac25e8b77a0e049d5d2f2 +Subproject commit 5e295b80d793748515957c5d309d2f8e8e83af29 diff --git a/package.json b/package.json index 6d7791f0..b9a9c52c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.9", + "version": "0.8.10", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 8e4cf8333016b4263d3474f8d9386bf119b2bb17 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 18 Jan 2016 17:21:47 +0100 Subject: [PATCH 22/43] typos --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 17775e63..471a26f6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ # ![Yjs](http://y-js.org/images/yjs.png) -Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data. The framework provides similar functionality as [ShareJs] and [OpenCoweb], but it implements a new algorithm that also support peer-to-peer communication protocols. Yjs was designed to handle concurrent actions on arbitrary data types like Text, Json, and XML. We also provide support for storing and manipulating your shared data offline. For more information and demo applications visit our [homepage](http://y-js.org/). +Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data. The framework provides similar functionality as [ShareJs] and [OpenCoweb], but supports peer-to-peer communication protocols by default. Yjs was designed to handle concurrent actions on arbitrary data like Text, Json, and XML. We also provide support for storing and manipulating your shared data offline. For more information and demo applications visit our [homepage](http://y-js.org/). **NOTE** This project is currently migrating. So there may exist some information that is not true anymore.. -You can create you own shared types easily. Therefore, you can take matters into your own hand by defining the meaning of the shared types, and ensure that it is valid, while Yjs ensures data consistency (everyone will eventually end up with the same data). We already provide data types for +You can create you own shared types easily. +Therefore, you can design the sturcture of your custom type, +and ensure data validity, while Yjs ensures data consistency (everyone will eventually end up with the same data). +We already provide data types for | Name | Description | |----------|-------------------| @@ -15,9 +18,11 @@ You can create you own shared types easily. Therefore, you can take matters into |[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *h1*, or *p*)| |[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to several editors| -Unlike other frameworks, Yjs supports P2P message propagation and is not bound to a specific communication protocol. Therefore, Yjs is extremely scalable and can be used in a wide range of application scenarios. +Yjs supports P2P message propagation, and not bound to a specific communication protocol. Therefore, Yjs is extremely scalable and can be used in a wide range of application scenarios. -We support several communication protocols as so called *Connectors*. You can create your own connector too - read [this wiki page](https://github.com/y-js/yjs/wiki/Custom-Connectors). Currently, we support the following communication protocols: +We support several communication protocols as so called *Connectors*. +You can create your own connector too - read [this wiki page](https://github.com/y-js/yjs/wiki/Custom-Connectors). +Currently, we support the following communication protocols: |Name | Description | |----------------|-----------------------------------| From 37ac7787d0e48832f11d422fbf6f115e321ad3fb Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 21 Jan 2016 21:08:20 +0100 Subject: [PATCH 23/43] Update garbage collect algorithm. Fixed some tests appearantly :) --- src/Database.js | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/Database.js b/src/Database.js index 57aa3884..cc20e1eb 100644 --- a/src/Database.js +++ b/src/Database.js @@ -71,21 +71,32 @@ module.exports = function (Y /* :any */) { this.gcTimeout = opts.gcTimeout || 5000 var os = this function garbageCollect () { - return new Promise((resolve) => { - os.requestTransaction(function * () { - if (os.y.connector != null && os.y.connector.isSynced) { - for (var i = 0; i < os.gc2.length; i++) { - var oid = os.gc2[i] - yield* this.garbageCollectOperation(oid) - } - os.gc2 = os.gc1 - os.gc1 = [] - } + return os.whenTransactionsFinished().then(function () { + if (os.gc1.length > 0 || os.gc2.length > 0) { + return new Promise((resolve) => { + os.requestTransaction(function * () { + if (os.y.connector != null && os.y.connector.isSynced) { + for (var i = 0; i < os.gc2.length; i++) { + var oid = os.gc2[i] + yield* this.garbageCollectOperation(oid) + } + os.gc2 = os.gc1 + os.gc1 = [] + } + // TODO: Use setInterval here instead (when garbageCollect is called several times there will be several timeouts..) + if (os.gcTimeout > 0) { + os.gcInterval = setTimeout(garbageCollect, os.gcTimeout) + } + resolve() + }) + }) + } else { + // TODO: see above if (os.gcTimeout > 0) { os.gcInterval = setTimeout(garbageCollect, os.gcTimeout) } - resolve() - }) + return Promise.resolve() + } }) } this.garbageCollect = garbageCollect @@ -205,8 +216,10 @@ module.exports = function (Y /* :any */) { apply (ops) { for (var key in ops) { var o = ops[key] - var required = Y.Struct[o.struct].requiredOps(o) - this.whenOperationsExist(required, o) + if (o.id == null || o.id[0] !== this.y.connector.userId) { + var required = Y.Struct[o.struct].requiredOps(o) + this.whenOperationsExist(required, o) + } } } /* @@ -390,7 +403,7 @@ module.exports = function (Y /* :any */) { } } requestTransaction (makeGen/* :any */, callImmediately) { - if (true || callImmediately) { // TODO: decide whether this is ok or not.. + if (false || callImmediately) { // TODO: decide whether this is ok or not.. this.waitingTransactions.push(makeGen) if (!this.transactionInProgress) { this.transactionInProgress = true From 1b3f5443b3e5e2a4bd6fb51dddf674b7406a3d14 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 22 Jan 2016 14:09:51 +0100 Subject: [PATCH 24/43] implemented small lookup buffer. This heavily improves lookups for slow databases --- src/Database.js | 14 ++--- src/Utils.js | 160 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 9 deletions(-) diff --git a/src/Database.js b/src/Database.js index cc20e1eb..324f29c3 100644 --- a/src/Database.js +++ b/src/Database.js @@ -403,16 +403,12 @@ module.exports = function (Y /* :any */) { } } requestTransaction (makeGen/* :any */, callImmediately) { - if (false || callImmediately) { // TODO: decide whether this is ok or not.. - this.waitingTransactions.push(makeGen) - if (!this.transactionInProgress) { - this.transactionInProgress = true + this.waitingTransactions.push(makeGen) + if (!this.transactionInProgress) { + this.transactionInProgress = true + if (true || callImmediately) { // TODO: decide whether this is ok or not.. this.transact(this.getNextRequest()) - } - } else { - this.waitingTransactions.push(makeGen) - if (!this.transactionInProgress) { - this.transactionInProgress = true + } else { var self = this setTimeout(function () { self.transact(self.getNextRequest()) diff --git a/src/Utils.js b/src/Utils.js index 8cea17d7..08abdd2a 100644 --- a/src/Utils.js +++ b/src/Utils.js @@ -225,4 +225,164 @@ module.exports = function (Y /* : any*/) { } } Y.utils.compareIds = compareIds + + function createEmptyOpsArray (n) { + var a = new Array(n) + for (var i = 0; i < a.length; i++) { + a[i] = { + id: [null, null] + } + } + return a + } + + function createSmallLookupBuffer (Store) { + /* + This buffer implements a very small buffer that temporarily stores operations + after they are read / before they are written. + The buffer basically implements FIFO. Often requested lookups will be re-queued every time they are looked up / written. + + It can speed up lookups on Operation Stores and State Stores. But it does not require notable use of memory or processing power. + + Good for os and ss, bot not for ds (because it often uses methods that require a flush) + + I tried to optimize this for performance, therefore no highlevel operations. + */ + class SmallLookupBuffer extends Store { + constructor (read, write) { + super() + this.writeBuffer = createEmptyOpsArray(5) + this.readBuffer = createEmptyOpsArray(10) + } + * find (id) { + var i, r + for (i = this.readBuffer.length - 1; i >= 0; i--) { + r = this.readBuffer[i] + // we don't have to use compareids, because id is always defined! + if (r.id[1] === id[1] && r.id[0] === id[0]) { + // found r + // move r to the end of readBuffer + for (; i < this.readBuffer.length - 1; i++) { + this.readBuffer[i] = this.readBuffer[i + 1] + } + this.readBuffer[this.readBuffer.length - 1] = r + return r + } + } + var o + for (i = this.writeBuffer.length - 1; i >= 0; i--) { + r = this.writeBuffer[i] + if (r.id[1] === id[1] && r.id[0] === id[0]) { + o = r + break + } + } + if (i < 0) { + // did not reach break in last loop + // read id and put it to the end of readBuffer + o = yield* super.find(id) + } + if (o != null) { + for (i = 0; i < this.readBuffer.length - 1; i++) { + this.readBuffer[i] = this.readBuffer[i + 1] + } + this.readBuffer[this.readBuffer.length - 1] = o + } + return o + } + * put (o) { + var id = o.id + var i, r // helper variables + for (i = this.writeBuffer.length - 1; i >= 0; i--) { + r = this.writeBuffer[i] + if (r.id[1] === id[1] && r.id[0] === id[0]) { + // is already in buffer + // forget r, and move o to the end of writeBuffer + for (; i < this.writeBuffer.length - 1; i++) { + this.writeBuffer[i] = this.writeBuffer[i + 1] + } + this.writeBuffer[this.writeBuffer.length - 1] = o + break + } + } + if (i < 0) { + // did not reach break in last loop + // write writeBuffer[0] + var write = this.writeBuffer[0] + if (write.id[0] !== null) { + yield* super.put(write) + } + // put o to the end of writeBuffer + for (i = 0; i < this.writeBuffer.length - 1; i++) { + this.writeBuffer[i] = this.writeBuffer[i + 1] + } + this.writeBuffer[this.writeBuffer.length - 1] = o + } + // check readBuffer for every occurence of o.id, overwrite if found + // whether found or not, we'll append o to the readbuffer + for (i = 0; i < this.readBuffer.length - 1; i++) { + r = this.readBuffer[i + 1] + if (r.id[1] === id[1] && r.id[0] === id[0]) { + this.readBuffer[i] = o + } else { + this.readBuffer[i] = r + } + } + this.readBuffer[this.readBuffer.length - 1] = o + } + * delete (id) { + var i, r + for (i = 0; i < this.readBuffer.length; i++) { + r = this.readBuffer[i] + if (r.id[1] === id[1] && r.id[0] === id[0]) { + this.readBuffer[i] = { + id: [null, null] + } + } + } + for (i = 0; i < this.writeBuffer.length; i++) { + r = this.writeBuffer[i] + if (r.id[1] === id[1] && r.id[0] === id[0]) { + this.writeBuffer[i] = { + id: [null, null] + } + } + } + yield* super.delete(id) + } + * findWithLowerBound () { + yield* this.flush() + return yield* super.findWithLowerBound.apply(this, arguments) + } + * findWithUpperBound () { + yield* this.flush() + return yield* super.findWithUpperBound.apply(this, arguments) + } + * findNext () { + yield* this.flush() + return yield* super.findNext.apply(this, arguments) + } + * findPrev () { + yield* this.flush() + return yield* super.findPrev.apply(this, arguments) + } + * iterate () { + yield* this.flush() + yield* super.iterate.apply(this, arguments) + } + * flush () { + for (var i = 0; i < this.writeBuffer.length; i++) { + var write = this.writeBuffer[i] + if (write.id[0] !== null) { + yield* super.put(write) + this.writeBuffer[i] = { + id: [null, null] + } + } + } + } + } + return SmallLookupBuffer + } + Y.utils.createSmallLookupBuffer = createSmallLookupBuffer } From 364ed325b01b3fa40ae4ef6ad064f996f8f005b7 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 22 Jan 2016 14:16:16 +0100 Subject: [PATCH 25/43] fixed spec --- .vscode/launch.json | 2 +- src/Utils.js | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ad0153dd..f5f0d3e1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ "request": "launch", "program": "node_modules/gulp/bin/gulp.js", "stopOnEntry": false, - "args": ["dev:examples"], + "args": ["test"], "cwd": ".", "runtimeExecutable": null, "runtimeArgs": [ diff --git a/src/Utils.js b/src/Utils.js index 08abdd2a..03554f64 100644 --- a/src/Utils.js +++ b/src/Utils.js @@ -340,14 +340,7 @@ module.exports = function (Y /* : any*/) { } } } - for (i = 0; i < this.writeBuffer.length; i++) { - r = this.writeBuffer[i] - if (r.id[1] === id[1] && r.id[0] === id[0]) { - this.writeBuffer[i] = { - id: [null, null] - } - } - } + yield* this.flush() yield* super.delete(id) } * findWithLowerBound () { From 38bf398709f1a2d41b54e9378d1c097c66c370af Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 23 Jan 2016 01:02:01 +0100 Subject: [PATCH 26/43] Improvements that are required for offline editing --- gulpfile.helper.js | 2 +- gulpfile.js | 2 +- src/Database.js | 53 ++++++++++++++++++++++++++++------------------ src/SpecHelper.js | 2 +- src/Transaction.js | 5 +++++ src/Utils.js | 6 +++--- src/y.js | 5 +++-- 7 files changed, 46 insertions(+), 29 deletions(-) diff --git a/gulpfile.helper.js b/gulpfile.helper.js index 2e1ea71f..58ae80f3 100644 --- a/gulpfile.helper.js +++ b/gulpfile.helper.js @@ -94,7 +94,7 @@ module.exports = function (gulp, helperOptions) { return browserify({ entries: files.specs, - debug: options.debug + debug: true }).bundle() .pipe(source('specs.js')) .pipe(buffer()) diff --git a/gulpfile.js b/gulpfile.js index b4eee461..668f2f07 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -54,7 +54,7 @@ require('./gulpfile.helper.js')(gulp, { moduleName: 'yjs', includeRuntime: true, specs: [ - './src/Types/Map.spec.js', + '../y-array/src/Array.spec.js', './src/Database.spec.js' ] }) diff --git a/src/Database.js b/src/Database.js index 324f29c3..0dd2972c 100644 --- a/src/Database.js +++ b/src/Database.js @@ -39,6 +39,13 @@ module.exports = function (Y /* :any */) { */ constructor (y, opts) { this.y = y + var os = this + this.userId = null + var resolve + this.userIdPromise = new Promise(function (r) { + resolve = r + }) + this.userIdPromise.resolve = resolve // whether to broadcast all applied operations (insert & delete hook) this.forwardAppliedOperations = false // E.g. this.listenersById[id] : Array @@ -60,16 +67,15 @@ module.exports = function (Y /* :any */) { // TODO: Use ES7 Weak Maps. This way types that are no longer user, // wont be kept in memory. this.initializedTypes = {} - this.whenUserIdSetListener = null this.waitingTransactions = [] this.transactionInProgress = false + this.transactionIsFlushed = false if (typeof YConcurrency_TestingMode !== 'undefined') { this.executeOrder = [] } this.gc1 = [] // first stage this.gc2 = [] // second stage -> after that, remove the op this.gcTimeout = opts.gcTimeout || 5000 - var os = this function garbageCollect () { return os.whenTransactionsFinished().then(function () { if (os.gc1.length > 0 || os.gc2.length > 0) { @@ -175,26 +181,20 @@ module.exports = function (Y /* :any */) { this.gcInterval = null } setUserId (userId) { - var self = this - return new Promise(function (resolve) { + if (!this.userIdPromise.inProgress) { + this.userIdPromise.inProgress = true + var self = this self.requestTransaction(function * () { self.userId = userId var state = yield* this.getState(userId) self.opClock = state.clock - if (self.whenUserIdSetListener != null) { - self.whenUserIdSetListener() - self.whenUserIdSetListener = null - } - resolve() + self.userIdPromise.resolve(userId) }) - }) + } + return this.userIdPromise } whenUserIdSet (f) { - if (this.userId != null) { - f() - } else { - this.whenUserIdSetListener = f - } + this.userIdPromise.then(f) } getNextOpId () { if (this._nextUserId != null) { @@ -390,15 +390,26 @@ module.exports = function (Y /* :any */) { return Promise.resolve() } } + // Check if there is another transaction request. + // * the last transaction is always a flush :) getNextRequest () { if (this.waitingTransactions.length === 0) { - this.transactionInProgress = false - if (this.transactionsFinished != null) { - this.transactionsFinished.resolve() - this.transactionsFinished = null + if (this.transactionIsFlushed) { + this.transactionInProgress = false + this.transactionIsFlushed = false + if (this.transactionsFinished != null) { + this.transactionsFinished.resolve() + this.transactionsFinished = null + } + return null + } else { + this.transactionIsFlushed = true + return function * () { + yield* this.flush() + } } - return null } else { + this.transactionIsFlushed = false return this.waitingTransactions.shift() } } @@ -406,7 +417,7 @@ module.exports = function (Y /* :any */) { this.waitingTransactions.push(makeGen) if (!this.transactionInProgress) { this.transactionInProgress = true - if (true || callImmediately) { // TODO: decide whether this is ok or not.. + if (false || callImmediately) { // TODO: decide whether this is ok or not.. this.transact(this.getNextRequest()) } else { var self = this diff --git a/src/SpecHelper.js b/src/SpecHelper.js index baa9d586..4097b7a7 100644 --- a/src/SpecHelper.js +++ b/src/SpecHelper.js @@ -25,7 +25,7 @@ g.g = g g.YConcurrency_TestingMode = true -jasmine.DEFAULT_TIMEOUT_INTERVAL = 8000 +jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000 g.describeManyTimes = function describeManyTimes (times, name, f) { for (var i = 0; i < times; i++) { diff --git a/src/Transaction.js b/src/Transaction.js index da82223c..45d2e8d6 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -763,6 +763,11 @@ module.exports = function (Y/* :any */) { return op } */ + * flush () { + yield* this.os.flush() + yield* this.ss.flush() + yield* this.ds.flush() + } } Y.Transaction = TransactionInterface } diff --git a/src/Utils.js b/src/Utils.js index 03554f64..e6b1bdf6 100644 --- a/src/Utils.js +++ b/src/Utils.js @@ -207,7 +207,7 @@ module.exports = function (Y /* : any*/) { Defines a smaller relation on Id's */ function smaller (a, b) { - return a[0] < b[0] || (a[0] === b[0] && a[1] < b[1]) + return a[0] < b[0] || (a[0] === b[0] && (a[1] < b[1] || typeof a[1] < typeof b[1])) } Y.utils.smaller = smaller @@ -249,8 +249,8 @@ module.exports = function (Y /* : any*/) { I tried to optimize this for performance, therefore no highlevel operations. */ class SmallLookupBuffer extends Store { - constructor (read, write) { - super() + constructor () { + super(...arguments) this.writeBuffer = createEmptyOpsArray(5) this.readBuffer = createEmptyOpsArray(10) } diff --git a/src/y.js b/src/y.js index 7f7d2485..3718f311 100644 --- a/src/y.js +++ b/src/y.js @@ -110,9 +110,9 @@ class YConfig { share: {[key: string]: any}; */ constructor (opts, callback) { + this.options = opts this.db = new Y[opts.db.name](this, opts.db) this.connector = new Y[opts.connector.name](this, opts.connector) - this.options = opts } init (callback) { var opts = this.options @@ -131,7 +131,8 @@ class YConfig { } share[propertyname] = yield* this.getType(id) } - setTimeout(callback, 0) + this.store.whenTransactionsFinished() + .then(callback) }) } isConnected () { From 39dc2317b71ec322a63bdfc05a811d051ccf94d8 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 23 Jan 2016 20:09:30 +0100 Subject: [PATCH 27/43] Implemented more efficient garbage collectior from worst case of O(n) -> O(1) - where n is the number of insertions in a list So this is a huge improvement, I guess :) --- src/SpecHelper.js | 2 ++ src/Struct.js | 13 +++++++++++ src/Transaction.js | 56 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/SpecHelper.js b/src/SpecHelper.js index 4097b7a7..28abcf5b 100644 --- a/src/SpecHelper.js +++ b/src/SpecHelper.js @@ -208,6 +208,7 @@ g.compareAllUsers = async(function * compareAllUsers (users) { yield* this.os.iterate(this, null, null, function * (o) { o = Y.utils.copyObject(o) delete o.origin + delete o.originOf db1.push(o) }) }) @@ -222,6 +223,7 @@ g.compareAllUsers = async(function * compareAllUsers (users) { yield* this.os.iterate(this, null, null, function * (o) { o = Y.utils.copyObject(o) delete o.origin + delete o.originOf expect(db1[count++]).toEqual(o) }) }) diff --git a/src/Struct.js b/src/Struct.js index 0ea0dd13..66381278 100644 --- a/src/Struct.js +++ b/src/Struct.js @@ -126,6 +126,19 @@ module.exports = function (Y/* :any */) { execute: function *(op) { var i // loop counter var distanceToOrigin = i = yield* Struct.Insert.getDistanceToOrigin.call(this, op) // most cases: 0 (starts from 0) + + if (op.origin != null) { + // we save in origin that op originates in it + // we need that later when we eventually garbage collect origin (see transaction) + var origin = yield* this.getOperation(op.origin) + if (origin.originOf == null) { + origin.originOf = [] + } + origin.originOf.push(op.id) + yield* this.setOperation(origin) + } + + // now we begin to insert op in the list of insertions.. var o var parent var start diff --git a/src/Transaction.js b/src/Transaction.js index 45d2e8d6..c1e1db14 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -362,23 +362,31 @@ module.exports = function (Y/* :any */) { if (o.right != null) { var right = yield* this.getOperation(o.right) right.left = o.left - if (Y.utils.compareIds(right.origin, o.id)) { // rights origin is o + + if (Y.utils.compareIds(right.origin, id)) { // rights origin is o // find new origin of right ops // origin is the first left deleted operation var neworigin = o.left + var neworigin_ = null while (neworigin != null) { - var neworigin_ = yield* this.getOperation(neworigin) + neworigin_ = yield* this.getOperation(neworigin) if (neworigin_.deleted) { break } neworigin = neworigin_.left } + // reset origin of all right ops (except first right - duh!), + + /* ** The following code does not rely on the the originOf property ** + I recently added originOf to all Insert Operations (see Struct.Insert.execute), + which saves which operations originate in a Insert operation. + Garbage collecting without originOf is more memory efficient, but is nearly impossible for large texts, or lists! + But I keep this code for now + ``` // reset origin of right right.origin = neworigin - - // reset origin of all right ops (except first right - duh!), - // until you find origin pointer to the left of o + // search until you find origin pointer to the left of o if (right.right != null) { var i = yield* this.getOperation(right.right) var ids = [o.id, o.right] @@ -399,9 +407,41 @@ module.exports = function (Y/* :any */) { } } } - } /* otherwise, rights origin is to the left of o, - then there is no right op (from o), that origins in o */ - yield* this.setOperation(right) + ``` + */ + // ** Now the new implementation starts ** + // reset neworigin of all originOf[*] + for (var _i in o.originOf) { + var originsIn = yield* this.getOperation(o.originOf[_i]) + if (originsIn != null) { + originsIn.origin = neworigin + yield* this.setOperation(originsIn) + } + } + if (neworigin != null) { + if (neworigin_.originOf == null) { + neworigin_.originOf = o.originOf + } else { + neworigin_.originOf = o.originOf.concat(neworigin_.originOf) + } + yield* this.setOperation(neworigin_) + } + // we don't need to set right here, because + // right should be in o.originOf => it is set it the previous for loop + } else { + // we didn't need to reset the origin of right + // so we have to set right here + yield* this.setOperation(right) + } + // o may originate in another operation. + // Since o is deleted, we have to reset o.origin's `originOf` property + if (o.origin != null) { + var origin = yield* this.getOperation(o.origin) + origin.originOf = origin.originOf.filter(function (_id) { + return !Y.utils.compareIds(id, _id) + }) + yield* this.setOperation(origin) + } } if (o.parent != null) { From 6b1cf18822f1f734df59687fd471990255d093b6 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 26 Jan 2016 11:29:58 +0100 Subject: [PATCH 28/43] Improvements on DS lookups --- gulpfile.helper.js | 4 ++-- src/Transaction.js | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/gulpfile.helper.js b/gulpfile.helper.js index 58ae80f3..f57169d2 100644 --- a/gulpfile.helper.js +++ b/gulpfile.helper.js @@ -98,8 +98,8 @@ module.exports = function (gulp, helperOptions) { }).bundle() .pipe(source('specs.js')) .pipe(buffer()) - .pipe($.sourcemaps.init({loadMaps: true})) - .pipe($.sourcemaps.write('.')) + // .pipe($.sourcemaps.init({loadMaps: true})) + // .pipe($.sourcemaps.write('.')) .pipe(gulp.dest('./build/')) }) diff --git a/src/Transaction.js b/src/Transaction.js index c1e1db14..a4404f63 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -331,6 +331,7 @@ module.exports = function (Y/* :any */) { * garbageCollectOperation (id) { this.store.addToDebug('yield* this.garbageCollectOperation(', id, ')') // check to increase the state of the respective user + var o = null var state = yield* this.getState(id[0]) if (state.clock === id[1]) { state.clock++ @@ -338,11 +339,11 @@ module.exports = function (Y/* :any */) { yield* this.checkDeleteStoreForState(state) // then set the state yield* this.setState(state) - } - yield* this.markGarbageCollected(id) - + } else if (state.clock > id[1]) { + o = yield* this.getOperation(id) + } // else state.clock < id[1], don't clean up + yield* this.markGarbageCollected(id) // always mark gc'd // if op exists, then clean that mess up.. - var o = yield* this.getOperation(id) if (o != null) { /* if (!o.deleted) { @@ -363,7 +364,7 @@ module.exports = function (Y/* :any */) { var right = yield* this.getOperation(o.right) right.left = o.left - if (Y.utils.compareIds(right.origin, id)) { // rights origin is o + if (o.originOf != null && o.originOf.length > 0) { // find new origin of right ops // origin is the first left deleted operation var neworigin = o.left @@ -545,10 +546,15 @@ module.exports = function (Y/* :any */) { var del = deletions[i] var id = [del[0], del[1]] // always try to delete.. - var addOperation = yield* this.deleteOperation(id) - if (addOperation) { - // TODO:.. really .. here? You could prevent calling all these functions in operationAdded - yield* this.store.operationAdded(this, {struct: 'Delete', target: id}) + var state = yield* this.getState(id[0]) + if (id[1] < state.clock) { + var addOperation = yield* this.deleteOperation(id) + if (addOperation) { + // TODO:.. really .. here? You could prevent calling all these functions in operationAdded + yield* this.store.operationAdded(this, {struct: 'Delete', target: id}) + } + } else { + yield* this.markDeleted(id) } if (del[2]) { // gc From 31d2a231e3a7e0d77ff0b4443a5e9c19d8054ae6 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 26 Jan 2016 15:30:19 +0100 Subject: [PATCH 29/43] Further reduced number of db requests --- gulpfile.js | 3 +- src/Database.js | 2 +- src/Database.spec.js | 32 +++---- src/Transaction.js | 220 ++++++++++++++++++++++++++----------------- 4 files changed, 151 insertions(+), 106 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 668f2f07..b1dd3602 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -54,8 +54,9 @@ require('./gulpfile.helper.js')(gulp, { moduleName: 'yjs', includeRuntime: true, specs: [ + './src/Database.spec.js', '../y-array/src/Array.spec.js', - './src/Database.spec.js' + '../y-map/src/Map.spec.js' ] }) diff --git a/src/Database.js b/src/Database.js index 0dd2972c..438be1a9 100644 --- a/src/Database.js +++ b/src/Database.js @@ -75,7 +75,7 @@ module.exports = function (Y /* :any */) { } this.gc1 = [] // first stage this.gc2 = [] // second stage -> after that, remove the op - this.gcTimeout = opts.gcTimeout || 5000 + this.gcTimeout = opts.gcTimeout || 50000 function garbageCollect () { return os.whenTransactionsFinished().then(function () { if (os.gc1.length > 0 || os.gc2.length > 0) { diff --git a/src/Database.spec.js b/src/Database.spec.js index 96b07a96..0a91044d 100644 --- a/src/Database.spec.js +++ b/src/Database.spec.js @@ -54,10 +54,10 @@ for (let database of databases) { yield* this.markDeleted(['166', 2]) yield* this.markDeleted(['166', 0]) yield* this.markDeleted(['166', 2]) - yield* this.markGarbageCollected(['166', 2]) + yield* this.markGarbageCollected(['166', 2], 1) yield* this.markDeleted(['166', 1]) yield* this.markDeleted(['166', 3]) - yield* this.markGarbageCollected(['166', 3]) + yield* this.markGarbageCollected(['166', 3], 1) yield* this.markDeleted(['166', 0]) expect(yield* this.getDeleteSet()).toEqual({'166': [[0, 2, false], [2, 2, true]]}) done() @@ -68,9 +68,9 @@ for (let database of databases) { yield* this.markDeleted(['293', 0]) yield* this.markDeleted(['291', 2]) yield* this.markDeleted(['291', 2]) - yield* this.markGarbageCollected(['293', 0]) + yield* this.markGarbageCollected(['293', 0], 1) yield* this.markDeleted(['293', 1]) - yield* this.markGarbageCollected(['291', 2]) + yield* this.markGarbageCollected(['291', 2], 1) expect(yield* this.getDeleteSet()).toEqual({'291': [[2, 1, true]], '293': [[0, 1, true], [1, 1, false]]}) done() }) @@ -81,15 +81,15 @@ for (let database of databases) { yield* this.markDeleted(['581', 1]) yield* this.markDeleted(['580', 0]) yield* this.markDeleted(['580', 0]) - yield* this.markGarbageCollected(['581', 0]) + yield* this.markGarbageCollected(['581', 0], 1) yield* this.markDeleted(['581', 2]) yield* this.markDeleted(['580', 1]) yield* this.markDeleted(['580', 2]) yield* this.markDeleted(['580', 1]) yield* this.markDeleted(['580', 2]) - yield* this.markGarbageCollected(['581', 2]) - yield* this.markGarbageCollected(['581', 1]) - yield* this.markGarbageCollected(['580', 1]) + yield* this.markGarbageCollected(['581', 2], 1) + yield* this.markGarbageCollected(['581', 1], 1) + yield* this.markGarbageCollected(['580', 1], 1) expect(yield* this.getDeleteSet()).toEqual({'580': [[0, 1, false], [1, 1, true], [2, 1, false]], '581': [[0, 3, true]]}) done() }) @@ -100,7 +100,7 @@ for (let database of databases) { yield* this.markDeleted(['543', 2]) yield* this.markDeleted(['544', 0]) yield* this.markDeleted(['543', 2]) - yield* this.markGarbageCollected(['544', 0]) + yield* this.markGarbageCollected(['544', 0], 1) yield* this.markDeleted(['545', 1]) yield* this.markDeleted(['543', 4]) yield* this.markDeleted(['543', 3]) @@ -108,10 +108,10 @@ for (let database of databases) { yield* this.markDeleted(['544', 2]) yield* this.markDeleted(['544', 1]) yield* this.markDeleted(['544', 2]) - yield* this.markGarbageCollected(['543', 2]) - yield* this.markGarbageCollected(['543', 4]) - yield* this.markGarbageCollected(['544', 2]) - yield* this.markGarbageCollected(['543', 3]) + yield* this.markGarbageCollected(['543', 2], 1) + yield* this.markGarbageCollected(['543', 4], 1) + yield* this.markGarbageCollected(['544', 2], 1) + yield* this.markGarbageCollected(['543', 3], 1) expect(yield* this.getDeleteSet()).toEqual({'543': [[2, 3, true]], '544': [[0, 1, true], [1, 1, false], [2, 1, true]], '545': [[1, 1, false]]}) done() }) @@ -142,14 +142,14 @@ for (let database of databases) { yield* this.markDeleted(['11', 1]) yield* this.markDeleted(['9', 4]) yield* this.markDeleted(['10', 0]) - yield* this.markGarbageCollected(['11', 2]) + yield* this.markGarbageCollected(['11', 2], 1) yield* this.markDeleted(['11', 2]) - yield* this.markGarbageCollected(['11', 3]) + yield* this.markGarbageCollected(['11', 3], 1) yield* this.markDeleted(['11', 3]) yield* this.markDeleted(['11', 3]) yield* this.markDeleted(['9', 4]) yield* this.markDeleted(['10', 0]) - yield* this.markGarbageCollected(['11', 1]) + yield* this.markGarbageCollected(['11', 1], 1) yield* this.markDeleted(['11', 1]) expect(yield* this.getDeleteSet()).toEqual({'9': [[2, 1, false], [4, 1, false]], '10': [[0, 1, false]], '11': [[1, 3, true], [4, 1, false]]}) done() diff --git a/src/Transaction.js b/src/Transaction.js index a4404f63..46ba388a 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -222,86 +222,125 @@ module.exports = function (Y/* :any */) { /* Mark an operation as deleted&gc'd */ - * markGarbageCollected (id) { + * markGarbageCollected (id, len) { // this.mem.push(["gc", id]); - var n = yield* this.markDeleted(id) - if (!n.gc) { - if (n.id[1] < id[1]) { - // un-extend left - var newlen = n.len - (id[1] - n.id[1]) - n.len -= newlen - yield* this.ds.put(n) - n = {id: id, len: newlen, gc: false} - yield* this.ds.put(n) - } - // get prev&next before adding a new operation - var prev = yield* this.ds.findPrev(id) - var next = yield* this.ds.findNext(id) - - if (id[1] < n.id[1] + n.len - 1) { - // un-extend right - yield* this.ds.put({id: [id[0], id[1] + 1], len: n.len - 1, gc: false}) - n.len = 1 - } - // set gc'd - n.gc = true - // can extend left? - if ( - prev != null && - prev.gc && - Y.utils.compareIds([prev.id[0], prev.id[1] + prev.len], n.id) - ) { - prev.len += n.len - yield* this.ds.delete(n.id) - n = prev - // ds.put n here? - } - // can extend right? - if ( - next != null && - next.gc && - Y.utils.compareIds([n.id[0], n.id[1] + n.len], next.id) - ) { - n.len += next.len - yield* this.ds.delete(next.id) - } + var n = yield* this.markDeleted(id, len) + if (n.id[1] < id[1] && !n.gc) { + // un-extend left + var newlen = n.len - (id[1] - n.id[1]) + n.len -= newlen + yield* this.ds.put(n) + n = {id: id, len: newlen, gc: false} yield* this.ds.put(n) } + // get prev&next before adding a new operation + var prev = yield* this.ds.findPrev(id) + var next = yield* this.ds.findNext(id) + + if (id[1] < n.id[1] + n.len - len && !n.gc) { + // un-extend right + yield* this.ds.put({id: [id[0], id[1] + 1], len: n.len - 1, gc: false}) + n.len = 1 + } + // set gc'd + n.gc = true + // can extend left? + if ( + prev != null && + prev.gc && + Y.utils.compareIds([prev.id[0], prev.id[1] + prev.len], n.id) + ) { + prev.len += n.len + yield* this.ds.delete(n.id) + n = prev + // ds.put n here? + } + // can extend right? + if ( + next != null && + next.gc && + Y.utils.compareIds([n.id[0], n.id[1] + n.len], next.id) + ) { + n.len += next.len + yield* this.ds.delete(next.id) + } + yield* this.ds.put(n) } /* Mark an operation as deleted. returns the delete node */ - * markDeleted (id) { + * markDeleted (id, length) { + if (length == null) { + length = 1 + // debugger // TODO!! + } // this.mem.push(["del", id]); var n = yield* this.ds.findWithUpperBound(id) if (n != null && n.id[0] === id[0]) { - if (n.id[1] <= id[1] && id[1] < n.id[1] + n.len) { - // already deleted - return n - } else if (n.id[1] + n.len === id[1] && !n.gc) { - // can extend existing deletion - n.len++ + if (n.id[1] <= id[1] && id[1] <= n.id[1] + n.len) { + // id is in n's range + var diff = id[1] + length - (n.id[1] + n.len) // overlapping right + if (diff > 0) { + // id+length overlaps n + if (!n.gc) { + n.len += diff + } else { + diff = n.id[1] + n.len - id[1] // overlapping left (id till n.end) + if (diff < length) { + // a partial deletion + n = {id: [id[0], id[1] + diff], len: length - diff, gc: false} + yield* this.ds.put(n) + } else { + // already gc'd + throw new Error('Cannot happen! (it dit though.. :()') + // return n + } + } + } else { + // no overlapping, already deleted + return n + } } else { - // cannot extend left - n = {id: id, len: 1, gc: false} - yield* this.ds.put(n) + // cannot extend left (there is no left!) + n = {id: id, len: length, gc: false} + yield* this.ds.put(n) // TODO: you double-put !! } } else { // cannot extend left - n = {id: id, len: 1, gc: false} + n = {id: id, len: length, gc: false} yield* this.ds.put(n) } // can extend right? var next = yield* this.ds.findNext(n.id) if ( next != null && - Y.utils.compareIds([n.id[0], n.id[1] + n.len], next.id) && - !next.gc + n.id[0] === next.id[0] && + n.id[1] + n.len >= next.id[1] ) { - n.len = n.len + next.len - yield* this.ds.delete(next.id) + diff = n.id[1] + n.len - next.id[1] // from next.start to n.end + if (next.gc) { + if (diff >= 0) { + n.len -= diff + if (diff > next.len) { + // need to create another deletion after $next + // TODO: (may not be necessary, because this case shouldn't happen!) + // also this is supposed to return a deletion range. which one to choose? n or the new created deletion? + throw new Error('This case is not handled (on purpose!)') + } + } // else: everything is fine :) + } else { + if (diff >= 0) { + if (diff > next.len) { + // may be neccessary to extend next.next! + // TODO: (may not be necessary, because this case shouldn't happen!) + throw new Error('This case is not handled (on purpose!)') + } + n.len += next.len - diff + yield* this.ds.delete(next.id) + } + } } yield* this.ds.put(n) return n @@ -330,19 +369,8 @@ module.exports = function (Y/* :any */) { */ * garbageCollectOperation (id) { this.store.addToDebug('yield* this.garbageCollectOperation(', id, ')') - // check to increase the state of the respective user - var o = null - var state = yield* this.getState(id[0]) - if (state.clock === id[1]) { - state.clock++ - // also check if more expected operations were gc'd - yield* this.checkDeleteStoreForState(state) - // then set the state - yield* this.setState(state) - } else if (state.clock > id[1]) { - o = yield* this.getOperation(id) - } // else state.clock < id[1], don't clean up - yield* this.markGarbageCollected(id) // always mark gc'd + var o = yield* this.getOperation(id) + yield* this.markGarbageCollected(id, 1) // always mark gc'd // if op exists, then clean that mess up.. if (o != null) { /* @@ -487,9 +515,7 @@ module.exports = function (Y/* :any */) { * applyDeleteSet (ds) { var deletions = [] function createDeletions (user, start, len, gc) { - for (var c = start; c < start + len; c++) { - deletions.push([user, c, gc]) - } + deletions.push([user, start, len, gc]) } for (var user in ds) { @@ -544,28 +570,46 @@ module.exports = function (Y/* :any */) { } for (var i = 0; i < deletions.length; i++) { var del = deletions[i] - var id = [del[0], del[1]] // always try to delete.. - var state = yield* this.getState(id[0]) - if (id[1] < state.clock) { - var addOperation = yield* this.deleteOperation(id) - if (addOperation) { - // TODO:.. really .. here? You could prevent calling all these functions in operationAdded - yield* this.store.operationAdded(this, {struct: 'Delete', target: id}) + var state = yield* this.getState(del[0]) + if (del[1] < state.clock) { + for (let c = del[1]; c < del[1] + del[2]; c++) { + var id = [del[0], c] + var addOperation = yield* this.deleteOperation(id) + if (addOperation) { + // TODO:.. really .. here? You could prevent calling all these functions in operationAdded + yield* this.store.operationAdded(this, {struct: 'Delete', target: id}) + } + if (del[3]) { + // gc + yield* this.garbageCollectOperation(id) + } } } else { - yield* this.markDeleted(id) + if (del[3]) { + yield* this.markGarbageCollected([del[0], del[1]], del[2]) + } else { + yield* this.markDeleted([del[0], del[1]], del[2]) + } } - if (del[2]) { - // gc - yield* this.garbageCollectOperation(id) + if (del[3]) { + // check to increase the state of the respective user + if (state.clock >= del[1] && state.clock < del[1] + del[2]) { + state.clock = del[1] + del[2] + // also check if more expected operations were gc'd + yield* this.checkDeleteStoreForState(state) // TODO: unneccessary? + // then set the state + yield* this.setState(state) + } } } if (this.store.forwardAppliedOperations) { - var ops = deletions.map(function (d) { - return {struct: 'Delete', target: [d[0], d[1]]} - }) - this.store.y.connector.broadcastOps(ops) + for (let c = del[1]; c < del[1] + del[2]; c++) { + var ops = deletions.map(function (d) { + return {struct: 'Delete', target: [d[0], c]} // TODO: implement Delete with deletion length! + }) + this.store.y.connector.broadcastOps(ops) + } } } * isGarbageCollected (id) { From 4b6352b11a4a90c0fc4c40523de8c458b0dd37bf Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 27 Jan 2016 11:34:11 +0100 Subject: [PATCH 30/43] typo --- src/Utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utils.js b/src/Utils.js index e6b1bdf6..a851192b 100644 --- a/src/Utils.js +++ b/src/Utils.js @@ -20,7 +20,7 @@ ``` The structures usually work asynchronously (you have to wait for the - database request to finish). EventHandler will help you to make your type + database request to finish). EventHandler helps you to make your type synchronous. */ module.exports = function (Y /* : any*/) { From 89a6ec374ed13f5bf03f36bde8dbad7ae661a3ce Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 27 Jan 2016 17:05:28 +0100 Subject: [PATCH 31/43] update --- dist | 2 +- src/Database.js | 2 +- src/Database.spec.js | 98 ++++++++++++++++++++++---------------------- src/Struct.js | 2 +- src/Transaction.js | 5 ++- 5 files changed, 55 insertions(+), 54 deletions(-) diff --git a/dist b/dist index 5e295b80..8ba1e4ce 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 5e295b80d793748515957c5d309d2f8e8e83af29 +Subproject commit 8ba1e4ce27a75bb025a5478723ea57f7f437ccf0 diff --git a/src/Database.js b/src/Database.js index 438be1a9..5ad9f4a8 100644 --- a/src/Database.js +++ b/src/Database.js @@ -365,7 +365,7 @@ module.exports = function (Y /* :any */) { yield* Y.Struct['Delete'].execute.call(transaction, delop) } - // notify parent, if it has been initialized as a custom type + // notify parent, if it was instanciated as a custom type if (t != null) { yield* t._changed(transaction, Y.utils.copyObject(op)) } diff --git a/src/Database.spec.js b/src/Database.spec.js index 0a91044d..4aa3a825 100644 --- a/src/Database.spec.js +++ b/src/Database.spec.js @@ -23,7 +23,7 @@ for (let database of databases) { }) it('Deleted operation is deleted', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['u1', 10]) + yield* this.markDeleted(['u1', 10], 1) expect(yield* this.isDeleted(['u1', 10])).toBeTruthy() expect(yield* this.getDeleteSet()).toEqual({'u1': [[10, 1, false]]}) done() @@ -31,8 +31,8 @@ for (let database of databases) { })) it('Deleted operation extends other deleted operation', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['u1', 10]) - yield* this.markDeleted(['u1', 11]) + yield* this.markDeleted(['u1', 10], 1) + yield* this.markDeleted(['u1', 11], 1) expect(yield* this.isDeleted(['u1', 10])).toBeTruthy() expect(yield* this.isDeleted(['u1', 11])).toBeTruthy() expect(yield* this.getDeleteSet()).toEqual({'u1': [[10, 2, false]]}) @@ -41,35 +41,35 @@ for (let database of databases) { })) it('Deleted operation extends other deleted operation', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['0', 3]) - yield* this.markDeleted(['0', 4]) - yield* this.markDeleted(['0', 2]) + yield* this.markDeleted(['0', 3], 1) + yield* this.markDeleted(['0', 4], 1) + yield* this.markDeleted(['0', 2], 1) expect(yield* this.getDeleteSet()).toEqual({'0': [[2, 3, false]]}) done() }) })) it('Debug #1', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['166', 0]) - yield* this.markDeleted(['166', 2]) - yield* this.markDeleted(['166', 0]) - yield* this.markDeleted(['166', 2]) + yield* this.markDeleted(['166', 0], 1) + yield* this.markDeleted(['166', 2], 1) + yield* this.markDeleted(['166', 0], 1) + yield* this.markDeleted(['166', 2], 1) yield* this.markGarbageCollected(['166', 2], 1) - yield* this.markDeleted(['166', 1]) - yield* this.markDeleted(['166', 3]) + yield* this.markDeleted(['166', 1], 1) + yield* this.markDeleted(['166', 3], 1) yield* this.markGarbageCollected(['166', 3], 1) - yield* this.markDeleted(['166', 0]) + yield* this.markDeleted(['166', 0], 1) expect(yield* this.getDeleteSet()).toEqual({'166': [[0, 2, false], [2, 2, true]]}) done() }) })) it('Debug #2', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['293', 0]) - yield* this.markDeleted(['291', 2]) - yield* this.markDeleted(['291', 2]) + yield* this.markDeleted(['293', 0], 1) + yield* this.markDeleted(['291', 2], 1) + yield* this.markDeleted(['291', 2], 1) yield* this.markGarbageCollected(['293', 0], 1) - yield* this.markDeleted(['293', 1]) + yield* this.markDeleted(['293', 1], 1) yield* this.markGarbageCollected(['291', 2], 1) expect(yield* this.getDeleteSet()).toEqual({'291': [[2, 1, true]], '293': [[0, 1, true], [1, 1, false]]}) done() @@ -77,16 +77,16 @@ for (let database of databases) { })) it('Debug #3', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['581', 0]) - yield* this.markDeleted(['581', 1]) - yield* this.markDeleted(['580', 0]) - yield* this.markDeleted(['580', 0]) + yield* this.markDeleted(['581', 0], 1) + yield* this.markDeleted(['581', 1], 1) + yield* this.markDeleted(['580', 0], 1) + yield* this.markDeleted(['580', 0], 1) yield* this.markGarbageCollected(['581', 0], 1) - yield* this.markDeleted(['581', 2]) - yield* this.markDeleted(['580', 1]) - yield* this.markDeleted(['580', 2]) - yield* this.markDeleted(['580', 1]) - yield* this.markDeleted(['580', 2]) + yield* this.markDeleted(['581', 2], 1) + yield* this.markDeleted(['580', 1], 1) + yield* this.markDeleted(['580', 2], 1) + yield* this.markDeleted(['580', 1], 1) + yield* this.markDeleted(['580', 2], 1) yield* this.markGarbageCollected(['581', 2], 1) yield* this.markGarbageCollected(['581', 1], 1) yield* this.markGarbageCollected(['580', 1], 1) @@ -96,18 +96,18 @@ for (let database of databases) { })) it('Debug #4', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['544', 0]) - yield* this.markDeleted(['543', 2]) - yield* this.markDeleted(['544', 0]) - yield* this.markDeleted(['543', 2]) + yield* this.markDeleted(['544', 0], 1) + yield* this.markDeleted(['543', 2], 1) + yield* this.markDeleted(['544', 0], 1) + yield* this.markDeleted(['543', 2], 1) yield* this.markGarbageCollected(['544', 0], 1) - yield* this.markDeleted(['545', 1]) - yield* this.markDeleted(['543', 4]) - yield* this.markDeleted(['543', 3]) - yield* this.markDeleted(['544', 1]) - yield* this.markDeleted(['544', 2]) - yield* this.markDeleted(['544', 1]) - yield* this.markDeleted(['544', 2]) + yield* this.markDeleted(['545', 1], 1) + yield* this.markDeleted(['543', 4], 1) + yield* this.markDeleted(['543', 3], 1) + yield* this.markDeleted(['544', 1], 1) + yield* this.markDeleted(['544', 2], 1) + yield* this.markDeleted(['544', 1], 1) + yield* this.markDeleted(['544', 2], 1) yield* this.markGarbageCollected(['543', 2], 1) yield* this.markGarbageCollected(['543', 4], 1) yield* this.markGarbageCollected(['544', 2], 1) @@ -136,21 +136,21 @@ for (let database of databases) { })) it('Debug #7', async(function * (done) { store.requestTransaction(function * () { - yield* this.markDeleted(['9', 2]) - yield* this.markDeleted(['11', 2]) - yield* this.markDeleted(['11', 4]) - yield* this.markDeleted(['11', 1]) - yield* this.markDeleted(['9', 4]) - yield* this.markDeleted(['10', 0]) + yield* this.markDeleted(['9', 2], 1) + yield* this.markDeleted(['11', 2], 1) + yield* this.markDeleted(['11', 4], 1) + yield* this.markDeleted(['11', 1], 1) + yield* this.markDeleted(['9', 4], 1) + yield* this.markDeleted(['10', 0], 1) yield* this.markGarbageCollected(['11', 2], 1) - yield* this.markDeleted(['11', 2]) + yield* this.markDeleted(['11', 2], 1) yield* this.markGarbageCollected(['11', 3], 1) - yield* this.markDeleted(['11', 3]) - yield* this.markDeleted(['11', 3]) - yield* this.markDeleted(['9', 4]) - yield* this.markDeleted(['10', 0]) + yield* this.markDeleted(['11', 3], 1) + yield* this.markDeleted(['11', 3], 1) + yield* this.markDeleted(['9', 4], 1) + yield* this.markDeleted(['10', 0], 1) yield* this.markGarbageCollected(['11', 1], 1) - yield* this.markDeleted(['11', 1]) + yield* this.markDeleted(['11', 1], 1) expect(yield* this.getDeleteSet()).toEqual({'9': [[2, 1, false], [4, 1, false]], '10': [[0, 1, false]], '11': [[1, 3, true], [4, 1, false]]}) done() }) diff --git a/src/Struct.js b/src/Struct.js index 66381278..de0c5f54 100644 --- a/src/Struct.js +++ b/src/Struct.js @@ -127,7 +127,7 @@ module.exports = function (Y/* :any */) { var i // loop counter var distanceToOrigin = i = yield* Struct.Insert.getDistanceToOrigin.call(this, op) // most cases: 0 (starts from 0) - if (op.origin != null) { + if (op.origin != null) { // TODO: !== instead of != // we save in origin that op originates in it // we need that later when we eventually garbage collect origin (see transaction) var origin = yield* this.getOperation(op.origin) diff --git a/src/Transaction.js b/src/Transaction.js index 46ba388a..691afb1d 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -150,7 +150,7 @@ module.exports = function (Y/* :any */) { var callType = false if (target == null || !target.deleted) { - yield* this.markDeleted(targetId) + yield* this.markDeleted(targetId, 1) } if (target != null && target.gc == null) { @@ -274,7 +274,8 @@ module.exports = function (Y/* :any */) { * markDeleted (id, length) { if (length == null) { length = 1 - // debugger // TODO!! + debugger // TODO!! + throw new Error("dtrnd") } // this.mem.push(["del", id]); var n = yield* this.ds.findWithUpperBound(id) From dab72be87fec221886c5e78284f4677733048d0c Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 3 Feb 2016 11:37:36 +0100 Subject: [PATCH 32/43] update --- dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist b/dist index 5e295b80..6904e601 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 5e295b80d793748515957c5d309d2f8e8e83af29 +Subproject commit 6904e6018d63951c91fd28586570a6f9153dcdd3 From a5760a45bb3da9c976b4ce2b4670fd7bc851b91e Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 10:53:04 +0100 Subject: [PATCH 33/43] Release 0.8.11 --- dist | 2 +- package.json | 2 +- src/Transaction.js | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dist b/dist index 6904e601..6a4aabe6 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 6904e6018d63951c91fd28586570a6f9153dcdd3 +Subproject commit 6a4aabe69cce5165307e571e8fc99b01e8d9dac8 diff --git a/package.json b/package.json index b9a9c52c..ac517303 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.10", + "version": "0.8.12", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { diff --git a/src/Transaction.js b/src/Transaction.js index 691afb1d..310e804a 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -274,8 +274,6 @@ module.exports = function (Y/* :any */) { * markDeleted (id, length) { if (length == null) { length = 1 - debugger // TODO!! - throw new Error("dtrnd") } // this.mem.push(["del", id]); var n = yield* this.ds.findWithUpperBound(id) From d532fc530f5ff9fd77472c4fafd8634499c80504 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 12:06:06 +0100 Subject: [PATCH 34/43] update dist --- dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist b/dist index 6a4aabe6..4340f028 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 6a4aabe69cce5165307e571e8fc99b01e8d9dac8 +Subproject commit 4340f028b5a7e309cd14f6c97bbfb1a626a1330c From 608b5e33193c6de0dfd5bad6fc573a53efc628ba Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 12:12:57 +0100 Subject: [PATCH 35/43] Release 0.8.12 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index 4340f028..f9a9edcc 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 4340f028b5a7e309cd14f6c97bbfb1a626a1330c +Subproject commit f9a9edcc260146a9e6d1ff5ee215890fd0cbe39b diff --git a/package.json b/package.json index ac517303..c7572dc2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.12", + "version": "0.8.13", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 8924c3e1635da9c90d91f441a367b95320c01282 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 12:47:09 +0100 Subject: [PATCH 36/43] Release 0.8.13 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index f9a9edcc..ddbe19d7 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit f9a9edcc260146a9e6d1ff5ee215890fd0cbe39b +Subproject commit ddbe19d78e4a75f54988670e3d69a4b86ce9bdc4 diff --git a/package.json b/package.json index c7572dc2..03a7e261 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.13", + "version": "0.8.14", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 1da76dbc20a3ecb565aa83c26dbe5b181a04c0b9 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 12:53:39 +0100 Subject: [PATCH 37/43] update dist --- dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist b/dist index ddbe19d7..e48608a7 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit ddbe19d78e4a75f54988670e3d69a4b86ce9bdc4 +Subproject commit e48608a7d4d881bc2ef9f05c3d3a1f76f3cc699d From 7be262e9f3a88afcee5358ab1312214698bbf4dd Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 15:26:09 +0100 Subject: [PATCH 38/43] fixing @Joeao bug --- src/Transaction.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Transaction.js b/src/Transaction.js index 310e804a..26e42677 100644 --- a/src/Transaction.js +++ b/src/Transaction.js @@ -601,12 +601,11 @@ module.exports = function (Y/* :any */) { yield* this.setState(state) } } - } - if (this.store.forwardAppliedOperations) { - for (let c = del[1]; c < del[1] + del[2]; c++) { - var ops = deletions.map(function (d) { - return {struct: 'Delete', target: [d[0], c]} // TODO: implement Delete with deletion length! - }) + if (this.store.forwardAppliedOperations) { + var ops = [] + for (let c = del[1]; c < del[1] + del[2]; c++) { + ops.push({struct: 'Delete', target: [d[0], c]}) // TODO: implement Delete with deletion length! + } this.store.y.connector.broadcastOps(ops) } } From 56ba55cbab208d70079e49bb1e7ce3eef7c68d29 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 15:26:32 +0100 Subject: [PATCH 39/43] Release 0.8.14 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index e48608a7..98085b28 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit e48608a7d4d881bc2ef9f05c3d3a1f76f3cc699d +Subproject commit 98085b28074c0097904f194e043319951f35609c diff --git a/package.json b/package.json index 03a7e261..470e00e2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.14", + "version": "0.8.15", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From 5f2a81d0648dee8ca2a583e3afcd49bd74abe77d Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 23:08:29 +0100 Subject: [PATCH 40/43] updated api documentation --- README.md | 101 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 471a26f6..7ebeb098 100644 --- a/README.md +++ b/README.md @@ -3,22 +3,20 @@ Yjs is a framework for optimistic concurrency control and automatic conflict resolution on shared data. The framework provides similar functionality as [ShareJs] and [OpenCoweb], but supports peer-to-peer communication protocols by default. Yjs was designed to handle concurrent actions on arbitrary data like Text, Json, and XML. We also provide support for storing and manipulating your shared data offline. For more information and demo applications visit our [homepage](http://y-js.org/). -**NOTE** This project is currently migrating. So there may exist some information that is not true anymore.. - You can create you own shared types easily. Therefore, you can design the sturcture of your custom type, and ensure data validity, while Yjs ensures data consistency (everyone will eventually end up with the same data). -We already provide data types for +We already provide abstract data types for | Name | Description | |----------|-------------------| -|[map](https://github.com/y-js/y-map) | Add, update, and remove properties of an object. Included in Yjs| -|[array](https://github.com/y-js/y-array) | A shared linked list implementation | -|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects| -|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *h1*, or *p*)| -|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to several editors| +|[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | +|[array](https://github.com/y-js/y-array) | A shared Array implementation | +|[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | +|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *h1*, or *p*). Also supports the [Ace Editor](https://ace.c9.io) | +|[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| -Yjs supports P2P message propagation, and not bound to a specific communication protocol. Therefore, Yjs is extremely scalable and can be used in a wide range of application scenarios. +Yjs supports P2P message propagation, and is not bound to a specific communication protocol. Therefore, Yjs is extremely scalable and can be used in a wide range of application scenarios. We support several communication protocols as so called *Connectors*. You can create your own connector too - read [this wiki page](https://github.com/y-js/yjs/wiki/Custom-Connectors). @@ -36,13 +34,13 @@ You are not limited to use a specific database to store the shared data. We prov |Name | Description | |----------------|-----------------------------------| |[memory](https://github.com/y-js/y-memory) | In-memory storage. | -|[IndexedDb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | +|[indexeddb](https://github.com/y-js/y-indexeddb) | Offline storage for the browser | You can use Yjs client-, and server- side. You can get it as via npm, and bower. We even provide polymer elements for Yjs! The advantages over similar frameworks are support for * .. P2P message propagation and arbitrary communication protocols -* .. arbitrary complex data types +* .. share any type of data. The types provide a convenient interface * .. offline support: Changes are stored persistently and only relevant changes are propagated on rejoin * .. Intention Preservation: When working on Text, the intention of your changes are preserved. This is particularily important when working offline. Every type has a notion on how we define Intention Preservation on it. @@ -85,13 +83,92 @@ Y({ share: { textarea: 'Text' // y.share.textarea is of type Y.Text } - // modules: ['Richtext', 'Array'] // optional list of modules you want to import + // types: ['Richtext', 'Array'] // optional list of types you want to import }).then(function (y) { // bind the textarea to a shared text element y.share.textarea.bind(document.getElementById('textfield')) } ``` +# Api + +### Y(options) +* options.db + * Will be forwarded to the database adapter. Specify the database adaper on `options.db.name`. + * Have a look at the used database adapter repository to see all available options. +* options.connector + * Will be forwarded to the connector adapter. Specify the connector adaper on `options.connector.name`. + * All our connectors implement a `room` property. Clients that specify the same room share the same data. + * All of our connectors specify an `url` property that defines the connection endpoint of the used connector. + * All of our connectors also have a default connection endpoint that you can use for development. + * Have a look at the used connector repository to see all available options. +* options.sourceDir + * Path where all y-* modules are stored. + * Defaults to `/bower_components` + * Not required when running on `nodejs` / `iojs` + * When using browserify you can specify all used modules like this: +``` +var Y = require('yjs') +// you need to require the db, connector, and *all* types you use! +require('y-memory')(Y) +require('y-webrtc')(Y) +require('y-map')(Y) +// .. +``` +* options.share + * Specify on `options.share[arbitraryName]` types that are shared among all users. + * E.g. Specify `options.share[arbitraryName] = 'Array'` to require y-array and create an Y.Array type on `y.share[arbitraryName]`. + * If userA doesn't specify `options.share[arbitraryName]`, it won't be available for userA. + * If userB specifies `options.share[arbitraryName]`, it still won't be available for userA. But all the updates are send from userB to userA. + * In contrast to Y.Map, types on `y.share.*` cannot be overwritten or deleted. Instead, they are merged among all users. This feature is only available on `y.share.*` + * Weird behavior: It is supported that two users specify different types with the same property name. + E.g. userA specifies `options.share.x = 'Array'`, and userB specifies `options.share.x = 'Text'`. But they'll only share data if they specified the same type with the same property name +* options.type + * Array of modules that Yjs needs to require, before instantiating a shared type. + * By default Yjs requires the specified database adapter, the specified connector, and all modules that are used in `options.share.*` + * Put all types here that you intend to use, but are not used in y.share.* + +### Instantiated Y object (y) +`Y(options)` returns a promise that is fulfilled when.. + +* All modules are loaded + * The specified database adapter is loaded + * The specified connector is loaded + * All types are included +* The connector is initialized, and a unique user id is set (received from the server) + * Note: When using y-indexeddb, a retrieved user id is stored on `localStorage` + +The promise returns an instance of Y. We denote it with a lower case `y`. + +* y.share.* + * Instances of the types you specified on options.share.* + * y.share.* can only be defined once when you instantiate Y! +* y.connector is an instance of Y.AbstractConnector +* y.connector.onUserEvent(function (event) {..}) + * Observe user events (event.action is either 'userLeft' or 'userJoined') +* y.connector.whenSynced(listener) + * `listener` is executed when y synced with at least one user. + * `listener` is not called when no other user is in the same room. + * y-websockets-client aways waits to sync with the server +* y.connector.disconnect() + * Force to disconnect this instance from the other instances +* y.connector.reconnect() + * Try to reconnect to the other instances (needs to be supported by the connector) + * Not supported by y-xmpp +* y.destroy() + * Destroy this object. + * Destroys all types (they will throw weird errors if you still use them) + * Disconnects from the other instances (via connector) + * Removes all data from the database +* y.db.stopGarbageCollector() + * Stop the garbage collector. Call y.db.garbageCollect() to continue garbage collection +* y.db.gcTimeout :: Number (defaults to 50000 ms) + * Time interval between two garbage collect cycles + * It is required that all instances exchanged all messages after two garbage collect cycles (after 100000 ms per default) +* y.db.userId :: String + * The used user id for this client. ** Never overwrite this ** + + # Status Yjs is a work in progress. Different versions of the *y-* repositories may not work together. Just drop me a line if you run into troubles. From d20141fec12e40b11f10338dce79397e6506773c Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 23:11:11 +0100 Subject: [PATCH 41/43] Release 0.8.15 --- README.md | 4 ++-- dist | 2 +- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7ebeb098..be14aaa5 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ We already provide abstract data types for |[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | |[array](https://github.com/y-js/y-array) | A shared Array implementation | |[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | -|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *h1*, or *p*). Also supports the [Ace Editor](https://ace.c9.io) | +|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *

*, or *

*). Also supports the [Ace Editor](https://ace.c9.io) | |[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| Yjs supports P2P message propagation, and is not bound to a specific communication protocol. Therefore, Yjs is extremely scalable and can be used in a wide range of application scenarios. @@ -166,7 +166,7 @@ The promise returns an instance of Y. We denote it with a lower case `y`. * Time interval between two garbage collect cycles * It is required that all instances exchanged all messages after two garbage collect cycles (after 100000 ms per default) * y.db.userId :: String - * The used user id for this client. ** Never overwrite this ** + * The used user id for this client. **Never overwrite this** # Status diff --git a/dist b/dist index 98085b28..e74c22f0 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 98085b28074c0097904f194e043319951f35609c +Subproject commit e74c22f0a9b269a9f673c595b5c04ab570e04bdd diff --git a/package.json b/package.json index 470e00e2..13dff30c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.15", + "version": "0.8.16", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From b40b7e10ab1162c01d3977a70a2bea511d26bd91 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 23:12:53 +0100 Subject: [PATCH 42/43] Release 0.8.16 --- dist | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist b/dist index e74c22f0..941dd36e 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit e74c22f0a9b269a9f673c595b5c04ab570e04bdd +Subproject commit 941dd36e74b3c3ec16e4dc5b18343112f98e5142 diff --git a/package.json b/package.json index 13dff30c..5eb5ac9f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.16", + "version": "0.8.17", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": { From bca7477ca58b7a2f13908ab8d0c2f4ae3cde9a38 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 4 Feb 2016 23:13:51 +0100 Subject: [PATCH 43/43] Release 0.8.17 --- README.md | 2 +- dist | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index be14aaa5..cc946f02 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ We already provide abstract data types for |[map](https://github.com/y-js/y-map) | A shared Map implementation. Maps from text to any stringify-able object | |[array](https://github.com/y-js/y-array) | A shared Array implementation | |[xml](https://github.com/y-js/y-xml) | An implementation of the DOM. You can create a two way binding to Browser DOM objects | -|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. *

*, or *

*). Also supports the [Ace Editor](https://ace.c9.io) | +|[text](https://github.com/y-js/y-text) | Collaborate on text. Supports two way binding to textareas, input elements, or HTML elements (e.g. <*h1*>, or <*p*>). Also supports the [Ace Editor](https://ace.c9.io) | |[richtext](https://github.com/y-js/y-richtext) | Collaborate on rich text. Supports two way binding to the [Quill Rich Text Editor](http://quilljs.com/)| Yjs supports P2P message propagation, and is not bound to a specific communication protocol. Therefore, Yjs is extremely scalable and can be used in a wide range of application scenarios. diff --git a/dist b/dist index 941dd36e..ef6d63c1 160000 --- a/dist +++ b/dist @@ -1 +1 @@ -Subproject commit 941dd36e74b3c3ec16e4dc5b18343112f98e5142 +Subproject commit ef6d63c19af25d7d1f09f83b0487f0edf5bfe196 diff --git a/package.json b/package.json index 5eb5ac9f..7d3a0346 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yjs", - "version": "0.8.17", + "version": "0.8.18", "description": "A framework for real-time p2p shared editing on arbitrary complex data types", "main": "./src/y.js", "scripts": {