diff --git a/examples/chat/index.html b/examples/chat/index.html index 51acbab2..20f5417f 100644 --- a/examples/chat/index.html +++ b/examples/chat/index.html @@ -13,10 +13,6 @@ - - - - diff --git a/examples/chat/index.js b/examples/chat/index.js index c9f9a990..c7717770 100644 --- a/examples/chat/index.js +++ b/examples/chat/index.js @@ -1,73 +1,71 @@ -/* global Y, chat */ +/* global Y */ // initialize a shared object. This function call returns a promise! -Y({ - db: { - name: 'memory' - }, +var y = new Y({ connector: { name: 'websockets-client', room: 'chat-example' - }, - sourceDir: '/bower_components', - share: { - chat: 'Array' - } -}).then(function (y) { - window.yChat = y - // This functions inserts a message at the specified position in the DOM - function appendMessage (message, position) { - var p = document.createElement('p') - var uname = document.createElement('span') - uname.appendChild(document.createTextNode(message.username + ': ')) - p.appendChild(uname) - p.appendChild(document.createTextNode(message.message)) - document.querySelector('#chat').insertBefore(p, chat.children[position] || null) - } - // This function makes sure that only 7 messages exist in the chat history. - // The rest is deleted - function cleanupChat () { - if (y.share.chat.length > 7) { - y.share.chat.delete(0, y.chat.length - 7) - } - } - // Insert the initial content - y.share.chat.toArray().forEach(appendMessage) - cleanupChat() - - // whenever content changes, make sure to reflect the changes in the DOM - y.share.chat.observe(function (event) { - if (event.type === 'insert') { - for (let i = 0; i < event.length; i++) { - appendMessage(event.values[i], event.index + i) - } - } else if (event.type === 'delete') { - for (let i = 0; i < event.length; i++) { - chat.children[event.index].remove() - } - } - // concurrent insertions may result in a history > 7, so cleanup here - cleanupChat() - }) - document.querySelector('#chatform').onsubmit = function (event) { - // the form is submitted - var message = { - username: this.querySelector('[name=username]').value, - message: this.querySelector('[name=message]').value - } - if (message.username.length > 0 && message.message.length > 0) { - if (y.share.chat.length > 6) { - // If we are goint to insert the 8th element, make sure to delete first. - y.share.chat.delete(0) - } - // Here we insert a message in the shared chat type. - // This will call the observe function (see line 40) - // and reflect the change in the DOM - y.share.chat.push([message]) - this.querySelector('[name=message]').value = '' - } - // Do not send this form! - event.preventDefault() - return false } }) + +window.yChat = y + +let chatprotocol = y.get('chatprotocol', Y.Array) + +let chatcontainer = document.querySelector('#chat') + +// This functions inserts a message at the specified position in the DOM +function appendMessage (message, position) { + var p = document.createElement('p') + var uname = document.createElement('span') + uname.appendChild(document.createTextNode(message.username + ': ')) + p.appendChild(uname) + p.appendChild(document.createTextNode(message.message)) + chatcontainer.insertBefore(p, chatcontainer.children[position] || null) +} +// This function makes sure that only 7 messages exist in the chat history. +// The rest is deleted +function cleanupChat () { + if (chatprotocol.length > 7) { + chatprotocol.delete(0, chatprotocol.length - 7) + } +} +// Insert the initial content +chatprotocol.toArray().forEach(appendMessage) +cleanupChat() + +// whenever content changes, make sure to reflect the changes in the DOM +chatprotocol.observe(function (event) { + if (event.type === 'insert') { + for (let i = 0; i < event.length; i++) { + appendMessage(event.values[i], event.index + i) + } + } else if (event.type === 'delete') { + for (let i = 0; i < event.length; i++) { + chatcontainer.children[event.index].remove() + } + } + // concurrent insertions may result in a history > 7, so cleanup here + cleanupChat() +}) +document.querySelector('#chatform').onsubmit = function (event) { + // the form is submitted + var message = { + username: this.querySelector('[name=username]').value, + message: this.querySelector('[name=message]').value + } + if (message.username.length > 0 && message.message.length > 0) { + if (chatprotocol.length > 6) { + // If we are goint to insert the 8th element, make sure to delete first. + chatprotocol.delete(0) + } + // Here we insert a message in the shared chat type. + // This will call the observe function (see line 40) + // and reflect the change in the DOM + chatprotocol.push([message]) + this.querySelector('[name=message]').value = '' + } + // Do not send this form! + event.preventDefault() + return false +} diff --git a/src/Encoding.js b/src/Binary/Decoder.js similarity index 52% rename from src/Encoding.js rename to src/Binary/Decoder.js index 6bbbc292..8bd5fe83 100644 --- a/src/Encoding.js +++ b/src/Binary/Decoder.js @@ -1,85 +1,6 @@ import utf8 from 'utf-8' -const bits7 = 0b1111111 -const bits8 = 0b11111111 - -export class BinaryEncoder { - constructor () { - this.data = [] - } - - get length () { - return this.data.length - } - - get pos () { - return this.data.length - } - - createBuffer () { - return Uint8Array.from(this.data).buffer - } - - writeUint8 (num) { - this.data.push(num & bits8) - } - - setUint8 (pos, num) { - this.data[pos] = num & bits8 - } - - writeUint16 (num) { - this.data.push(num & bits8, (num >>> 8) & bits8) - } - - setUint16 (pos, num) { - this.data[pos] = num & bits8 - this.data[pos + 1] = (num >>> 8) & bits8 - } - - writeUint32 (num) { - for (let i = 0; i < 4; i++) { - this.data.push(num & bits8) - num >>>= 8 - } - } - - setUint32 (pos, num) { - for (let i = 0; i < 4; i++) { - this.data[pos + i] = num & bits8 - num >>>= 8 - } - } - - writeVarUint (num) { - while (num >= 0b10000000) { - this.data.push(0b10000000 | (bits7 & num)) - num >>>= 7 - } - this.data.push(bits7 & num) - } - - writeVarString (str) { - let bytes = utf8.setBytesFromString(str) - let len = bytes.length - this.writeVarUint(len) - for (let i = 0; i < len; i++) { - this.data.push(bytes[i]) - } - } - - writeOpID (id) { - let user = id[0] - this.writeVarUint(user) - if (user !== 0xFFFFFF) { - this.writeVarUint(id[1]) - } else { - this.writeVarString(id[1]) - } - } -} - -export class BinaryDecoder { +export default class BinaryDecoder { constructor (buffer) { if (buffer instanceof ArrayBuffer) { this.uint8arr = new Uint8Array(buffer) @@ -91,6 +12,16 @@ export class BinaryDecoder { this.pos = 0 } + clone (newPos = this.pos) { + let decoder = new BinaryDecoder(this.uint8arr) + decoder.pos = newPos + return decoder + } + + get length () { + return this.uint8arr.length + } + skip8 () { this.pos++ } @@ -118,7 +49,7 @@ export class BinaryDecoder { let len = 0 while (true) { let r = this.uint8arr[this.pos++] - num = num | ((r & bits7) << len) + num = num | ((r & 0b1111111) << len) len += 7 if (r < 1 << 7) { return num >>> 0 // return unsigned number! diff --git a/src/Binary/Encoder.js b/src/Binary/Encoder.js new file mode 100644 index 00000000..dfa62a5f --- /dev/null +++ b/src/Binary/Encoder.js @@ -0,0 +1,80 @@ +import utf8 from 'utf-8' + +const bits7 = 0b1111111 +const bits8 = 0b11111111 + +export default class BinaryEncoder { + constructor () { + this.data = [] + } + + get length () { + return this.data.length + } + + get pos () { + return this.data.length + } + + createBuffer () { + return Uint8Array.from(this.data).buffer + } + + writeUint8 (num) { + this.data.push(num & bits8) + } + + setUint8 (pos, num) { + this.data[pos] = num & bits8 + } + + writeUint16 (num) { + this.data.push(num & bits8, (num >>> 8) & bits8) + } + + setUint16 (pos, num) { + this.data[pos] = num & bits8 + this.data[pos + 1] = (num >>> 8) & bits8 + } + + writeUint32 (num) { + for (let i = 0; i < 4; i++) { + this.data.push(num & bits8) + num >>>= 8 + } + } + + setUint32 (pos, num) { + for (let i = 0; i < 4; i++) { + this.data[pos + i] = num & bits8 + num >>>= 8 + } + } + + writeVarUint (num) { + while (num >= 0b10000000) { + this.data.push(0b10000000 | (bits7 & num)) + num >>>= 7 + } + this.data.push(bits7 & num) + } + + writeVarString (str) { + let bytes = utf8.setBytesFromString(str) + let len = bytes.length + this.writeVarUint(len) + for (let i = 0; i < len; i++) { + this.data.push(bytes[i]) + } + } + + writeOpID (id) { + let user = id[0] + this.writeVarUint(user) + if (user !== 0xFFFFFF) { + this.writeVarUint(id[1]) + } else { + this.writeVarString(id[1]) + } + } +} diff --git a/src/Connector.js b/src/Connector.js index 5bcba919..85ffa927 100644 --- a/src/Connector.js +++ b/src/Connector.js @@ -1,318 +1,268 @@ import { BinaryEncoder, BinaryDecoder } from './Encoding.js' -import { sendSyncStep1, computeMessageSyncStep1, computeMessageSyncStep2, computeMessageUpdate } from './MessageHandler.js' -export default function extendConnector (Y/* :any */) { - class AbstractConnector { - /* - opts contains the following information: - role : String Role of this client ("master" or "slave") - */ - constructor (y, opts) { - this.y = y - if (opts == null) { - opts = {} - } - this.opts = opts - // Prefer to receive untransformed operations. This does only work if - // this client receives operations from only one other client. - // In particular, this does not work with y-webrtc. - // It will work with y-websockets-client - this.preferUntransformed = opts.preferUntransformed || false - if (opts.role == null || opts.role === 'master') { - this.role = 'master' - } else if (opts.role === 'slave') { - this.role = 'slave' - } else { - throw new Error("Role must be either 'master' or 'slave'!") - } - this.log = Y.debug('y:connector') - this.logMessage = Y.debug('y:connector-message') - this.y.db.forwardAppliedOperations = opts.forwardAppliedOperations || false - this.role = opts.role - this.connections = new Map() - this.isSynced = false - this.userEventListeners = [] - this.whenSyncedListeners = [] - this.currentSyncTarget = null - this.debug = opts.debug === true - this.broadcastOpBuffer = [] - this.protocolVersion = 11 - this.authInfo = opts.auth || null - this.checkAuth = opts.checkAuth || function () { return Promise.resolve('write') } // default is everyone has write access - if (opts.generateUserId !== false) { - this.setUserId(Y.utils.generateUserId()) - } - if (opts.maxBufferLength == null) { - this.maxBufferLength = -1 - } else { - this.maxBufferLength = opts.maxBufferLength - } +import { sendSyncStep1, readSyncStep1 } from './MessageHandler/syncStep1' +import { readSyncStep2 } from './MessageHandler/syncStep2' +import { readUpdate } from './MessageHandler/update.js' + +import debug from 'debug' + +export default class AbstractConnector { + constructor (y, opts) { + this.y = y + this.opts = opts + if (opts.role == null || opts.role === 'master') { + this.role = 'master' + } else if (opts.role === 'slave') { + this.role = 'slave' + } else { + throw new Error("Role must be either 'master' or 'slave'!") } - - reconnect () { - this.log('reconnecting..') - return this.y.db.startGarbageCollector() + this.log = debug('y:connector') + this.logMessage = debug('y:connector-message') + this._forwardAppliedStructs = opts.forwardAppliedOperations || false // TODO: rename + this.role = opts.role + this.connections = new Map() + this.isSynced = false + this.userEventListeners = [] + this.whenSyncedListeners = [] + this.currentSyncTarget = null + this.debug = opts.debug === true + this.broadcastBuffer = new BinaryEncoder() + this.protocolVersion = 11 + this.authInfo = opts.auth || null + this.checkAuth = opts.checkAuth || function () { return Promise.resolve('write') } // default is everyone has write access + if (opts.maxBufferLength == null) { + this.maxBufferLength = -1 + } else { + this.maxBufferLength = opts.maxBufferLength } + } - disconnect () { - this.log('discronnecting..') - this.connections = new Map() - this.isSynced = false - this.currentSyncTarget = null - this.whenSyncedListeners = [] - this.y.db.stopGarbageCollector() - return this.y.db.whenTransactionsFinished() - } + reconnect () { + this.log('reconnecting..') + return this.y.db.startGarbageCollector() + } - repair () { - this.log('Repairing the state of Yjs. This can happen if messages get lost, and Yjs detects that something is wrong. If this happens often, please report an issue here: https://github.com/y-js/yjs/issues') - this.isSynced = false - this.connections.forEach((user, userId) => { - user.isSynced = false - this._syncWithUser(userId) - }) - } + disconnect () { + this.log('discronnecting..') + this.connections = new Map() + this.isSynced = false + this.currentSyncTarget = null + this.whenSyncedListeners = [] + return Promise.resolve() + } - setUserId (userId) { - if (this.userId == null) { - if (!Number.isInteger(userId)) { - let err = new Error('UserId must be an integer!') - this.y.emit('error', err) - throw err - } - this.log('Set userId to "%s"', userId) - this.userId = userId - return this.y.db.setUserId(userId) - } else { - return null - } - } + onUserEvent (f) { + this.userEventListeners.push(f) + } - onUserEvent (f) { - this.userEventListeners.push(f) - } + removeUserEventListener (f) { + this.userEventListeners = this.userEventListeners.filter(g => f !== g) + } - removeUserEventListener (f) { - this.userEventListeners = this.userEventListeners.filter(g => f !== g) - } - - userLeft (user) { - if (this.connections.has(user)) { - this.log('%s: User left %s', this.userId, user) - this.connections.delete(user) - // check if isSynced event can be sent now - this._setSyncedWith(null) - for (var f of this.userEventListeners) { - f({ - action: 'userLeft', - user: user - }) - } - } - } - - userJoined (user, role, auth) { - if (role == null) { - throw new Error('You must specify the role of the joined user!') - } - if (this.connections.has(user)) { - throw new Error('This user already joined!') - } - this.log('%s: User joined %s', this.userId, user) - this.connections.set(user, { - uid: user, - isSynced: false, - role: role, - processAfterAuth: [], - auth: auth || null, - receivedSyncStep2: false - }) - let defer = {} - defer.promise = new Promise(function (resolve) { defer.resolve = resolve }) - this.connections.get(user).syncStep2 = defer + userLeft (user) { + if (this.connections.has(user)) { + this.log('%s: User left %s', this.userId, user) + this.connections.delete(user) + // check if isSynced event can be sent now + this._setSyncedWith(null) for (var f of this.userEventListeners) { f({ - action: 'userJoined', - user: user, - role: role + action: 'userLeft', + user: user }) } - this._syncWithUser(user) - } - - // Execute a function _when_ we are connected. - // If not connected, wait until connected - whenSynced (f) { - if (this.isSynced) { - f() - } else { - this.whenSyncedListeners.push(f) - } - } - - _syncWithUser (userid) { - if (this.role === 'slave') { - return // "The current sync has not finished or this is controlled by a master!" - } - sendSyncStep1(this, userid) - } - - _fireIsSyncedListeners () { - this.y.db.whenTransactionsFinished().then(() => { - if (!this.isSynced) { - this.isSynced = true - // It is safer to remove this! - // TODO: remove: this.garbageCollectAfterSync() - // call whensynced listeners - for (var f of this.whenSyncedListeners) { - f() - } - this.whenSyncedListeners = [] - } - }) - } - - send (uid, buffer) { - if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) { - throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages') - } - this.log('%s: Send \'%y\' to %s', this.userId, buffer, uid) - this.logMessage('Message: %Y', buffer) - } - - broadcast (buffer) { - if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) { - throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages') - } - this.log('%s: Broadcast \'%y\'', this.userId, buffer) - this.logMessage('Message: %Y', buffer) - } - - /* - Buffer operations, and broadcast them when ready. - */ - broadcastOps (ops) { - ops = ops.map(function (op) { - return Y.Struct[op.struct].encode(op) - }) - var self = this - function broadcastOperations () { - if (self.broadcastOpBuffer.length > 0) { - let encoder = new BinaryEncoder() - encoder.writeVarString(self.opts.room) - encoder.writeVarString('update') - let ops = self.broadcastOpBuffer - let length = ops.length - let encoderPosLen = encoder.pos - encoder.writeUint32(0) - for (var i = 0; i < length && (self.maxBufferLength < 0 || encoder.length < self.maxBufferLength); i++) { - let op = ops[i] - Y.Struct[op.struct].binaryEncode(encoder, op) - } - encoder.setUint32(encoderPosLen, i) - self.broadcastOpBuffer = ops.slice(i) - self.broadcast(encoder.createBuffer()) - if (i !== length) { - self.whenRemoteResponsive().then(broadcastOperations) - } - } - } - if (this.broadcastOpBuffer.length === 0) { - this.broadcastOpBuffer = ops - this.y.db.whenTransactionsFinished().then(broadcastOperations) - } else { - this.broadcastOpBuffer = this.broadcastOpBuffer.concat(ops) - } - } - - /* - * Somehow check the responsiveness of the remote clients/server - * Default behavior: - * Wait 100ms before broadcasting the next batch of operations - * - * Only used when maxBufferLength is set - * - */ - whenRemoteResponsive () { - return new Promise(function (resolve) { - setTimeout(resolve, 100) - }) - } - - /* - You received a raw message, and you know that it is intended for Yjs. Then call this function. - */ - receiveMessage (sender, buffer, skipAuth) { - skipAuth = skipAuth || false - if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) { - return Promise.reject(new Error('Expected Message to be an ArrayBuffer or Uint8Array!')) - } - if (sender === this.userId) { - return Promise.resolve() - } - let decoder = new BinaryDecoder(buffer) - let encoder = new BinaryEncoder() - let roomname = decoder.readVarString() // read room name - encoder.writeVarString(roomname) - let messageType = decoder.readVarString() - let senderConn = this.connections.get(sender) - this.log('%s: Receive \'%s\' from %s', this.userId, messageType, sender) - this.logMessage('Message: %Y', buffer) - if (senderConn == null && !skipAuth) { - throw new Error('Received message from unknown peer!') - } - if (messageType === 'sync step 1' || messageType === 'sync step 2') { - let auth = decoder.readVarUint() - if (senderConn.auth == null) { - senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender]) - // check auth - return this.checkAuth(auth, this.y, sender).then(authPermissions => { - if (senderConn.auth == null) { - senderConn.auth = authPermissions - this.y.emit('userAuthenticated', { - user: senderConn.uid, - auth: authPermissions - }) - } - let messages = senderConn.processAfterAuth - senderConn.processAfterAuth = [] - - return messages.reduce((p, m) => - p.then(() => this.computeMessage(m[0], m[1], m[2], m[3], m[4])) - , Promise.resolve()) - }) - } - } - if (skipAuth || senderConn.auth != null) { - return this.computeMessage(messageType, senderConn, decoder, encoder, sender, skipAuth) - } else { - senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender, false]) - } - } - - computeMessage (messageType, senderConn, decoder, encoder, sender, skipAuth) { - if (messageType === 'sync step 1' && (senderConn.auth === 'write' || senderConn.auth === 'read')) { - // cannot wait for sync step 1 to finish, because we may wait for sync step 2 in sync step 1 (->lock) - computeMessageSyncStep1(decoder, encoder, this, senderConn, sender) - return this.y.db.whenTransactionsFinished() - } else if (messageType === 'sync step 2' && senderConn.auth === 'write') { - return computeMessageSyncStep2(decoder, encoder, this, senderConn, sender) - } else if (messageType === 'update' && (skipAuth || senderConn.auth === 'write')) { - return computeMessageUpdate(decoder, encoder, this, senderConn, sender) - } else { - return Promise.reject(new Error('Unable to receive message')) - } - } - - _setSyncedWith (user) { - if (user != null) { - this.connections.get(user).isSynced = true - } - let conns = Array.from(this.connections.values()) - if (conns.length > 0 && conns.every(u => u.isSynced)) { - this._fireIsSyncedListeners() - } } } - Y.AbstractConnector = AbstractConnector + + userJoined (user, role, auth) { + if (role == null) { + throw new Error('You must specify the role of the joined user!') + } + if (this.connections.has(user)) { + throw new Error('This user already joined!') + } + this.log('%s: User joined %s', this.userId, user) + this.connections.set(user, { + uid: user, + isSynced: false, + role: role, + processAfterAuth: [], + auth: auth || null, + receivedSyncStep2: false + }) + let defer = {} + defer.promise = new Promise(function (resolve) { defer.resolve = resolve }) + this.connections.get(user).syncStep2 = defer + for (var f of this.userEventListeners) { + f({ + action: 'userJoined', + user: user, + role: role + }) + } + this._syncWithUser(user) + } + + // Execute a function _when_ we are connected. + // If not connected, wait until connected + whenSynced (f) { + if (this.isSynced) { + f() + } else { + this.whenSyncedListeners.push(f) + } + } + + _syncWithUser (userid) { + if (this.role === 'slave') { + return // "The current sync has not finished or this is controlled by a master!" + } + sendSyncStep1(this, userid) + } + + _fireIsSyncedListeners () { + new Promise().then(() => { + if (!this.isSynced) { + this.isSynced = true + // It is safer to remove this! + // TODO: remove: this.garbageCollectAfterSync() + // call whensynced listeners + for (var f of this.whenSyncedListeners) { + f() + } + this.whenSyncedListeners = [] + } + }) + } + + send (uid, buffer) { + if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) { + throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - don\'t use this method to send custom messages') + } + this.log('%s: Send \'%y\' to %s', this.userId, buffer, uid) + this.logMessage('Message: %Y', buffer) + } + + broadcast (buffer) { + if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) { + throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - don\'t use this method to send custom messages') + } + this.log('%s: Broadcast \'%y\'', this.userId, buffer) + this.logMessage('Message: %Y', buffer) + } + + /* + Buffer operations, and broadcast them when ready. + */ + broadcastStruct (struct) { + let firstContent = this.broadcastBuffer.length === 0 + struct._toBinary(this.broadcastBuffer) + if (this.maxBufferLength > 0 && this.broadcastBuffer.length > this.maxBufferLength) { + // it is necessary to send the buffer now + // cache the buffer and check if server is responsive + let buffer = this.broadcastBuffer + this.broadcastBuffer = new BinaryEncoder() + this.whenRemoteResponsive().then(() => { + this.broadcast(buffer) + }) + } else if (firstContent) { + // send the buffer when all transactions are finished + // (or buffer exceeds maxBufferLength) + setTimeout(() => { + if (this.broadcastBuffer.length > 0) { + this.broadcast(this.broadcastBuffer) + this.broadcastBuffer = new BinaryEncoder() + } + }) + } + } + + /* + * Somehow check the responsiveness of the remote clients/server + * Default behavior: + * Wait 100ms before broadcasting the next batch of operations + * + * Only used when maxBufferLength is set + * + */ + whenRemoteResponsive () { + return new Promise(function (resolve) { + setTimeout(resolve, 100) + }) + } + + /* + You received a raw message, and you know that it is intended for Yjs. Then call this function. + */ + receiveMessage (sender, buffer, skipAuth) { + skipAuth = skipAuth || false + if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) { + return Promise.reject(new Error('Expected Message to be an ArrayBuffer or Uint8Array!')) + } + if (sender === this.userId) { + return Promise.resolve() + } + let decoder = new BinaryDecoder(buffer) + let encoder = new BinaryEncoder() + let roomname = decoder.readVarString() // read room name + encoder.writeVarString(roomname) + let messageType = decoder.readVarString() + let senderConn = this.connections.get(sender) + this.log('%s: Receive \'%s\' from %s', this.userId, messageType, sender) + this.logMessage('Message: %Y', buffer) + if (senderConn == null && !skipAuth) { + throw new Error('Received message from unknown peer!') + } + if (messageType === 'sync step 1' || messageType === 'sync step 2') { + let auth = decoder.readVarUint() + if (senderConn.auth == null) { + senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender]) + // check auth + return this.checkAuth(auth, this.y, sender).then(authPermissions => { + if (senderConn.auth == null) { + senderConn.auth = authPermissions + this.y.emit('userAuthenticated', { + user: senderConn.uid, + auth: authPermissions + }) + } + let messages = senderConn.processAfterAuth + senderConn.processAfterAuth = [] + + return messages.reduce((p, m) => + p.then(() => this.computeMessage(m[0], m[1], m[2], m[3], m[4])) + , Promise.resolve()) + }) + } + } + if (skipAuth || senderConn.auth != null) { + return this.computeMessage(messageType, senderConn, decoder, encoder, sender, skipAuth) + } else { + senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender, false]) + } + } + + computeMessage (messageType, senderConn, decoder, encoder, sender, skipAuth) { + if (messageType === 'sync step 1' && (senderConn.auth === 'write' || senderConn.auth === 'read')) { + // cannot wait for sync step 1 to finish, because we may wait for sync step 2 in sync step 1 (->lock) + readSyncStep1()(decoder, encoder, this.y, senderConn, sender) + } else if (messageType === 'sync step 2' && senderConn.auth === 'write') { + readSyncStep2(decoder, encoder, this.y, senderConn, sender) + } else if (messageType === 'update' && (skipAuth || senderConn.auth === 'write')) { + readUpdate(decoder, encoder, this.y, senderConn, sender) + } else { + throw new Error('Unable to receive message') + } + } + + _setSyncedWith (user) { + if (user != null) { + this.connections.get(user).isSynced = true + } + let conns = Array.from(this.connections.values()) + if (conns.length > 0 && conns.every(u => u.isSynced)) { + this._fireIsSyncedListeners() + } + } } diff --git a/src/Database.js b/src/Database.js deleted file mode 100644 index 8933793d..00000000 --- a/src/Database.js +++ /dev/null @@ -1,609 +0,0 @@ -/* @flow */ -'use strict' - -export default function extendDatabase (Y /* :any */) { - /* - Partial definition of an OperationStore. - TODO: name it Database, operation store only holds operations. - - A database definition must alse define the following methods: - * logTable() (optional) - - show relevant information information in a table - * requestTransaction(makeGen) - - request a transaction - * destroy() - - destroy the database - */ - class AbstractDatabase { - /* :: - y: YConfig; - forwardAppliedOperations: boolean; - listenersById: Object; - listenersByIdExecuteNow: Array; - listenersByIdRequestPending: boolean; - initializedTypes: Object; - whenUserIdSetListener: ?Function; - waitingTransactions: Array; - transactionInProgress: boolean; - executeOrder: Array; - gc1: Array; - gc2: Array; - gcTimeout: number; - gcInterval: any; - garbageCollect: Function; - executeOrder: Array; // for debugging only - userId: UserId; - opClock: number; - transactionsFinished: ?{promise: Promise, resolve: any}; - transact: (x: ?Generator) => any; - */ - constructor (y, opts) { - this.y = y - opts.gc = opts.gc === true - this.dbOpts = opts - var os = this - this.userId = null - var resolve_ - this.userIdPromise = new Promise(function (resolve) { - resolve_ = resolve - }) - this.userIdPromise.resolve = resolve_ - // whether to broadcast all applied operations (insert & delete hook) - this.forwardAppliedOperations = false - // E.g. this.listenersById[id] : Array - this.listenersById = {} - // Execute the next time a transaction is requested - this.listenersByIdExecuteNow = [] - // A transaction is requested - this.listenersByIdRequestPending = false - /* To make things more clear, the following naming conventions: - * ls : we put this.listenersById on ls - * l : Array - * id : Id (can't use as property name) - * sid : String (converted from id via JSON.stringify - so we can use it as a property name) - - Always remember to first overwrite - a property before you iterate over it! - */ - // TODO: Use ES7 Weak Maps. This way types that are no longer user, - // wont be kept in memory. - this.initializedTypes = {} - this.waitingTransactions = [] - this.transactionInProgress = false - this.transactionIsFlushed = false - if (typeof YConcurrencyTestingMode !== 'undefined') { - this.executeOrder = [] - } - this.gc1 = [] // first stage - this.gc2 = [] // second stage -> after that, remove the op - - function garbageCollect () { - return os.whenTransactionsFinished().then(function () { - if (os.gcTimeout > 0 && (os.gc1.length > 0 || os.gc2.length > 0)) { - if (!os.y.connector.isSynced) { - console.warn('gc should be empty when not synced!') - } - return new Promise((resolve) => { - os.requestTransaction(function () { - if (os.y.connector != null && os.y.connector.isSynced) { - for (var i = 0; i < os.gc2.length; i++) { - var oid = os.gc2[i] - this.garbageCollectOperation(oid) - } - os.gc2 = os.gc1 - os.gc1 = [] - } - // TODO: Use setInterval here instead (when garbageCollect is called several times there will be several timeouts..) - if (os.gcTimeout > 0) { - os.gcInterval = setTimeout(garbageCollect, os.gcTimeout) - } - resolve() - }) - }) - } else { - // TODO: see above - if (os.gcTimeout > 0) { - os.gcInterval = setTimeout(garbageCollect, os.gcTimeout) - } - return Promise.resolve() - } - }) - } - this.garbageCollect = garbageCollect - this.startGarbageCollector() - - this.repairCheckInterval = !opts.repairCheckInterval ? 6000 : opts.repairCheckInterval - this.opsReceivedTimestamp = new Date() - this.startRepairCheck() - } - startGarbageCollector () { - this.gc = this.dbOpts.gc - if (this.gc) { - this.gcTimeout = !this.dbOpts.gcTimeout ? 30000 : this.dbOpts.gcTimeout - } else { - this.gcTimeout = -1 - } - if (this.gcTimeout > 0) { - this.garbageCollect() - } - } - startRepairCheck () { - var os = this - if (this.repairCheckInterval > 0) { - this.repairCheckIntervalHandler = setInterval(function repairOnMissingOperations () { - /* - Case 1. No ops have been received in a while (new Date() - os.opsReceivedTimestamp > os.repairCheckInterval) - - 1.1 os.listenersById is empty. Then the state was correct the whole time. -> Nothing to do (nor to update) - - 1.2 os.listenersById is not empty. - * Then the state was incorrect for at least {os.repairCheckInterval} seconds. - * -> Remove everything in os.listenersById and sync again (connector.repair()) - Case 2. An op has been received in the last {os.repairCheckInterval } seconds. - It is not yet necessary to check for faulty behavior. Everything can still resolve itself. Wait for more messages. - If nothing was received for a while and os.listenersById is still not emty, we are in case 1.2 - -> Do nothing - - Baseline here is: we really only have to catch case 1.2.. - */ - if ( - new Date() - os.opsReceivedTimestamp > os.repairCheckInterval && - Object.keys(os.listenersById).length > 0 // os.listenersById is not empty - ) { - // haven't received operations for over {os.repairCheckInterval} seconds, resend state vector - os.listenersById = {} - os.opsReceivedTimestamp = new Date() // update so you don't send repair several times in a row - os.y.connector.repair() - } - }, this.repairCheckInterval) - } - } - stopRepairCheck () { - clearInterval(this.repairCheckIntervalHandler) - } - queueGarbageCollector (id) { - if (this.y.connector.isSynced && this.gc) { - this.gc1.push(id) - } - } - emptyGarbageCollector () { - return new Promise(resolve => { - var check = () => { - if (this.gc1.length > 0 || this.gc2.length > 0) { - this.garbageCollect().then(check) - } else { - resolve() - } - } - setTimeout(check, 0) - }) - } - addToDebug () { - if (typeof YConcurrencyTestingMode !== 'undefined') { - var command /* :string */ = Array.prototype.map.call(arguments, function (s) { - if (typeof s === 'string') { - return s - } else { - return JSON.stringify(s) - } - }).join('').replace(/"/g, "'").replace(/,/g, ', ').replace(/:/g, ': ') - this.executeOrder.push(command) - } - } - getDebugData () { - console.log(this.executeOrder.join('\n')) - } - stopGarbageCollector () { - var self = this - this.gc = false - this.gcTimeout = -1 - return new Promise(function (resolve) { - self.requestTransaction(function () { - var ungc /* :Array */ = self.gc1.concat(self.gc2) - self.gc1 = [] - self.gc2 = [] - for (var i = 0; i < ungc.length; i++) { - var op = this.getOperation(ungc[i]) - if (op != null) { - delete op.gc - this.setOperation(op) - } - } - resolve() - }) - }) - } - /* - Try to add to GC. - - TODO: rename this function - - Rulez: - * Only gc if this user is online & gc turned on - * The most left element in a list must not be gc'd. - => There is at least one element in the list - - returns true iff op was added to GC - */ - addToGarbageCollector (op, left) { - if ( - op.gc == null && - op.deleted === true && - this.store.gc && - this.store.y.connector.isSynced - ) { - var gc = false - if (left != null && left.deleted === true) { - gc = true - } else if (op.content != null && op.content.length > 1) { - op = this.getInsertionCleanStart([op.id[0], op.id[1] + 1]) - gc = true - } - if (gc) { - op.gc = true - this.setOperation(op) - this.store.queueGarbageCollector(op.id) - return true - } - } - return false - } - removeFromGarbageCollector (op) { - function filter (o) { - return !Y.utils.compareIds(o, op.id) - } - this.gc1 = this.gc1.filter(filter) - this.gc2 = this.gc2.filter(filter) - delete op.gc - } - destroyTypes () { - for (var key in this.initializedTypes) { - var type = this.initializedTypes[key] - if (type._destroy != null) { - type._destroy() - } else { - console.error('The type you included does not provide destroy functionality, it will remain in memory (updating your packages will help).') - } - } - } - destroy () { - clearTimeout(this.gcInterval) - this.gcInterval = null - this.stopRepairCheck() - } - setUserId (userId) { - if (!this.userIdPromise.inProgress) { - this.userIdPromise.inProgress = true - var self = this - self.requestTransaction(function () { - self.userId = userId - var state = this.getState(userId) - self.opClock = state.clock - self.userIdPromise.resolve(userId) - }) - } - return this.userIdPromise - } - whenUserIdSet (f) { - this.userIdPromise.then(f) - } - getNextOpId (numberOfIds) { - if (numberOfIds == null) { - throw new Error('getNextOpId expects the number of created ids to create!') - } else if (this.userId == null) { - throw new Error('OperationStore not yet initialized!') - } else { - var id = [this.userId, this.opClock] - this.opClock += numberOfIds - return id - } - } - /* - Apply a list of operations. - - * we save a timestamp, because we received new operations that could resolve ops in this.listenersById (see this.startRepairCheck) - * get a transaction - * check whether all Struct.*.requiredOps are in the OS - * check if it is an expected op (otherwise wait for it) - * check if was deleted, apply a delete operation after op was applied - */ - applyOperations (decoder) { - this.opsReceivedTimestamp = new Date() - let length = decoder.readUint32() - - for (var i = 0; i < length; i++) { - let o = Y.Struct.binaryDecodeOperation(decoder) - if (o.id == null || o.id[0] !== this.y.connector.userId) { - var required = Y.Struct[o.struct].requiredOps(o) - if (o.requires != null) { - required = required.concat(o.requires) - } - this.whenOperationsExist(required, o) - } - } - } - /* - op is executed as soon as every operation requested is available. - Note that Transaction can (and should) buffer requests. - */ - whenOperationsExist (ids, op) { - if (ids.length > 0) { - let listener = { - op: op, - missing: ids.length - } - - for (let i = 0; i < ids.length; i++) { - let id = ids[i] - let sid = JSON.stringify(id) - let l = this.listenersById[sid] - if (l == null) { - l = [] - this.listenersById[sid] = l - } - l.push(listener) - } - } else { - this.listenersByIdExecuteNow.push({ - op: op - }) - } - - if (this.listenersByIdRequestPending) { - return - } - - this.listenersByIdRequestPending = true - var store = this - - this.requestTransaction(function () { - var exeNow = store.listenersByIdExecuteNow - store.listenersByIdExecuteNow = [] - - var ls = store.listenersById - store.listenersById = {} - - store.listenersByIdRequestPending = false - - for (let key = 0; key < exeNow.length; key++) { - let o = exeNow[key].op - store.tryExecute.call(this, o) - } - - for (var sid in ls) { - var l = ls[sid] - var id = JSON.parse(sid) - var op - if (typeof id[1] === 'string') { - op = this.getOperation(id) - } else { - op = this.getInsertion(id) - } - if (op == null) { - store.listenersById[sid] = l - } else { - for (let i = 0; i < l.length; i++) { - let listener = l[i] - let o = listener.op - if (--listener.missing === 0) { - store.tryExecute.call(this, o) - } - } - } - } - }) - } - /* - Actually execute an operation, when all expected operations are available. - */ - /* :: // TODO: this belongs somehow to transaction - store: Object; - getOperation: any; - isGarbageCollected: any; - addOperation: any; - whenOperationsExist: any; - */ - tryExecute (op) { - this.store.addToDebug('this.store.tryExecute.call(this, ', JSON.stringify(op), ')') - if (op.struct === 'Delete') { - Y.Struct.Delete.execute.call(this, op) - // this is now called in Transaction.deleteOperation! - // this.store.operationAdded(this, op) - } else { - // check if this op was defined - var defined = this.getInsertion(op.id) - while (defined != null && defined.content != null) { - // check if this op has a longer content in the case it is defined - if (defined.id[1] + defined.content.length < op.id[1] + op.content.length) { - var overlapSize = defined.content.length - (op.id[1] - defined.id[1]) - op.content.splice(0, overlapSize) - op.id = [op.id[0], op.id[1] + overlapSize] - op.left = Y.utils.getLastId(defined) - op.origin = op.left - defined = this.getOperation(op.id) // getOperation suffices here - } else { - break - } - } - if (defined == null) { - var opid = op.id - var isGarbageCollected = this.isGarbageCollected(opid) - if (!isGarbageCollected) { - // TODO: reduce number of get / put calls for op .. - Y.Struct[op.struct].execute.call(this, op) - this.addOperation(op) - this.store.operationAdded(this, op) - // operationAdded can change op.. - op = this.getOperation(opid) - // if insertion, try to combine with left - this.tryCombineWithLeft(op) - } - } - } - } - /* - * Called by a transaction when an operation is added. - * This function is especially important for y-indexeddb, where several instances may share a single database. - * Every time an operation is created by one instance, it is send to all other instances and operationAdded is called - * - * If it's not a Delete operation: - * * Checks if another operation is executable (listenersById) - * * Update state, if possible - * - * Always: - * * Call type - */ - operationAdded (transaction, op) { - if (op.struct === 'Delete') { - var type = this.initializedTypes[JSON.stringify(op.targetParent)] - if (type != null) { - type._changed(transaction, op) - } - } else { - // increase SS - transaction.updateState(op.id[0]) - var opLen = op.content != null ? op.content.length : 1 - for (let i = 0; i < opLen; i++) { - // notify whenOperation listeners (by id) - var sid = JSON.stringify([op.id[0], op.id[1] + i]) - var l = this.listenersById[sid] - delete this.listenersById[sid] - if (l != null) { - for (var key in l) { - var listener = l[key] - if (--listener.missing === 0) { - this.whenOperationsExist([], listener.op) - } - } - } - } - var t = this.initializedTypes[JSON.stringify(op.parent)] - - // if parent is deleted, mark as gc'd and return - if (op.parent != null) { - var parentIsDeleted = transaction.isDeleted(op.parent) - if (parentIsDeleted) { - transaction.deleteList(op.id) - return - } - } - - // notify parent, if it was instanciated as a custom type - if (t != null) { - let o = Y.utils.copyOperation(op) - t._changed(transaction, o) - } - if (!op.deleted) { - // Delete if DS says this is actually deleted - var len = op.content != null ? op.content.length : 1 - var startId = op.id // You must not use op.id in the following loop, because op will change when deleted - // TODO: !! console.log('TODO: change this before commiting') - for (let i = 0; i < len; i++) { - var id = [startId[0], startId[1] + i] - var opIsDeleted = transaction.isDeleted(id) - if (opIsDeleted) { - var delop = { - struct: 'Delete', - target: id - } - this.tryExecute.call(transaction, delop) - } - } - } - } - } - - whenTransactionsFinished () { - if (this.transactionInProgress) { - if (this.transactionsFinished == null) { - var resolve_ - var promise = new Promise(function (resolve) { - resolve_ = resolve - }) - this.transactionsFinished = { - resolve: resolve_, - promise: promise - } - } - return this.transactionsFinished.promise - } else { - return Promise.resolve() - } - } - - // Check if there is another transaction request. - // * the last transaction is always a flush :) - getNextRequest () { - if (this.waitingTransactions.length === 0) { - if (this.transactionIsFlushed) { - this.transactionInProgress = false - this.transactionIsFlushed = false - if (this.transactionsFinished != null) { - this.transactionsFinished.resolve() - this.transactionsFinished = null - } - return null - } else { - this.transactionIsFlushed = true - return function () { - this.flush() - } - } - } else { - this.transactionIsFlushed = false - return this.waitingTransactions.shift() - } - } - requestTransaction (makeGen/* :any */, callImmediately) { - this.waitingTransactions.push(makeGen) - if (!this.transactionInProgress) { - this.transactionInProgress = true - setTimeout(() => { - this.transact(this.getNextRequest()) - }, 0) - } - } - /* - Get a created/initialized type. - */ - getType (id) { - return this.initializedTypes[JSON.stringify(id)] - } - /* - Init type. This is called when a remote operation is retrieved, and transformed to a type - TODO: delete type from store.initializedTypes[id] when corresponding id was deleted! - */ - initType (id, args) { - var sid = JSON.stringify(id) - var t = this.store.initializedTypes[sid] - if (t == null) { - var op/* :MapStruct | ListStruct */ = this.getOperation(id) - if (op != null) { - t = Y[op.type].typeDefinition.initType.call(this, this.store, op, args) - this.store.initializedTypes[sid] = t - } - } - return t - } - /* - Create type. This is called when the local user creates a type (which is a synchronous action) - */ - createType (typedefinition, id) { - var structname = typedefinition[0].struct - id = id || this.getNextOpId(1) - var op = Y.Struct[structname].create(id, typedefinition[1]) - op.type = typedefinition[0].name - - this.requestTransaction(function () { - if (op.id[0] === 0xFFFFFF) { - this.setOperation(op) - } else { - this.applyCreatedOperations([op]) - } - }) - var t = Y[op.type].typeDefinition.createType(this, op, typedefinition[1]) - this.initializedTypes[JSON.stringify(op.id)] = t - return t - } - } - Y.AbstractDatabase = AbstractDatabase -} diff --git a/src/MessageHandler.js b/src/MessageHandler.js deleted file mode 100644 index 6a01c2e4..00000000 --- a/src/MessageHandler.js +++ /dev/null @@ -1,193 +0,0 @@ - -import Y from './y.js' -import { BinaryDecoder, BinaryEncoder } from './Encoding.js' - -export function formatYjsMessage (buffer) { - let decoder = new BinaryDecoder(buffer) - decoder.readVarString() // read roomname - let type = decoder.readVarString() - let strBuilder = [] - strBuilder.push('\n === ' + type + ' ===\n') - if (type === 'update') { - logMessageUpdate(decoder, strBuilder) - } else if (type === 'sync step 1') { - logMessageSyncStep1(decoder, strBuilder) - } else if (type === 'sync step 2') { - logMessageSyncStep2(decoder, strBuilder) - } else { - strBuilder.push('-- Unknown message type - probably an encoding issue!!!') - } - return strBuilder.join('') -} - -export function formatYjsMessageType (buffer) { - let decoder = new BinaryDecoder(buffer) - decoder.readVarString() // roomname - return decoder.readVarString() -} - -export function logMessageUpdate (decoder, strBuilder) { - let len = decoder.readUint32() - for (let i = 0; i < len; i++) { - strBuilder.push(JSON.stringify(Y.Struct.binaryDecodeOperation(decoder)) + '\n') - } -} - -export function computeMessageUpdate (decoder, encoder, conn) { - if (conn.y.db.forwardAppliedOperations || conn.y.persistence != null) { - let messagePosition = decoder.pos - let len = decoder.readUint32() - let delops = [] - for (let i = 0; i < len; i++) { - let op = Y.Struct.binaryDecodeOperation(decoder) - if (op.struct === 'Delete') { - delops.push(op) - } - } - if (delops.length > 0) { - if (conn.y.db.forwardAppliedOperations) { - conn.broadcastOps(delops) - } - if (conn.y.persistence) { - conn.y.persistence.saveOperations(delops) - } - } - decoder.pos = messagePosition - } - conn.y.db.applyOperations(decoder) -} - -export function sendSyncStep1 (conn, syncUser) { - conn.y.db.requestTransaction(function () { - let encoder = new BinaryEncoder() - encoder.writeVarString(conn.opts.room || '') - encoder.writeVarString('sync step 1') - encoder.writeVarString(conn.authInfo || '') - encoder.writeVarUint(conn.protocolVersion) - let preferUntransformed = conn.preferUntransformed && this.os.length === 0 // TODO: length may not be defined - encoder.writeUint8(preferUntransformed ? 1 : 0) - this.writeStateSet(encoder) - conn.send(syncUser, encoder.createBuffer()) - }) -} - -export function logMessageSyncStep1 (decoder, strBuilder) { - let auth = decoder.readVarString() - let protocolVersion = decoder.readVarUint() - let preferUntransformed = decoder.readUint8() === 1 - strBuilder.push(` - - auth: "${auth}" - - protocolVersion: ${protocolVersion} - - preferUntransformed: ${preferUntransformed} -`) - logSS(decoder, strBuilder) -} - -export function computeMessageSyncStep1 (decoder, encoder, conn, senderConn, sender) { - let protocolVersion = decoder.readVarUint() - let preferUntransformed = decoder.readUint8() === 1 - - // check protocol version - if (protocolVersion !== conn.protocolVersion) { - console.warn( - `You tried to sync with a yjs instance that has a different protocol version - (You: ${protocolVersion}, Client: ${protocolVersion}). - The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)! - `) - conn.y.destroy() - } - - return conn.y.db.whenTransactionsFinished().then(() => { - // send sync step 2 - conn.y.db.requestTransaction(function () { - encoder.writeVarString('sync step 2') - encoder.writeVarString(conn.authInfo || '') - - if (preferUntransformed) { - encoder.writeUint8(1) - this.writeOperationsUntransformed(encoder) - } else { - encoder.writeUint8(0) - this.writeOperations(encoder, decoder) - } - - this.writeDeleteSet(encoder) - conn.send(senderConn.uid, encoder.createBuffer()) - senderConn.receivedSyncStep2 = true - }) - return conn.y.db.whenTransactionsFinished().then(() => { - if (conn.role === 'slave') { - sendSyncStep1(conn, sender) - } - }) - }) -} - -export function logSS (decoder, strBuilder) { - strBuilder.push(' == SS: \n') - let len = decoder.readUint32() - for (let i = 0; i < len; i++) { - let user = decoder.readVarUint() - let clock = decoder.readVarUint() - strBuilder.push(` ${user}: ${clock}\n`) - } -} - -export function logOS (decoder, strBuilder) { - strBuilder.push(' == OS: \n') - let len = decoder.readUint32() - for (let i = 0; i < len; i++) { - let op = Y.Struct.binaryDecodeOperation(decoder) - strBuilder.push(JSON.stringify(op) + '\n') - } -} - -export function logDS (decoder, strBuilder) { - strBuilder.push(' == DS: \n') - let len = decoder.readUint32() - for (let i = 0; i < len; i++) { - let user = decoder.readVarUint() - strBuilder.push(` User: ${user}: `) - let len2 = decoder.readVarUint() - for (let j = 0; j < len2; j++) { - let from = decoder.readVarUint() - let to = decoder.readVarUint() - let gc = decoder.readUint8() === 1 - strBuilder.push(`[${from}, ${to}, ${gc}]`) - } - } -} - -export function logMessageSyncStep2 (decoder, strBuilder) { - strBuilder.push(' - auth: ' + decoder.readVarString() + '\n') - let osTransformed = decoder.readUint8() === 1 - strBuilder.push(' - osUntransformed: ' + osTransformed + '\n') - logOS(decoder, strBuilder) - if (osTransformed) { - logSS(decoder, strBuilder) - } - logDS(decoder, strBuilder) -} - -export function computeMessageSyncStep2 (decoder, encoder, conn, senderConn, sender) { - var db = conn.y.db - let defer = senderConn.syncStep2 - - // apply operations first - db.requestTransaction(function () { - let osUntransformed = decoder.readUint8() - if (osUntransformed === 1) { - this.applyOperationsUntransformed(decoder) - } else { - this.store.applyOperations(decoder) - } - }) - // then apply ds - db.requestTransaction(function () { - this.applyDeleteSet(decoder) - }) - return db.whenTransactionsFinished().then(() => { - conn._setSyncedWith(sender) - defer.resolve() - }) -} diff --git a/src/MessageHandler/deleteSet.js b/src/MessageHandler/deleteSet.js new file mode 100644 index 00000000..4a22f2be --- /dev/null +++ b/src/MessageHandler/deleteSet.js @@ -0,0 +1,117 @@ +import deleteItemRange from 'deleteItemRange' + +export function stringifyDeleteSet (y, decoder, strBuilder) { + let dsLength = decoder.readUint32() + for (let i = 0; i < dsLength; i++) { + let user = decoder.readVarUint() + strBuilder.push(' -' + user + ':') + let dvLength = decoder.readVarUint() + for (let j = 0; j < dvLength; j++) { + let from = decoder.readVarUint() + let len = decoder.readVarUint() + let gc = decoder.readUint8() === 1 + strBuilder.push(`clock: ${from}, length: ${len}, gc: ${gc}`) + } + } + return strBuilder +} + +export function writeDeleteSet (y, encoder) { + let currentUser = null + let currentLength = 0 + let lastLenPos + + let numberOfUsers = 0 + let laterDSLenPus = encoder.pos + encoder.writeUint32(0) + + y.ds.iterate(null, null, function (n) { + var user = n._id.user + var clock = n._id.clock + var len = n.len + var gc = n.gc + if (currentUser !== user) { + numberOfUsers++ + // a new user was found + if (currentUser !== null) { // happens on first iteration + encoder.setUint32(lastLenPos, currentLength) + } + encoder.writeVarUint(user) + // pseudo-fill pos + lastLenPos = encoder.pos + encoder.writeUint32(0) + } + encoder.writeVarUint(clock) + encoder.writeVarUint(len) + encoder.writeUint8(gc ? 1 : 0) + }) + if (currentUser !== null) { // happens on first iteration + encoder.setUint32(lastLenPos, currentLength) + } + encoder.writeUint32(laterDSLenPus, numberOfUsers) +} + +export function readDeleteSet (y, decoder) { + let dsLength = decoder.readUint32() + for (let i = 0; i < dsLength; i++) { + let user = decoder.readVarUint() + let dv = [] + let dvLength = decoder.readVarUint() + for (let j = 0; j < dvLength; j++) { + let from = decoder.readVarUint() + let len = decoder.readVarUint() + let gc = decoder.readUint8() === 1 + dv.push([from, len, gc]) + } + var pos = 0 + var d = dv[pos] + y.ds.iterate(this, [user, 0], [user, Number.MAX_VALUE], function (n) { + // cases: + // 1. d deletes something to the right of n + // => go to next n (break) + // 2. d deletes something to the left of n + // => create deletions + // => reset d accordingly + // *)=> if d doesn't delete anything anymore, go to next d (continue) + // 3. not 2) and d deletes something that also n deletes + // => reset d so that it doesn't contain n's deletion + // *)=> if d does not delete anything anymore, go to next d (continue) + while (d != null) { + var diff = 0 // describe the diff of length in 1) and 2) + if (n.id[1] + n.len <= d[0]) { + // 1) + break + } else if (d[0] < n.id[1]) { + // 2) + // delete maximum the len of d + // else delete as much as possible + diff = Math.min(n.id[1] - d[0], d[1]) + deleteItemRange(y, user, d[0], diff) + // deletions.push([user, d[0], diff, d[2]]) + } else { + // 3) + diff = n.id[1] + n.len - d[0] // never null (see 1) + if (d[2] && !n.gc) { + // d marks as gc'd but n does not + // then delete either way + deleteItemRange(y, user, d[0], Math.min(diff, d[1])) + // deletions.push([user, d[0], Math.min(diff, d[1]), d[2]]) + } + } + if (d[1] <= diff) { + // d doesn't delete anything anymore + d = dv[++pos] + } else { + d[0] = d[0] + diff // reset pos + d[1] = d[1] - diff // reset length + } + } + }) + // for the rest.. just apply it + for (; pos < dv.length; pos++) { + d = dv[pos] + deleteItemRange(y, user, d[0], d[1]) + // deletions.push([user, d[0], d[1], d[2]]) + } + } +} diff --git a/src/MessageHandler/integrateRemoteStructs.js b/src/MessageHandler/integrateRemoteStructs.js new file mode 100644 index 00000000..13749ee7 --- /dev/null +++ b/src/MessageHandler/integrateRemoteStructs.js @@ -0,0 +1,73 @@ +import { getStruct } from '../Util/StructReferences' +import BinaryDecoder from '../Util/Binary/Decoder' + +class MissingEntry { + constructor (decoder, missing, struct) { + this.decoder = decoder + this.missing = missing.length + this.struct = struct + } +} + +/** + * Integrate remote struct + * When a remote struct is integrated, other structs might be ready to ready to + * integrate. + */ +function _integrateRemoteStructHelper (y, struct) { + struct._integrate(y) + let msu = y._missingStructs.get(struct._id.user) + if (msu != null) { + let len = struct._length + for (let i = 0; i < len; i++) { + if (msu.has(struct._id.clock + i)) { + let msuc = msu.get(struct._id.clock + i) + msuc.forEach(missingDef => { + missingDef.missing-- + if (missingDef.missing === 0) { + let missing = missingDef.struct._fromBinary(y, missingDef.decoder) + if (missing.length > 0) { + console.error('Missing should be empty!') + } else { + y._readyToIntegrate.push(missingDef.struct) + } + } + }) + msu.delete(struct._id.clock) + } + } + } +} + +export default function integrateRemoteStructs (decoder, encoder, y) { + while (decoder.length !== decoder.pos) { + let decoderPos = decoder.pos + let reference = decoder.readVarUint() + let Constr = getStruct(reference) + let struct = new Constr() + let missing = struct._fromBinary(decoder) + if (missing.length === 0) { + while (struct != null) { + _integrateRemoteStructHelper(y, struct) + struct = y._readyToIntegrate.shift() + } + } else { + let _decoder = new BinaryDecoder(decoder.uint8arr) + _decoder.pos = decoderPos + let missingEntry = new MissingEntry(_decoder, missing, struct) + let missingStructs = y._missingStructs + for (let i = missing.length - 1; i >= 0; i--) { + let m = missing[i] + if (!missingStructs.has(m.user)) { + missingStructs.set(m.user, new Map()) + } + let msu = missingStructs.get(m.user) + if (!msu.has(m.clock)) { + msu.set(m.clock, []) + } + let mArray = msu = msu.get(m.clock) + mArray.push(missingEntry) + } + } + } +} diff --git a/src/MessageHandler/messageToString.js b/src/MessageHandler/messageToString.js new file mode 100644 index 00000000..c6521f2a --- /dev/null +++ b/src/MessageHandler/messageToString.js @@ -0,0 +1,28 @@ +import BinaryDecoder from '../Utily/Binary/Decoder' +import { stringifyUpdate } from './update' +import { stringifySyncStep1 } from './syncStep1' +import { stringifySyncStep2 } from './syncStep2' + +export function messageToString (buffer) { + let decoder = new BinaryDecoder(buffer) + decoder.readVarString() // read roomname + let type = decoder.readVarString() + let strBuilder = [] + strBuilder.push('\n === ' + type + ' ===\n') + if (type === 'update') { + stringifyUpdate(decoder, strBuilder) + } else if (type === 'sync step 1') { + stringifySyncStep1(decoder, strBuilder) + } else if (type === 'sync step 2') { + stringifySyncStep2(decoder, strBuilder) + } else { + strBuilder.push('-- Unknown message type - probably an encoding issue!!!') + } + return strBuilder.join('\n') +} + +export function messageToRoomname (buffer) { + let decoder = new BinaryDecoder(buffer) + decoder.readVarString() // roomname + return decoder.readVarString() // messageType +} diff --git a/src/MessageHandler/stateSet.js b/src/MessageHandler/stateSet.js new file mode 100644 index 00000000..057d5518 --- /dev/null +++ b/src/MessageHandler/stateSet.js @@ -0,0 +1,24 @@ + +export function readStateSet (decoder) { + let ss = new Map() + let ssLength = decoder.readUint32() + for (let i = 0; i < ssLength; i++) { + let user = decoder.readVarUint() + let clock = decoder.readVarUint() + ss.set(user, clock) + } + return ss +} + +export function writeStateSet (encoder) { + let lenPosition = encoder.pos + let len = 0 + encoder.writeUint32(0) + this.ss.iterate(this, null, null, function (n) { + encoder.writeVarUint(n.id[0]) + encoder.writeVarUint(n.clock) + len++ + }) + encoder.setUint32(lenPosition, len) + return len === 0 +} diff --git a/src/MessageHandler/syncStep1.js b/src/MessageHandler/syncStep1.js new file mode 100644 index 00000000..b9e95193 --- /dev/null +++ b/src/MessageHandler/syncStep1.js @@ -0,0 +1,53 @@ +import BinaryEncoder from './Util/Binary/Encoder.js' + +export function stringifySyncStep1 (decoder, strBuilder) { + let auth = decoder.readVarString() + let protocolVersion = decoder.readVarUint() + strBuilder.push(` + - auth: "${auth}" + - protocolVersion: ${protocolVersion} +`) + // write SS + strBuilder.push(' == SS: \n') + let len = decoder.readUint32() + for (let i = 0; i < len; i++) { + let user = decoder.readVarUint() + let clock = decoder.readVarUint() + strBuilder.push(` ${user}: ${clock}\n`) + } +} + +export function sendSyncStep1 (y, syncUser) { + let encoder = new BinaryEncoder() + encoder.writeVarString(y.room) + encoder.writeVarString('sync step 1') + encoder.writeVarString(y.connector.authInfo || '') + encoder.writeVarUint(y.connector.protocolVersion) + y.ss.writeStateSet(encoder) + y.connector.send(syncUser, encoder.createBuffer()) +} + +export function readSyncStep1 (decoder, encoder, y, senderConn, sender) { + let protocolVersion = decoder.readVarUint() + // check protocol version + if (protocolVersion !== y.connector.protocolVersion) { + console.warn( + `You tried to sync with a yjs instance that has a different protocol version + (You: ${protocolVersion}, Client: ${protocolVersion}). + The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)! + `) + y.destroy() + } + + // send sync step 2 + encoder.writeVarString('sync step 2') + encoder.writeVarString(y.connector.authInfo || '') + writeDeleteSet(encoder) + // reads ss and writes os + writeOperations(encoder, decoder) + y.connector.send(senderConn.uid, encoder.createBuffer()) + senderConn.receivedSyncStep2 = true + if (y.connector.role === 'slave') { + sendSyncStep1(y, sender) + } +} diff --git a/src/MessageHandler/syncStep2.js b/src/MessageHandler/syncStep2.js new file mode 100644 index 00000000..11a98262 --- /dev/null +++ b/src/MessageHandler/syncStep2.js @@ -0,0 +1,48 @@ +import integrateRemoteStructs from './integrateRemoteStructs' +import { stringifyUpdate } from './update.js' +import ID from '../Util/ID' + +export function stringifySyncStep2 (decoder, strBuilder) { + strBuilder.push(' - auth: ' + decoder.readVarString() + '\n') + strBuilder.push(' == OS: \n') + stringifyUpdate(decoder, strBuilder) + // write DS to string + strBuilder.push(' == DS: \n') + let len = decoder.readUint32() + for (let i = 0; i < len; i++) { + let user = decoder.readVarUint() + strBuilder.push(` User: ${user}: `) + let len2 = decoder.readVarUint() + for (let j = 0; j < len2; j++) { + let from = decoder.readVarUint() + let to = decoder.readVarUint() + let gc = decoder.readUint8() === 1 + strBuilder.push(`[${from}, ${to}, ${gc}]`) + } + } +} + +export function writeSyncStep2 () { + // TODO +} + +export default function writeStructs (encoder, decoder, y, ss) { + let lenPos = encoder.pos + let len = 0 + encoder.writeUint32(0) + for (let [user, clock] of ss) { + y.os.iterate(new ID(user, clock), null, function (struct) { + struct._toBinary(y, encoder) + len++ + }) + } + encoder.setUint32(lenPos, len) +} + +export function readSyncStep2 (decoder, encoder, y, senderConn, sender) { + // apply operations first + applyDeleteSet(decoder) + integrateRemoteStructs(decoder, encoder, y) + // then apply ds + y.connector._setSyncedWith(sender) +} diff --git a/src/MessageHandler/update.js b/src/MessageHandler/update.js new file mode 100644 index 00000000..36121c58 --- /dev/null +++ b/src/MessageHandler/update.js @@ -0,0 +1,19 @@ + +import { getStruct } from '../Util/StructReferences' + +export function stringifyUpdate (decoder, strBuilder) { + while (decoder.length !== decoder.pos) { + let reference = decoder.readVarUint() + let Constr = getStruct(reference) + let struct = new Constr() + let missing = struct._fromBinary(decoder) + let logMessage = struct._logString() + if (missing.length > 0) { + logMessage += missing.map(m => m._logString()).join(', ') + } + logMessage += '\n' + strBuilder.push(logMessage) + } +} + +export { integrateRemoteStructs as readUpdate } from './integrateRemoteStructs' diff --git a/src/Notes.md b/src/Notes.md deleted file mode 100644 index f9d2236e..00000000 --- a/src/Notes.md +++ /dev/null @@ -1,12 +0,0 @@ - -# Notes - -### Terminology - -* DB: DataBase that holds all the information of the shared object. It is devided into the OS, DS, and SS. This can be a persistent database or an in-memory database. Depending on the type of database, it could make sense to store OS, DS, and SS in different tables, or maybe different databases. -* OS: OperationStore holds all the operations. An operation is a js object with a fixed number of name fields. -* DS: DeleteStore holds the information about which operations are deleted and which operations were garbage collected (no longer available in the OS). -* SS: StateSet holds the current state of the OS. SS.getState(username) refers to the amount of operations that were received by that respective user. -* Op: Operation defines an action on a shared type. But it is also the format in which we store the model of a type. This is why it is also called a Struct/Structure. -* Type and Structure: We crearly distinguish between type and structure. Short explanation: A type (e.g. Strings, Numbers) have a number of functions that you can apply on them. (+) is well defined on both of them. They are *modeled* by a structure - the functions really change the structure of a type. Types can be implemented differently but still provide the same functionality. In Yjs, almost all types are realized as a doubly linked list (on which Yjs can provide eventual convergence) -* \ No newline at end of file diff --git a/src/RedBlackTree.js b/src/RedBlackTree.js deleted file mode 100644 index 40483e99..00000000 --- a/src/RedBlackTree.js +++ /dev/null @@ -1,506 +0,0 @@ - -export default function extendRBTree (Y) { - class N { - // A created node is always red! - constructor (val) { - this.val = val - this.color = true - this._left = null - this._right = null - this._parent = null - if (val.id === null) { - throw new Error('You must define id!') - } - } - isRed () { return this.color } - isBlack () { return !this.color } - redden () { this.color = true; return this } - blacken () { this.color = false; return this } - get grandparent () { - return this.parent.parent - } - get parent () { - return this._parent - } - get sibling () { - return (this === this.parent.left) - ? this.parent.right : this.parent.left - } - get left () { - return this._left - } - get right () { - return this._right - } - set left (n) { - if (n !== null) { - n._parent = this - } - this._left = n - } - set right (n) { - if (n !== null) { - n._parent = this - } - this._right = n - } - rotateLeft (tree) { - var parent = this.parent - var newParent = this.right - var newRight = this.right.left - newParent.left = this - this.right = newRight - if (parent === null) { - tree.root = newParent - newParent._parent = null - } else if (parent.left === this) { - parent.left = newParent - } else if (parent.right === this) { - parent.right = newParent - } else { - throw new Error('The elements are wrongly connected!') - } - } - next () { - if (this.right !== null) { - // search the most left node in the right tree - var o = this.right - while (o.left !== null) { - o = o.left - } - return o - } else { - var p = this - while (p.parent !== null && p !== p.parent.left) { - p = p.parent - } - return p.parent - } - } - prev () { - if (this.left !== null) { - // search the most right node in the left tree - var o = this.left - while (o.right !== null) { - o = o.right - } - return o - } else { - var p = this - while (p.parent !== null && p !== p.parent.right) { - p = p.parent - } - return p.parent - } - } - rotateRight (tree) { - var parent = this.parent - var newParent = this.left - var newLeft = this.left.right - newParent.right = this - this.left = newLeft - if (parent === null) { - tree.root = newParent - newParent._parent = null - } else if (parent.left === this) { - parent.left = newParent - } else if (parent.right === this) { - parent.right = newParent - } else { - throw new Error('The elements are wrongly connected!') - } - } - getUncle () { - // we can assume that grandparent exists when this is called! - if (this.parent === this.parent.parent.left) { - return this.parent.parent.right - } else { - return this.parent.parent.left - } - } - } - - class RBTree { - constructor () { - this.root = null - this.length = 0 - } - findNext (id) { - return this.findWithLowerBound([id[0], id[1] + 1]) - } - findPrev (id) { - return this.findWithUpperBound([id[0], id[1] - 1]) - } - findNodeWithLowerBound (from) { - if (from === void 0) { - throw new Error('You must define from!') - } - var o = this.root - if (o === null) { - return null - } else { - while (true) { - if ((from === null || Y.utils.smaller(from, o.val.id)) && o.left !== null) { - // o is included in the bound - // try to find an element that is closer to the bound - o = o.left - } else if (from !== null && Y.utils.smaller(o.val.id, from)) { - // o is not within the bound, maybe one of the right elements is.. - if (o.right !== null) { - o = o.right - } else { - // there is no right element. Search for the next bigger element, - // this should be within the bounds - return o.next() - } - } else { - return o - } - } - } - } - findNodeWithUpperBound (to) { - if (to === void 0) { - throw new Error('You must define from!') - } - var o = this.root - if (o === null) { - return null - } else { - while (true) { - if ((to === null || Y.utils.smaller(o.val.id, to)) && o.right !== null) { - // o is included in the bound - // try to find an element that is closer to the bound - o = o.right - } else if (to !== null && Y.utils.smaller(to, o.val.id)) { - // o is not within the bound, maybe one of the left elements is.. - if (o.left !== null) { - o = o.left - } else { - // there is no left element. Search for the prev smaller element, - // this should be within the bounds - return o.prev() - } - } else { - return o - } - } - } - } - findSmallestNode () { - var o = this.root - while (o != null && o.left != null) { - o = o.left - } - return o - } - findWithLowerBound (from) { - var n = this.findNodeWithLowerBound(from) - return n == null ? null : n.val - } - findWithUpperBound (to) { - var n = this.findNodeWithUpperBound(to) - return n == null ? null : n.val - } - iterate (t, from, to, f) { - var o - if (from === null) { - o = this.findSmallestNode() - } else { - o = this.findNodeWithLowerBound(from) - } - while ( - o !== null && - ( - to === null || // eslint-disable-line no-unmodified-loop-condition - Y.utils.smaller(o.val.id, to) || - Y.utils.compareIds(o.val.id, to) - ) - ) { - f.call(t, o.val) - o = o.next() - } - return true - } - logTable (from, to, filter) { - if (filter == null) { - filter = function () { - return true - } - } - if (from == null) { from = null } - if (to == null) { to = null } - var os = [] - this.iterate(this, from, to, function (o) { - if (filter(o)) { - var o_ = {} - for (var key in o) { - if (typeof o[key] === 'object') { - o_[key] = JSON.stringify(o[key]) - } else { - o_[key] = o[key] - } - } - os.push(o_) - } - }) - if (console.table != null) { - console.table(os) - } - } - find (id) { - var n - return (n = this.findNode(id)) ? n.val : null - } - findNode (id) { - if (id == null || id.constructor !== Array) { - throw new Error('Expect id to be an array!') - } - var o = this.root - if (o === null) { - return false - } else { - while (true) { - if (o === null) { - return false - } - if (Y.utils.smaller(id, o.val.id)) { - o = o.left - } else if (Y.utils.smaller(o.val.id, id)) { - o = o.right - } else { - return o - } - } - } - } - delete (id) { - if (id == null || id.constructor !== Array) { - throw new Error('id is expected to be an Array!') - } - var d = this.findNode(id) - if (d == null) { - // throw new Error('Element does not exist!') - return - } - this.length-- - if (d.left !== null && d.right !== null) { - // switch d with the greates element in the left subtree. - // o should have at most one child. - var o = d.left - // find - while (o.right !== null) { - o = o.right - } - // switch - d.val = o.val - d = o - } - // d has at most one child - // let n be the node that replaces d - var isFakeChild - var child = d.left || d.right - if (child === null) { - isFakeChild = true - child = new N({id: 0}) - child.blacken() - d.right = child - } else { - isFakeChild = false - } - - if (d.parent === null) { - if (!isFakeChild) { - this.root = child - child.blacken() - child._parent = null - } else { - this.root = null - } - return - } else if (d.parent.left === d) { - d.parent.left = child - } else if (d.parent.right === d) { - d.parent.right = child - } else { - throw new Error('Impossible!') - } - if (d.isBlack()) { - if (child.isRed()) { - child.blacken() - } else { - this._fixDelete(child) - } - } - this.root.blacken() - if (isFakeChild) { - if (child.parent.left === child) { - child.parent.left = null - } else if (child.parent.right === child) { - child.parent.right = null - } else { - throw new Error('Impossible #3') - } - } - } - _fixDelete (n) { - function isBlack (node) { - return node !== null ? node.isBlack() : true - } - function isRed (node) { - return node !== null ? node.isRed() : false - } - if (n.parent === null) { - // this can only be called after the first iteration of fixDelete. - return - } - // d was already replaced by the child - // d is not the root - // d and child are black - var sibling = n.sibling - if (isRed(sibling)) { - // make sibling the grandfather - n.parent.redden() - sibling.blacken() - if (n === n.parent.left) { - n.parent.rotateLeft(this) - } else if (n === n.parent.right) { - n.parent.rotateRight(this) - } else { - throw new Error('Impossible #2') - } - sibling = n.sibling - } - // parent, sibling, and children of n are black - if (n.parent.isBlack() && - sibling.isBlack() && - isBlack(sibling.left) && - isBlack(sibling.right) - ) { - sibling.redden() - this._fixDelete(n.parent) - } else if (n.parent.isRed() && - sibling.isBlack() && - isBlack(sibling.left) && - isBlack(sibling.right) - ) { - sibling.redden() - n.parent.blacken() - } else { - if (n === n.parent.left && - sibling.isBlack() && - isRed(sibling.left) && - isBlack(sibling.right) - ) { - sibling.redden() - sibling.left.blacken() - sibling.rotateRight(this) - sibling = n.sibling - } else if (n === n.parent.right && - sibling.isBlack() && - isRed(sibling.right) && - isBlack(sibling.left) - ) { - sibling.redden() - sibling.right.blacken() - sibling.rotateLeft(this) - sibling = n.sibling - } - sibling.color = n.parent.color - n.parent.blacken() - if (n === n.parent.left) { - sibling.right.blacken() - n.parent.rotateLeft(this) - } else { - sibling.left.blacken() - n.parent.rotateRight(this) - } - } - } - put (v) { - if (v == null || v.id == null || v.id.constructor !== Array) { - throw new Error('v is expected to have an id property which is an Array!') - } - var node = new N(v) - if (this.root !== null) { - var p = this.root // p abbrev. parent - while (true) { - if (Y.utils.smaller(node.val.id, p.val.id)) { - if (p.left === null) { - p.left = node - break - } else { - p = p.left - } - } else if (Y.utils.smaller(p.val.id, node.val.id)) { - if (p.right === null) { - p.right = node - break - } else { - p = p.right - } - } else { - p.val = node.val - return p - } - } - this._fixInsert(node) - } else { - this.root = node - } - this.length++ - this.root.blacken() - return node - } - _fixInsert (n) { - if (n.parent === null) { - n.blacken() - return - } else if (n.parent.isBlack()) { - return - } - var uncle = n.getUncle() - if (uncle !== null && uncle.isRed()) { - // Note: parent: red, uncle: red - n.parent.blacken() - uncle.blacken() - n.grandparent.redden() - this._fixInsert(n.grandparent) - } else { - // Note: parent: red, uncle: black or null - // Now we transform the tree in such a way that - // either of these holds: - // 1) grandparent.left.isRed - // and grandparent.left.left.isRed - // 2) grandparent.right.isRed - // and grandparent.right.right.isRed - if (n === n.parent.right && n.parent === n.grandparent.left) { - n.parent.rotateLeft(this) - // Since we rotated and want to use the previous - // cases, we need to set n in such a way that - // n.parent.isRed again - n = n.left - } else if (n === n.parent.left && n.parent === n.grandparent.right) { - n.parent.rotateRight(this) - // see above - n = n.right - } - // Case 1) or 2) hold from here on. - // Now traverse grandparent, make parent a black node - // on the highest level which holds two red nodes. - n.parent.blacken() - n.grandparent.redden() - if (n === n.parent.left) { - // Case 1 - n.grandparent.rotateRight(this) - } else { - // Case 2 - n.grandparent.rotateLeft(this) - } - } - } - flush () {} - } - - Y.utils.RBTree = RBTree -} diff --git a/src/Store/DeleteStore.js b/src/Store/DeleteStore.js new file mode 100644 index 00000000..546dec21 --- /dev/null +++ b/src/Store/DeleteStore.js @@ -0,0 +1,113 @@ +import Tree from '../Util/Tree' +import ID from '../Util/ID' + +class DSNode { + constructor (id, len, gc) { + this.id = id + this.len = len + this.gc = gc + } + clone () { + return new DSNode(this.id, this.len, this.gc) + } +} + +export default class DeleteStore extends Tree { + isDeleted (id) { + var n = this.ds.findWithUpperBound(id) + return n != null && n.id[0] === id[0] && id[1] < n.id[1] + n.len + } + /* + * Mark an operation as deleted. returns the deleted node + */ + markDeleted (id, length) { + if (length == null) { + throw new Error('length must be defined') + } + var n = this.findWithUpperBound(id) + if (n != null && n.id.user === id.user) { + if (n.id.clock <= id.clock && id.clock <= n.id.clock + n.len) { + // id is in n's range + var diff = id.clock + length - (n.id.clock + n.len) // overlapping right + if (diff > 0) { + // id+length overlaps n + if (!n.gc) { + n.len += diff + } else { + diff = n.id.clock + n.len - id.clock // overlapping left (id till n.end) + if (diff < length) { + // a partial deletion + let nId = id.clone() + nId.clock += diff + n = new DSNode(nId, length - diff, false) + this.ds.put(n) + } else { + // already gc'd + throw new Error( + 'DS reached an inconsistent state. Please report this issue!' + ) + } + } + } else { + // no overlapping, already deleted + return n + } + } else { + // cannot extend left (there is no left!) + n = new DSNode(id, length, false) + this.ds.put(n) // TODO: you double-put !! + } + } else { + // cannot extend left + n = new DSNode(id, length, false) + this.ds.put(n) + } + // can extend right? + var next = this.ds.findNext(n.id) + if ( + next != null && + n.id.user === next.id.user && + n.id.clock + n.len >= next.id.clock + ) { + diff = n.id.clock + n.len - next.id.clock // from next.start to n.end + while (diff >= 0) { + // n overlaps with next + if (next.gc) { + // gc is stronger, so reduce length of n + n.len -= diff + if (diff >= next.len) { + // delete the missing range after next + diff = diff - next.len // missing range after next + if (diff > 0) { + this.put(n) // unneccessary? TODO! + this.markDeleted(new ID(next.id.user, next.id.clock + next.len), diff) + } + } + break + } else { + // we can extend n with next + if (diff > next.len) { + // n is even longer than next + // get next.next, and try to extend it + var _next = this.findNext(next.id) + this.delete(next.id) + if (_next == null || n.id.user !== _next.id.user) { + break + } else { + next = _next + diff = n.id.clock + n.len - next.id.clock // from next.start to n.end + // continue! + } + } else { + // n just partially overlaps with next. extend n, delete next, and break this loop + n.len += next.len - diff + this.delete(next.id) + break + } + } + } + } + this.put(n) + return n + } +} diff --git a/src/Store/OperationStore.js b/src/Store/OperationStore.js new file mode 100644 index 00000000..49fe52ed --- /dev/null +++ b/src/Store/OperationStore.js @@ -0,0 +1,88 @@ +import Tree from '../Util/Tree' +import RootID from '../Util/ID' +import { getStruct } from '../Util/structReferences' + +export default class OperationStore extends Tree { + constructor () { + + } + get (id) { + let struct = this.find(id) + if (struct === null && id instanceof RootID) { + let Constr = getStruct(id.type) + struct = new Constr() + struct._id = id + this.put(struct) + } + return struct + } + getItem (id) { + var item = this.findWithUpperBound(id) + if (item == null) { + return null + } + var len = item.content != null ? item.content.length : 1 // in case of opContent + if (id[0] === item.id[0] && id[1] < item.id[1] + len) { + return item + } else { + return null + } + + } + // Return an insertion such that id is the first element of content + // This function manipulates an operation, if necessary + getInsertionCleanStart (id) { + var ins = this.getInsertion(id) + if (ins != null) { + if (ins.id[1] === id[1]) { + return ins + } else { + var left = Y.utils.copyObject(ins) + ins.content = left.content.splice(id[1] - ins.id[1]) + ins.id = id + var leftLid = Y.utils.getLastId(left) + ins.origin = leftLid + left.originOf = [ins.id] + left.right = ins.id + ins.left = leftLid + // debugger // check + this.setOperation(left) + this.setOperation(ins) + if (left.gc) { + this.store.queueGarbageCollector(ins.id) + } + return ins + } + } else { + return null + } + } + // Return an insertion such that id is the last element of content + // This function manipulates an operation, if necessary + getInsertionCleanEnd (id) { + var ins = this.getInsertion(id) + if (ins != null) { + if (ins.content == null || (ins.id[1] + ins.content.length - 1 === id[1])) { + return ins + } else { + var right = Y.utils.copyObject(ins) + right.content = ins.content.splice(id[1] - ins.id[1] + 1) // cut off remainder + right.id = [id[0], id[1] + 1] + var insLid = Y.utils.getLastId(ins) + right.origin = insLid + ins.originOf = [right.id] + ins.right = right.id + right.left = insLid + // debugger // check + this.setOperation(right) + this.setOperation(ins) + if (ins.gc) { + this.store.queueGarbageCollector(right.id) + } + return ins + } + } else { + return null + } + } +} diff --git a/src/Store/StateStore.js b/src/Store/StateStore.js new file mode 100644 index 00000000..b12f8d5d --- /dev/null +++ b/src/Store/StateStore.js @@ -0,0 +1,30 @@ +import ID from '../Util/ID' + +export default class StateStore { + constructor (y) { + this.y = y + this.state = new Map() + this.currentClock = 0 + } + getNextID (len) { + let id = new ID(this.y.userID, this.currentClock) + this.currentClock += len + return id + } + updateRemoteState (struct) { + let user = struct._id.user + let userState = this.state.get(user) + while (struct !== null && struct._id.clock === userState) { + userState += struct._length + struct = this.y.os.get(new ID(user, userState)) + } + this.state.set(user, userState) + } + getState (user) { + let state = this.state.get(user) + if (state == null) { + return 0 + } + return state + } +} diff --git a/src/Struct.js b/src/Struct.js deleted file mode 100644 index efda733a..00000000 --- a/src/Struct.js +++ /dev/null @@ -1,619 +0,0 @@ -const CDELETE = 0 -const CINSERT = 1 -const CLIST = 2 -const CMAP = 3 -const CXML = 4 - -/* - An operation also defines the structure of a type. This is why operation and - structure are used interchangeably here. - - It must be of the type Object. I hope to achieve some performance - improvements when working on databases that support the json format. - - An operation must have the following properties: - - * encode - - Encode the structure in a readable format (preferably string- todo) - * decode (todo) - - decode structure to json - * execute - - Execute the semantics of an operation. - * requiredOps - - Operations that are required to execute this operation. -*/ -export default function extendStruct (Y) { - let Struct = {} - Y.Struct = Struct - Struct.binaryDecodeOperation = function (decoder) { - let code = decoder.peekUint8() - if (code === CDELETE) { - return Struct.Delete.binaryDecode(decoder) - } else if (code === CINSERT) { - return Struct.Insert.binaryDecode(decoder) - } else if (code === CLIST) { - return Struct.List.binaryDecode(decoder) - } else if (code === CMAP) { - return Struct.Map.binaryDecode(decoder) - } else if (code === CXML) { - return Struct.Xml.binaryDecode(decoder) - } else { - throw new Error('Unable to decode operation!') - } - } - - /* This is the only operation that is actually not a structure, because - it is not stored in the OS. This is why it _does not_ have an id - - op = { - target: Id - } - */ - Struct.Delete = { - encode: function (op) { - return { - target: op.target, - length: op.length || 0, - struct: 'Delete' - } - }, - binaryEncode: function (encoder, op) { - encoder.writeUint8(CDELETE) - encoder.writeOpID(op.target) - encoder.writeVarUint(op.length || 0) - }, - binaryDecode: function (decoder) { - decoder.skip8() - return { - target: decoder.readOpID(), - length: decoder.readVarUint(), - struct: 'Delete' - } - }, - requiredOps: function (op) { - return [] // [op.target] - }, - execute: function (op) { - return this.deleteOperation(op.target, op.length || 1) - } - } - - /* { - content: [any], - opContent: Id, - id: Id, - left: Id, - origin: Id, - right: Id, - parent: Id, - parentSub: string (optional), // child of Map type - } - */ - Struct.Insert = { - encode: function (op/* :Insertion */) /* :Insertion */ { - // TODO: you could not send the "left" property, then you also have to - // "op.left = null" in $execute or $decode - var e/* :any */ = { - id: op.id, - left: op.left, - right: op.right, - origin: op.origin, - parent: op.parent, - struct: op.struct - } - if (op.parentSub != null) { - e.parentSub = op.parentSub - } - if (op.hasOwnProperty('opContent')) { - e.opContent = op.opContent - } else { - e.content = op.content.slice() - } - - return e - }, - binaryEncode: function (encoder, op) { - encoder.writeUint8(CINSERT) - // compute info property - let contentIsText = op.content != null && op.content.every(c => typeof c === 'string' && c.length === 1) - let originIsLeft = Y.utils.compareIds(op.left, op.origin) - let info = - (op.parentSub != null ? 1 : 0) | - (op.opContent != null ? 2 : 0) | - (contentIsText ? 4 : 0) | - (originIsLeft ? 8 : 0) | - (op.left != null ? 16 : 0) | - (op.right != null ? 32 : 0) | - (op.origin != null ? 64 : 0) - encoder.writeUint8(info) - encoder.writeOpID(op.id) - encoder.writeOpID(op.parent) - if (info & 16) { - encoder.writeOpID(op.left) - } - if (info & 32) { - encoder.writeOpID(op.right) - } - if (!originIsLeft && info & 64) { - encoder.writeOpID(op.origin) - } - if (info & 1) { - // write parentSub - encoder.writeVarString(op.parentSub) - } - if (info & 2) { - // write opContent - encoder.writeOpID(op.opContent) - } else if (info & 4) { - // write text - encoder.writeVarString(op.content.join('')) - } else { - // convert to JSON and write - encoder.writeVarString(JSON.stringify(op.content)) - } - }, - binaryDecode: function (decoder) { - let op = { - struct: 'Insert' - } - decoder.skip8() - // get info property - let info = decoder.readUint8() - - op.id = decoder.readOpID() - op.parent = decoder.readOpID() - if (info & 16) { - op.left = decoder.readOpID() - } else { - op.left = null - } - if (info & 32) { - op.right = decoder.readOpID() - } else { - op.right = null - } - if (info & 8) { - // origin is left - op.origin = op.left - } else if (info & 64) { - op.origin = decoder.readOpID() - } else { - op.origin = null - } - if (info & 1) { - // has parentSub - op.parentSub = decoder.readVarString() - } - if (info & 2) { - // has opContent - op.opContent = decoder.readOpID() - } else if (info & 4) { - // has pure text content - op.content = decoder.readVarString().split('') - } else { - // has mixed content - let s = decoder.readVarString() - op.content = JSON.parse(s) - } - return op - }, - requiredOps: function (op) { - var ids = [] - if (op.left != null) { - ids.push(op.left) - } - if (op.right != null) { - ids.push(op.right) - } - if (op.origin != null && !Y.utils.compareIds(op.left, op.origin)) { - ids.push(op.origin) - } - // if (op.right == null && op.left == null) { - ids.push(op.parent) - - if (op.opContent != null) { - ids.push(op.opContent) - } - return ids - }, - getDistanceToOrigin: function (op) { - if (op.left == null) { - return 0 - } else { - var d = 0 - var o = this.getInsertion(op.left) - while (!Y.utils.matchesId(o, op.origin)) { - d++ - if (o.left == null) { - break - } else { - o = this.getInsertion(o.left) - } - } - return d - } - }, - /* - # $this has to find a unique position between origin and the next known character - # case 1: $origin equals $o.origin: the $creator parameter decides if left or right - # let $OL= [o1,o2,o3,o4], whereby $this is to be inserted between o1 and o4 - # o2,o3 and o4 origin is 1 (the position of o2) - # there is the case that $this.creator < o2.creator, but o3.creator < $this.creator - # then o2 knows o3. Since on another client $OL could be [o1,o3,o4] the problem is complex - # therefore $this would be always to the right of o3 - # case 2: $origin < $o.origin - # if current $this insert_position > $o origin: $this ins - # else $insert_position will not change - # (maybe we encounter case 1 later, then this will be to the right of $o) - # case 3: $origin > $o.origin - # $this insert_position is to the left of $o (forever!) - */ - execute: function (op) { - var i // loop counter - - // during this function some ops may get split into two pieces (e.g. with getInsertionCleanEnd) - // We try to merge them later, if possible - var tryToRemergeLater = [] - - if (op.origin != null) { // TODO: !== instead of != - // we save in origin that op originates in it - // we need that later when we eventually garbage collect origin (see transaction) - var origin = this.getInsertionCleanEnd(op.origin) - if (origin.originOf == null) { - origin.originOf = [] - } - origin.originOf.push(op.id) - this.setOperation(origin) - if (origin.right != null) { - tryToRemergeLater.push(origin.right) - } - } - var distanceToOrigin = i = Struct.Insert.getDistanceToOrigin.call(this, op) // most cases: 0 (starts from 0) - - // now we begin to insert op in the list of insertions.. - var o - var parent - var start - - // find o. o is the first conflicting operation - if (op.left != null) { - o = this.getInsertionCleanEnd(op.left) - if (!Y.utils.compareIds(op.left, op.origin) && o.right != null) { - // only if not added previously - tryToRemergeLater.push(o.right) - } - o = (o.right == null) ? null : this.getOperation(o.right) - } else { // left == null - parent = this.getOperation(op.parent) - let startId = op.parentSub ? parent.map[op.parentSub] : parent.start - start = startId == null ? null : this.getOperation(startId) - o = start - } - - // make sure to split op.right if necessary (also add to tryCombineWithLeft) - if (op.right != null) { - tryToRemergeLater.push(op.right) - this.getInsertionCleanStart(op.right) - } - - // handle conflicts - while (true) { - if (o != null && !Y.utils.compareIds(o.id, op.right)) { - var oOriginDistance = Struct.Insert.getDistanceToOrigin.call(this, o) - if (oOriginDistance === i) { - // case 1 - if (o.id[0] < op.id[0]) { - op.left = Y.utils.getLastId(o) - distanceToOrigin = i + 1 // just ignore o.content.length, doesn't make a difference - } - } else if (oOriginDistance < i) { - // case 2 - if (i - distanceToOrigin <= oOriginDistance) { - op.left = Y.utils.getLastId(o) - distanceToOrigin = i + 1 // just ignore o.content.length, doesn't make a difference - } - } else { - break - } - i++ - if (o.right != null) { - o = this.getInsertion(o.right) - } else { - o = null - } - } else { - break - } - } - - // reconnect.. - var left = null - var right = null - if (parent == null) { - parent = this.getOperation(op.parent) - } - - // reconnect left and set right of op - if (op.left != null) { - left = this.getInsertion(op.left) - // link left - op.right = left.right - left.right = op.id - - this.setOperation(left) - } else { - // set op.right from parent, if necessary - op.right = op.parentSub ? parent.map[op.parentSub] || null : parent.start - } - // reconnect right - if (op.right != null) { - // TODO: wanna connect right too? - right = this.getOperation(op.right) - right.left = Y.utils.getLastId(op) - - // if right exists, and it is supposed to be gc'd. Remove it from the gc - if (right.gc != null) { - if (right.content != null && right.content.length > 1) { - right = this.getInsertionCleanEnd(right.id) - } - this.store.removeFromGarbageCollector(right) - } - this.setOperation(right) - } - - // update parents .map/start/end properties - if (op.parentSub != null) { - if (left == null) { - parent.map[op.parentSub] = op.id - this.setOperation(parent) - } - // is a child of a map struct. - // Then also make sure that only the most left element is not deleted - // We do not call the type in this case (this is what the third parameter is for) - if (op.right != null) { - this.deleteOperation(op.right, 1, true) - } - if (op.left != null) { - this.deleteOperation(op.id, 1, true) - } - } else { - if (right == null || left == null) { - if (right == null) { - parent.end = Y.utils.getLastId(op) - } - if (left == null) { - parent.start = op.id - } - this.setOperation(parent) - } - } - - // try to merge original op.left and op.origin - for (i = 0; i < tryToRemergeLater.length; i++) { - var m = this.getOperation(tryToRemergeLater[i]) - this.tryCombineWithLeft(m) - } - } - } - - /* - { - start: null, - end: null, - struct: "List", - type: "", - id: this.os.getNextOpId(1) - } - */ - Struct.List = { - create: function (id) { - return { - start: null, - end: null, - struct: 'List', - id: id - } - }, - encode: function (op) { - var e = { - struct: 'List', - id: op.id, - type: op.type - } - return e - }, - binaryEncode: function (encoder, op) { - encoder.writeUint8(CLIST) - encoder.writeOpID(op.id) - encoder.writeVarString(op.type) - }, - binaryDecode: function (decoder) { - decoder.skip8() - let op = { - id: decoder.readOpID(), - type: decoder.readVarString(), - struct: 'List', - start: null, - end: null - } - return op - }, - requiredOps: function () { - /* - var ids = [] - if (op.start != null) { - ids.push(op.start) - } - if (op.end != null){ - ids.push(op.end) - } - return ids - */ - return [] - }, - execute: function (op) { - op.start = null - op.end = null - }, - ref: function (op, pos) { - if (op.start == null) { - return null - } - var res = null - var o = this.getOperation(op.start) - - while (true) { - if (!o.deleted) { - res = o - pos-- - } - if (pos >= 0 && o.right != null) { - o = this.getOperation(o.right) - } else { - break - } - } - return res - }, - map: function (o, f) { - o = o.start - var res = [] - while (o != null) { // TODO: change to != (at least some convention) - var operation = this.getOperation(o) - if (!operation.deleted) { - res.push(f(operation)) - } - o = operation.right - } - return res - } - } - - /* - { - map: {}, - struct: "Map", - type: "", - id: this.os.getNextOpId(1) - } - */ - Struct.Map = { - create: function (id) { - return { - id: id, - map: {}, - struct: 'Map' - } - }, - encode: function (op) { - var e = { - struct: 'Map', - type: op.type, - id: op.id, - map: {} // overwrite map!! - } - return e - }, - binaryEncode: function (encoder, op) { - encoder.writeUint8(CMAP) - encoder.writeOpID(op.id) - encoder.writeVarString(op.type) - }, - binaryDecode: function (decoder) { - decoder.skip8() - let op = { - id: decoder.readOpID(), - type: decoder.readVarString(), - struct: 'Map', - map: {} - } - return op - }, - requiredOps: function () { - return [] - }, - execute: function (op) { - op.start = null - op.end = null - }, - /* - Get a property by name - */ - get: function (op, name) { - var oid = op.map[name] - if (oid != null) { - var res = this.getOperation(oid) - if (res == null || res.deleted) { - return void 0 - } else if (res.opContent == null) { - return res.content[0] - } else { - return this.getType(res.opContent) - } - } - } - } - - /* - { - map: {}, - start: null, - end: null, - struct: "Xml", - type: "", - id: this.os.getNextOpId(1) - } - */ - Struct.Xml = { - create: function (id, args) { - let nodeName = args != null ? args.nodeName : null - return { - id: id, - map: {}, - start: null, - end: null, - struct: 'Xml', - nodeName - } - }, - encode: function (op) { - var e = { - struct: 'Xml', - type: op.type, - id: op.id, - map: {}, - nodeName: op.nodeName - } - return e - }, - binaryEncode: function (encoder, op) { - encoder.writeUint8(CXML) - encoder.writeOpID(op.id) - encoder.writeVarString(op.type) - encoder.writeVarString(op.nodeName) - }, - binaryDecode: function (decoder) { - decoder.skip8() - let op = { - id: decoder.readOpID(), - type: decoder.readVarString(), - struct: 'Xml', - map: {}, - start: null, - end: null, - nodeName: decoder.readVarString() - } - return op - }, - requiredOps: function () { - return [] - }, - execute: function () {}, - ref: Struct.List.ref, - map: Struct.List.map, - /* - Get a property by name - */ - get: Struct.Map.get - } -} diff --git a/src/Struct/Delete.js b/src/Struct/Delete.js new file mode 100644 index 00000000..922a3552 --- /dev/null +++ b/src/Struct/Delete.js @@ -0,0 +1,31 @@ +import StructManager from '../Util/StructManager' + +export default class Delete { + constructor () { + this._target = null + this._length = null + } + _fromBinary (y, decoder) { + this._targetID = decoder.readOpID() + this._length = decoder.readVarUint() + } + _toBinary (y, encoder) { + encoder.writeUint8(StructManager.getReference(this.constructor)) + encoder.writeOpID(this._targetID) + encoder.writeVarUint(this._length) + } + _integrate (y) { + let items = y.os.getItems(this._target, this._length) + for (let i = items.length - 1; i >= 0; i--) { + items[i]._delete() + } + // TODO: only broadcast if created by local user or if y.connector._forwardAppliedStructs.. + y.connector.broadcastStruct(this) + if (y.persistence !== null) { + y.persistence.saveOperations(this) + } + } + _logString () { + return `Delete - target: ${this._target}, len: ${this._length}` + } +} diff --git a/src/Struct/Item.js b/src/Struct/Item.js new file mode 100644 index 00000000..56953b23 --- /dev/null +++ b/src/Struct/Item.js @@ -0,0 +1,159 @@ +import StructManager from '../Util/StructManager' + +export default class Item { + constructor () { + this._id = null + this._origin = null + this._left = null + this._right = null + this._right_origin = null + this._parent = null + this._parentSub = null + this._deleted = false + } + get _length () { + return 1 + } + _getDistanceToOrigin () { + if (this.left == null) { + return 0 + } else { + var d = 0 + var o = this.left + while (o !== null && !this.origin.equals(o.id)) { + d++ + o = o.left + } + return d + } + } + _delete (y) { + this._deleted = true + y.ds.markDeleted(this._id, this._length) + } + /* + * - Integrate the struct so that other types/structs can see it + * - Add this struct to y.os + * - Check if this is struct deleted + */ + _integrate (y) { + if (this._id === null) { + this._id = y.ss.getNextID(this._length) + } + /* + # $this has to find a unique position between origin and the next known character + # case 1: $origin equals $o.origin: the $creator parameter decides if left or right + # let $OL= [o1,o2,o3,o4], whereby $this is to be inserted between o1 and o4 + # o2,o3 and o4 origin is 1 (the position of o2) + # there is the case that $this.creator < o2.creator, but o3.creator < $this.creator + # then o2 knows o3. Since on another client $OL could be [o1,o3,o4] the problem is complex + # therefore $this would be always to the right of o3 + # case 2: $origin < $o.origin + # if current $this insert_position > $o origin: $this ins + # else $insert_position will not change + # (maybe we encounter case 1 later, then this will be to the right of $o) + # case 3: $origin > $o.origin + # $this insert_position is to the left of $o (forever!) + */ + // handle conflicts + let o + // set o to the first conflicting item + if (this._left !== null) { + o = this._left._right + } else if (this._parentSub !== null) { + o = this._parent._map.get(this._parentSub) + } else { + o = this._parent._start + } + let conflictingItems = new Set() + let itemsBeforeOrigin = new Set() + // Let c in conflictingItems, b in itemsBeforeOrigin + // ***{origin}bbbb{this}{c,b}{c,b}{o}*** + // Note that conflictingItems is a subset of itemsBeforeOrigin + while (o !== null && o !== this._right) { + itemsBeforeOrigin.add(o) + if (this.origin === o.origin) { + // case 1 + if (o._id.user < this._id.user) { + this.left = o + conflictingItems = new Set() + } + } else if (itemsBeforeOrigin.has(o)) { + // case 2 + if (conflictingItems.has(o)) { + this.left = o + conflictingItems = new Set() + } + } else { + break + } + o = o.right + } + y.os.set(this) + y.ds.checkIfDeleted(this) + if (y.connector._forwardAppliedStructs || this._id.user === y.userID) { + y.connector.broadcastStruct(this) + } + if (y.persistence !== null) { + y.persistence.saveOperations(this) + } + } + _toBinary (y, encoder) { + encoder.writeUint8(StructManager.getReference(this.constructor)) + encoder.writeOpID(this._id) + encoder.writeOpID(this._parent._id) + encoder.writeVarString(this.parentSub === null ? '' : JSON.stringify(this.parentSub)) + encoder.writeOpID(this._left === null ? null : this._left._id) + encoder.writeOpID(this._right_origin === null ? null : this._right_origin._id) + encoder.writeOpID(this._origin === null ? null : this._origin._id) + } + _fromBinary (y, decoder) { + let missing = [] + this._id = decoder.readOpID() + let parent = decoder.readOpID() + let parentSub = decoder.readVarString() + if (parentSub.length > 0) { + this._parentSub = JSON.parse(parentSub) + } + let left = decoder.readOpID() + let right = decoder.readOpId() + let origin = decoder.readOpID() + if (parent !== null && this._parent === null) { + let _parent = y.os.get(parent) + if (_parent === null) { + missing.push(parent) + } else { + this._parent = _parent + } + } + if (origin !== null && this._origin === null) { + let _origin = y.os.getCleanStart(origin) + if (_origin === null) { + missing.push(origin) + } else { + this._origin = _origin + } + } + if (left !== null && this._left === null) { + let _left = y.os.getCleanEnd(left) + if (_left === null) { + // use origin instead + this._left = this._origin + } else { + this._left = _left + } + } + if (right !== null && this._right_origin === null) { + let _right = y.os.getCleanStart(right) + if (_right === null) { + missing.push(right) + } else { + this._right = _right + this._right_origin = _right + } + } + } + _logString () { + return `left: ${this._left}, origin: ${this._origin}, right: ${this._right}, parent: ${this._parent}, parentSub: ${this._parentSub}` + } +} diff --git a/src/Struct/ItemJSON.js b/src/Struct/ItemJSON.js new file mode 100644 index 00000000..3d0ade7e --- /dev/null +++ b/src/Struct/ItemJSON.js @@ -0,0 +1,32 @@ +import Item from './Item' + +export default class ItemJSON extends Item { + constructor () { + super() + this._content = null + } + get _length () { + return this._content.length + } + _fromBinary (y, decoder) { + let missing = super._fromBinary(y, decoder) + let len = decoder.readVarUint() + this._content = new Array(len) + for (let i = 0; i < len; i++) { + this._content[i] = JSON.parse(decoder.readVarString()) + } + return missing + } + _toBinary (y, encoder) { + super._toBinary(y, encoder) + let len = this._content.length + encoder.writeVarUint(len) + for (let i = 0; i < len; i++) { + encoder.writeVarString(JSON.stringify(this._content[i])) + } + } + _logString () { + let s = super._logString() + return 'ItemJSON: ' + s + } +} diff --git a/src/Struct/ItemString.js b/src/Struct/ItemString.js new file mode 100644 index 00000000..8bf9b59d --- /dev/null +++ b/src/Struct/ItemString.js @@ -0,0 +1,24 @@ +import Item from './Item' + +export default class ItemString extends Item { + constructor () { + super() + this._content = null + } + get _length () { + return this._content.length + } + _fromBinary (y, decoder) { + let missing = super._fromBinary(y, decoder) + this._content = decoder.readVarString() + return missing + } + _toBinary (y, encoder) { + super._toBinary(y, encoder) + encoder.writeVarString(this._content) + } + _logString () { + let s = super._logString() + return 'ItemString: ' + s + } +} diff --git a/src/Struct/Type.js b/src/Struct/Type.js new file mode 100644 index 00000000..5776c9b0 --- /dev/null +++ b/src/Struct/Type.js @@ -0,0 +1,26 @@ +import Item from './Item' + +export default class Type extends Item { + constructor () { + super() + this._map = new Map() + this._start = null + } + _delete (y) { + super._delete(y) + // delete map types + for (let value of this._map.values()) { + if (value instanceof Item && !value._deleted) { + value._delete() + } + } + // delete array types + let t = this._start + while (t !== null) { + if (!t._deleted) { + t._delete() + } + t = t._right + } + } +} diff --git a/src/Transaction.js b/src/Transaction.js deleted file mode 100644 index 69fc8fd6..00000000 --- a/src/Transaction.js +++ /dev/null @@ -1,1196 +0,0 @@ -import { BinaryEncoder, BinaryDecoder } from './Encoding.js' - -/* - Partial definition of a transaction - - A transaction provides all the the async functionality on a database. - - By convention, a transaction has the following properties: - * ss for StateSet - * os for OperationStore - * ds for DeleteStore - - A transaction must also define the following methods: - * checkDeleteStoreForState(state) - - When increasing the state of a user, an operation with an higher id - may already be garbage collected, and therefore it will never be received. - update the state to reflect this knowledge. This won't call a method to save the state! - * getDeleteSet(id) - - Get the delete set in a readable format: - { - "userX": [ - [5,1], // starting from position 5, one operations is deleted - [9,4] // starting from position 9, four operations are deleted - ], - "userY": ... - } - * getOpsFromDeleteSet(ds) -- TODO: just call this.deleteOperation(id) here - - get a set of deletions that need to be applied in order to get to - achieve the state of the supplied ds - * setOperation(op) - - write `op` to the database. - Note: this is allowed to return an in-memory object. - E.g. the Memory adapter returns the object that it has in-memory. - Changing values on this object will be stored directly in the database - without calling this function. Therefore, - setOperation may have no functionality in some adapters. This also has - implications on the way we use operations that were served from the database. - We try not to call copyObject, if not necessary. - * addOperation(op) - - add an operation to the database. - This may only be called once for every op.id - Must return a function that returns the next operation in the database (ordered by id) - * getOperation(id) - * removeOperation(id) - - remove an operation from the database. This is called when an operation - is garbage collected. - * setState(state) - - `state` is of the form - { - user: "1", - clock: 4 - } <- meaning that we have four operations from user "1" - (with these id's respectively: 0, 1, 2, and 3) - * getState(user) - * getStateVector() - - Get the state of the OS in the form - [{ - user: "userX", - clock: 11 - }, - .. - ] - * getStateSet() - - Get the state of the OS in the form - { - "userX": 11, - "userY": 22 - } - * getOperations(startSS) - - Get the all the operations that are necessary in order to achive the - stateSet of this user, starting from a stateSet supplied by another user - * makeOperationReady(ss, op) - - this is called only by `getOperations(startSS)`. It makes an operation - applyable on a given SS. -*/ -export default function extendTransaction (Y) { - class TransactionInterface { - /* :: - store: Y.AbstractDatabase; - ds: Store; - os: Store; - ss: Store; - */ - /* - Apply operations that this user created (no remote ones!) - * does not check for Struct.*.requiredOps() - * also broadcasts it through the connector - */ - applyCreatedOperations (ops) { - var send = [] - for (var i = 0; i < ops.length; i++) { - var op = ops[i] - this.store.tryExecute.call(this, op) - if (op.id == null || typeof op.id[1] !== 'string') { - send.push(Y.Struct[op.struct].encode(op)) - } - } - if (send.length > 0) { // TODO: && !this.store.forwardAppliedOperations (but then i don't send delete ops) - // is connected, and this is not going to be send in addOperation - this.store.y.connector.broadcastOps(send) - if (this.store.y.persistence != null) { - this.store.y.persistence.saveOperations(send) - } - } - } - - deleteList (start) { - while (start != null) { - start = this.getOperation(start) - if (!start.gc) { - start.gc = true - start.deleted = true - this.setOperation(start) - var delLength = start.content != null ? start.content.length : 1 - this.markDeleted(start.id, delLength) - if (start.opContent != null) { - this.deleteOperation(start.opContent) - } - this.store.queueGarbageCollector(start.id) - } - start = start.right - } - } - - /* - Mark an operation as deleted, and add it to the GC, if possible. - */ - deleteOperation (targetId, length, preventCallType) /* :Generator */ { - if (length == null) { - length = 1 - } - this.markDeleted(targetId, length) - while (length > 0) { - var callType = false - var target = this.os.findWithUpperBound([targetId[0], targetId[1] + length - 1]) - var targetLength = target != null && target.content != null ? target.content.length : 1 - if (target == null || target.id[0] !== targetId[0] || target.id[1] + targetLength <= targetId[1]) { - // does not exist or is not in the range of the deletion - target = null - length = 0 - } else { - // does exist, check if it is too long - if (!target.deleted) { - if (target.id[1] < targetId[1]) { - // starts to the left of the deletion range - target = this.getInsertionCleanStart(targetId) - targetLength = target.content.length // must have content property! - } - if (target.id[1] + targetLength > targetId[1] + length) { - // ends to the right of the deletion range - target = this.getInsertionCleanEnd([targetId[0], targetId[1] + length - 1]) - targetLength = target.content.length - } - } - length = target.id[1] - targetId[1] - } - - if (target != null) { - if (!target.deleted) { - callType = true - // set deleted & notify type - target.deleted = true - // delete containing lists - if (target.start != null) { - // TODO: don't do it like this .. -.- - this.deleteList(target.start) - // this.deleteList(target.id) -- do not gc itself because this may still get referenced - } - if (target.map != null) { - for (var name in target.map) { - this.deleteList(target.map[name]) - } - // TODO: here to.. (see above) - // this.deleteList(target.id) -- see above - } - if (target.opContent != null) { - this.deleteOperation(target.opContent) - // target.opContent = null - } - if (target.requires != null) { - for (var i = 0; i < target.requires.length; i++) { - this.deleteOperation(target.requires[i]) - } - } - } - var left - if (target.left != null) { - left = this.getInsertion(target.left) - } else { - left = null - } - - // set here because it was deleted and/or gc'd - this.setOperation(target) - - /* - Check if it is possible to add right to the gc. - Because this delete can't be responsible for left being gc'd, - we don't have to add left to the gc.. - */ - var right - if (target.right != null) { - right = this.getOperation(target.right) - } else { - right = null - } - if (callType && !preventCallType) { - this.store.operationAdded(this, { - struct: 'Delete', - target: target.id, - length: targetLength, - targetParent: target.parent - }) - } - // need to gc in the end! - this.store.addToGarbageCollector.call(this, target, left) - if (right != null) { - this.store.addToGarbageCollector.call(this, right, target) - } - } - } - } - /* - Mark an operation as deleted&gc'd - */ - markGarbageCollected (id, len) { - // this.mem.push(["gc", id]); - this.store.addToDebug('this.markGarbageCollected(', id, ', ', len, ')') - var n = this.markDeleted(id, len) - if (n.id[1] < id[1] && !n.gc) { - // un-extend left - var newlen = n.len - (id[1] - n.id[1]) - n.len -= newlen - this.ds.put(n) - n = {id: id, len: newlen, gc: false} - this.ds.put(n) - } - // get prev&next before adding a new operation - var prev = this.ds.findPrev(id) - var next = this.ds.findNext(id) - - if (id[1] + len < n.id[1] + n.len && !n.gc) { - // un-extend right - this.ds.put({id: [id[0], id[1] + len], len: n.len - len, gc: false}) - n.len = len - } - // set gc'd - n.gc = true - // can extend left? - if ( - prev != null && - prev.gc && - Y.utils.compareIds([prev.id[0], prev.id[1] + prev.len], n.id) - ) { - prev.len += n.len - this.ds.delete(n.id) - n = prev - // ds.put n here? - } - // can extend right? - if ( - next != null && - next.gc && - Y.utils.compareIds([n.id[0], n.id[1] + n.len], next.id) - ) { - n.len += next.len - this.ds.delete(next.id) - } - this.ds.put(n) - this.updateState(n.id[0]) - } - /* - Mark an operation as deleted. - - returns the delete node - */ - markDeleted (id, length) { - if (length == null) { - length = 1 - } - // this.mem.push(["del", id]); - var n = this.ds.findWithUpperBound(id) - if (n != null && n.id[0] === id[0]) { - if (n.id[1] <= id[1] && id[1] <= n.id[1] + n.len) { - // id is in n's range - var diff = id[1] + length - (n.id[1] + n.len) // overlapping right - if (diff > 0) { - // id+length overlaps n - if (!n.gc) { - n.len += diff - } else { - diff = n.id[1] + n.len - id[1] // overlapping left (id till n.end) - if (diff < length) { - // a partial deletion - n = {id: [id[0], id[1] + diff], len: length - diff, gc: false} - this.ds.put(n) - } else { - // already gc'd - throw new Error( - 'DS reached an inconsistent state. Please report this issue!' - ) - } - } - } else { - // no overlapping, already deleted - return n - } - } else { - // cannot extend left (there is no left!) - n = {id: id, len: length, gc: false} - this.ds.put(n) // TODO: you double-put !! - } - } else { - // cannot extend left - n = {id: id, len: length, gc: false} - this.ds.put(n) - } - // can extend right? - var next = this.ds.findNext(n.id) - if ( - next != null && - n.id[0] === next.id[0] && - n.id[1] + n.len >= next.id[1] - ) { - diff = n.id[1] + n.len - next.id[1] // from next.start to n.end - while (diff >= 0) { - // n overlaps with next - if (next.gc) { - // gc is stronger, so reduce length of n - n.len -= diff - if (diff >= next.len) { - // delete the missing range after next - diff = diff - next.len // missing range after next - if (diff > 0) { - this.ds.put(n) // unneccessary? TODO! - this.markDeleted([next.id[0], next.id[1] + next.len], diff) - } - } - break - } else { - // we can extend n with next - if (diff > next.len) { - // n is even longer than next - // get next.next, and try to extend it - var _next = this.ds.findNext(next.id) - this.ds.delete(next.id) - if (_next == null || n.id[0] !== _next.id[0]) { - break - } else { - next = _next - diff = n.id[1] + n.len - next.id[1] // from next.start to n.end - // continue! - } - } else { - // n just partially overlaps with next. extend n, delete next, and break this loop - n.len += next.len - diff - this.ds.delete(next.id) - break - } - } - } - } - this.ds.put(n) - return n - } - /* - Call this method when the client is connected&synced with the - other clients (e.g. master). This will query the database for - operations that can be gc'd and add them to the garbage collector. - */ - garbageCollectAfterSync () { - // debugger - if (this.store.gc1.length > 0 || this.store.gc2.length > 0) { - console.warn('gc should be empty after sync') - } - if (!this.store.gc) { - return - } - this.os.iterate(this, null, null, function (op) { - if (op.gc) { - delete op.gc - this.setOperation(op) - } - if (op.parent != null) { - var parentDeleted = this.isDeleted(op.parent) - if (parentDeleted) { - op.gc = true - if (!op.deleted) { - this.markDeleted(op.id, op.content != null ? op.content.length : 1) - op.deleted = true - if (op.opContent != null) { - this.deleteOperation(op.opContent) - } - if (op.requires != null) { - for (var i = 0; i < op.requires.length; i++) { - this.deleteOperation(op.requires[i]) - } - } - } - this.setOperation(op) - this.store.gc1.push(op.id) // this is ok becaues its shortly before sync (otherwise use queueGarbageCollector!) - return - } - } - if (op.deleted) { - var left = null - if (op.left != null) { - left = this.getInsertion(op.left) - } - this.store.addToGarbageCollector.call(this, op, left) - } - }) - } - /* - Really remove an op and all its effects. - The complicated case here is the Insert operation: - * reset left - * reset right - * reset parent.start - * reset parent.end - * reset origins of all right ops - */ - garbageCollectOperation (id) { - this.store.addToDebug('this.garbageCollectOperation(', id, ')') - var o = this.getOperation(id) - this.markGarbageCollected(id, (o != null && o.content != null) ? o.content.length : 1) // always mark gc'd - // if op exists, then clean that mess up.. - if (o != null) { - var deps = [] - if (o.opContent != null) { - deps.push(o.opContent) - } - if (o.requires != null) { - deps = deps.concat(o.requires) - } - for (var i = 0; i < deps.length; i++) { - var dep = this.getOperation(deps[i]) - if (dep != null) { - if (!dep.deleted) { - this.deleteOperation(dep.id) - dep = this.getOperation(dep.id) - } - dep.gc = true - this.setOperation(dep) - this.store.queueGarbageCollector(dep.id) - } else { - this.markGarbageCollected(deps[i], 1) - } - } - - // remove gc'd op from the left op, if it exists - if (o.left != null) { - var left = this.getInsertion(o.left) - left.right = o.right - this.setOperation(left) - } - // remove gc'd op from the right op, if it exists - // also reset origins of right ops - if (o.right != null) { - var right = this.getOperation(o.right) - right.left = o.left - this.setOperation(right) - - if (o.originOf != null && o.originOf.length > 0) { - // find new origin of right ops - // origin is the first left operation - var neworigin = o.left - - // reset origin of all right ops (except first right - duh!), - - /* ** The following code does not rely on the the originOf property ** - I recently added originOf to all Insert Operations (see Struct.Insert.execute), - which saves which operations originate in a Insert operation. - Garbage collecting without originOf is more memory efficient, but is nearly impossible for large texts, or lists! - But I keep this code for now - ``` - // reset origin of right - right.origin = neworigin - // search until you find origin pointer to the left of o - if (right.right != null) { - var i = this.getOperation(right.right) - var ids = [o.id, o.right] - while (ids.some(function (id) { - return Y.utils.compareIds(id, i.origin) - })) { - if (Y.utils.compareIds(i.origin, o.id)) { - // reset origin of i - i.origin = neworigin - this.setOperation(i) - } - // get next i - if (i.right == null) { - break - } else { - ids.push(i.id) - i = this.getOperation(i.right) - } - } - } - ``` - */ - // ** Now the new implementation starts ** - // reset neworigin of all originOf[*] - for (var _i in o.originOf) { - var originsIn = this.getOperation(o.originOf[_i]) - if (originsIn != null) { - originsIn.origin = neworigin - this.setOperation(originsIn) - } - } - if (neworigin != null) { - var neworigin_ = this.getInsertion(neworigin) - if (neworigin_.originOf == null) { - neworigin_.originOf = o.originOf - } else { - neworigin_.originOf = o.originOf.concat(neworigin_.originOf) - } - this.setOperation(neworigin_) - } - // we don't need to set right here, because - // right should be in o.originOf => it is set it the previous for loop - } - } - // o may originate in another operation. - // Since o is deleted, we have to reset o.origin's `originOf` property - if (o.origin != null) { - var origin = this.getInsertion(o.origin) - origin.originOf = origin.originOf.filter(function (_id) { - return !Y.utils.compareIds(id, _id) - }) - this.setOperation(origin) - } - var parent - if (o.parent != null) { - parent = this.getOperation(o.parent) - } - // remove gc'd op from parent, if it exists - if (parent != null) { - var setParent = false // whether to save parent to the os - if (o.parentSub != null) { - if (Y.utils.compareIds(parent.map[o.parentSub], o.id)) { - setParent = true - if (o.right != null) { - parent.map[o.parentSub] = o.right - } else { - delete parent.map[o.parentSub] - } - } - } else { - if (Y.utils.compareIds(parent.start, o.id)) { - // gc'd op is the start - setParent = true - parent.start = o.right - } - if (Y.utils.matchesId(o, parent.end)) { - // gc'd op is the end - setParent = true - parent.end = o.left - } - } - if (setParent) { - this.setOperation(parent) - } - } - // finally remove it from the os - this.removeOperation(o.id) - } - } - checkDeleteStoreForState (state) { - var n = this.ds.findWithUpperBound([state.user, state.clock]) - if (n != null && n.id[0] === state.user && n.gc) { - state.clock = Math.max(state.clock, n.id[1] + n.len) - } - } - updateState (user) { - var state = this.getState(user) - this.checkDeleteStoreForState(state) - var o = this.getInsertion([user, state.clock]) - var oLength = (o != null && o.content != null) ? o.content.length : 1 - while (o != null && user === o.id[0] && o.id[1] <= state.clock && o.id[1] + oLength > state.clock) { - // either its a new operation (1. case), or it is an operation that was deleted, but is not yet in the OS - state.clock += oLength - this.checkDeleteStoreForState(state) - o = this.os.findNext(o.id) - oLength = (o != null && o.content != null) ? o.content.length : 1 - } - this.setState(state) - } - /* - apply a delete set in order to get - the state of the supplied ds - */ - applyDeleteSet (decoder) { - var deletions = [] - - let dsLength = decoder.readUint32() - for (let i = 0; i < dsLength; i++) { - let user = decoder.readVarUint() - let dv = [] - let dvLength = decoder.readVarUint() - for (let j = 0; j < dvLength; j++) { - let from = decoder.readVarUint() - let len = decoder.readVarUint() - let gc = decoder.readUint8() === 1 - dv.push([from, len, gc]) - } - var pos = 0 - var d = dv[pos] - this.ds.iterate(this, [user, 0], [user, Number.MAX_VALUE], function (n) { - // cases: - // 1. d deletes something to the right of n - // => go to next n (break) - // 2. d deletes something to the left of n - // => create deletions - // => reset d accordingly - // *)=> if d doesn't delete anything anymore, go to next d (continue) - // 3. not 2) and d deletes something that also n deletes - // => reset d so that it doesn't contain n's deletion - // *)=> if d does not delete anything anymore, go to next d (continue) - while (d != null) { - var diff = 0 // describe the diff of length in 1) and 2) - if (n.id[1] + n.len <= d[0]) { - // 1) - break - } else if (d[0] < n.id[1]) { - // 2) - // delete maximum the len of d - // else delete as much as possible - diff = Math.min(n.id[1] - d[0], d[1]) - deletions.push([user, d[0], diff, d[2]]) - } else { - // 3) - diff = n.id[1] + n.len - d[0] // never null (see 1) - if (d[2] && !n.gc) { - // d marks as gc'd but n does not - // then delete either way - deletions.push([user, d[0], Math.min(diff, d[1]), d[2]]) - } - } - if (d[1] <= diff) { - // d doesn't delete anything anymore - d = dv[++pos] - } else { - d[0] = d[0] + diff // reset pos - d[1] = d[1] - diff // reset length - } - } - }) - // for the rest.. just apply it - for (; pos < dv.length; pos++) { - d = dv[pos] - deletions.push([user, d[0], d[1], d[2]]) - } - } - for (var i = 0; i < deletions.length; i++) { - var del = deletions[i] - // always try to delete.. - this.deleteOperation([del[0], del[1]], del[2]) - if (del[3]) { - // gc.. - this.markGarbageCollected([del[0], del[1]], del[2]) // always mark gc'd - // remove operation.. - var counter = del[1] + del[2] - while (counter >= del[1]) { - var o = this.os.findWithUpperBound([del[0], counter - 1]) - if (o == null) { - break - } - var oLen = o.content != null ? o.content.length : 1 - if (o.id[0] !== del[0] || o.id[1] + oLen <= del[1]) { - // not in range - break - } - if (o.id[1] + oLen > del[1] + del[2]) { - // overlaps right - o = this.getInsertionCleanEnd([del[0], del[1] + del[2] - 1]) - } - if (o.id[1] < del[1]) { - // overlaps left - o = this.getInsertionCleanStart([del[0], del[1]]) - } - counter = o.id[1] - this.garbageCollectOperation(o.id) - } - } - if (this.store.forwardAppliedOperations || this.store.y.persistence != null) { - var ops = [] - ops.push({struct: 'Delete', target: [del[0], del[1]], length: del[2]}) - if (this.store.forwardAppliedOperations) { - this.store.y.connector.broadcastOps(ops) - } - if (this.store.y.persistence != null) { - this.store.y.persistence.saveOperations(ops) - } - } - } - } - isGarbageCollected (id) { - var n = this.ds.findWithUpperBound(id) - return n != null && n.id[0] === id[0] && id[1] < n.id[1] + n.len && n.gc - } - /* - A DeleteSet (ds) describes all the deleted ops in the OS - */ - writeDeleteSet (encoder) { - var ds = new Map() - this.ds.iterate(this, null, null, function (n) { - var user = n.id[0] - var counter = n.id[1] - var len = n.len - var gc = n.gc - var dv = ds.get(user) - if (dv === void 0) { - dv = [] - ds.set(user, dv) - } - dv.push([counter, len, gc]) - }) - let keys = Array.from(ds.keys()) - encoder.writeUint32(keys.length) - for (var i = 0; i < keys.length; i++) { - let user = keys[i] - let deletions = ds.get(user) - encoder.writeVarUint(user) - encoder.writeVarUint(deletions.length) - for (var j = 0; j < deletions.length; j++) { - let del = deletions[j] - encoder.writeVarUint(del[0]) - encoder.writeVarUint(del[1]) - encoder.writeUint8(del[2] ? 1 : 0) - } - } - } - isDeleted (id) { - var n = this.ds.findWithUpperBound(id) - return n != null && n.id[0] === id[0] && id[1] < n.id[1] + n.len - } - setOperation (op) { - this.os.put(op) - return op - } - addOperation (op) { - this.os.put(op) - // case op is created by this user, op is already broadcasted in applyCreatedOperations - if (op.id[0] !== this.store.userId && typeof op.id[1] !== 'string') { - if (this.store.forwardAppliedOperations) { - // is connected, and this is not going to be send in addOperation - this.store.y.connector.broadcastOps([op]) - } - if (this.store.y.persistence != null) { - this.store.y.persistence.saveOperations([op]) - } - } - } - // if insertion, try to combine with left insertion (if both have content property) - tryCombineWithLeft (op) { - if ( - op != null && - op.left != null && - op.content != null && - op.left[0] === op.id[0] && - Y.utils.compareIds(op.left, op.origin) - ) { - var left = this.getInsertion(op.left) - if (left.content != null && - left.id[1] + left.content.length === op.id[1] && - left.originOf.length === 1 && - !left.gc && !left.deleted && - !op.gc && !op.deleted - ) { - // combine! - if (op.originOf != null) { - left.originOf = op.originOf - } else { - delete left.originOf - } - left.content = left.content.concat(op.content) - left.right = op.right - this.os.delete(op.id) - this.setOperation(left) - } - } - } - getInsertion (id) { - var ins = this.os.findWithUpperBound(id) - if (ins == null) { - return null - } else { - var len = ins.content != null ? ins.content.length : 1 // in case of opContent - if (id[0] === ins.id[0] && id[1] < ins.id[1] + len) { - return ins - } else { - return null - } - } - } - getInsertionCleanStartEnd (id) { - this.getInsertionCleanStart(id) - return this.getInsertionCleanEnd(id) - } - // Return an insertion such that id is the first element of content - // This function manipulates an operation, if necessary - getInsertionCleanStart (id) { - var ins = this.getInsertion(id) - if (ins != null) { - if (ins.id[1] === id[1]) { - return ins - } else { - var left = Y.utils.copyObject(ins) - ins.content = left.content.splice(id[1] - ins.id[1]) - ins.id = id - var leftLid = Y.utils.getLastId(left) - ins.origin = leftLid - left.originOf = [ins.id] - left.right = ins.id - ins.left = leftLid - // debugger // check - this.setOperation(left) - this.setOperation(ins) - if (left.gc) { - this.store.queueGarbageCollector(ins.id) - } - return ins - } - } else { - return null - } - } - // Return an insertion such that id is the last element of content - // This function manipulates an operation, if necessary - getInsertionCleanEnd (id) { - var ins = this.getInsertion(id) - if (ins != null) { - if (ins.content == null || (ins.id[1] + ins.content.length - 1 === id[1])) { - return ins - } else { - var right = Y.utils.copyObject(ins) - right.content = ins.content.splice(id[1] - ins.id[1] + 1) // cut off remainder - right.id = [id[0], id[1] + 1] - var insLid = Y.utils.getLastId(ins) - right.origin = insLid - ins.originOf = [right.id] - ins.right = right.id - right.left = insLid - // debugger // check - this.setOperation(right) - this.setOperation(ins) - if (ins.gc) { - this.store.queueGarbageCollector(right.id) - } - return ins - } - } else { - return null - } - } - getOperation (id/* :any */)/* :Transaction */ { - var o = this.os.find(id) - if (id[0] !== 0xFFFFFF || o != null) { - return o - } else { // type is string - // generate this operation? - var comp = id[1].split('_') - if (comp.length > 1) { - var struct = comp[0] - let type = Y[comp[1]] - let args = null - if (type != null) { - args = Y.utils.parseTypeDefinition(type, comp[3]) - } - var op = Y.Struct[struct].create(id, args) - op.type = comp[1] - this.setOperation(op) - return op - } else { - throw new Error( - 'Unexpected case. Operation cannot be generated correctly!' + - 'Incompatible Yjs version?' - ) - } - } - } - removeOperation (id) { - this.os.delete(id) - } - setState (state) { - var val = { - id: [state.user], - clock: state.clock - } - this.ss.put(val) - } - getState (user) { - var n = this.ss.find([user]) - var clock = n == null ? null : n.clock - if (clock == null) { - clock = 0 - } - return { - user: user, - clock: clock - } - } - getStateVector () { - var stateVector = [] - this.ss.iterate(this, null, null, function (n) { - stateVector.push({ - user: n.id[0], - clock: n.clock - }) - }) - return stateVector - } - getStateSet () { - var ss = {} - this.ss.iterate(this, null, null, function (n) { - ss[n.id[0]] = n.clock - }) - return ss - } - writeStateSet (encoder) { - let lenPosition = encoder.pos - let len = 0 - encoder.writeUint32(0) - this.ss.iterate(this, null, null, function (n) { - encoder.writeVarUint(n.id[0]) - encoder.writeVarUint(n.clock) - len++ - }) - encoder.setUint32(lenPosition, len) - return len === 0 - } - /* - Here, we make all missing operations executable for the receiving user. - - Notes: - startSS: denotes to the SV that the remote user sent - currSS: denotes to the state vector that the user should have if he - applies all already sent operations (increases is each step) - - We face several problems: - * Execute op as is won't work because ops depend on each other - -> find a way so that they do not anymore - * When changing left, must not go more to the left than the origin - * When changing right, you have to consider that other ops may have op - as their origin, this means that you must not set one of these ops - as the new right (interdependencies of ops) - * can't just go to the right until you find the first known operation, - With currSS - -> interdependency of ops is a problem - With startSS - -> leads to inconsistencies when two users join at the same time. - Then the position depends on the order of execution -> error! - - Solution: - -> re-create originial situation - -> set op.left = op.origin (which never changes) - -> set op.right - to the first operation that is known (according to startSS) - or to the first operation that has an origin that is not to the - right of op. - -> Enforces unique execution order -> happy user - - Improvements: TODO - * Could set left to origin, or the first known operation - (startSS or currSS.. ?) - -> Could be necessary when I turn GC again. - -> Is a bad(ish) idea because it requires more computation - - What we do: - * Iterate over all missing operations. - * When there is an operation, where the right op is known, send this op all missing ops to the left to the user - * I explained above what we have to do with each operation. Here is how we do it efficiently: - 1. Go to the left until you find either op.origin, or a known operation (let o denote current operation in the iteration) - 2. Found a known operation -> set op.left = o, and send it to the user. stop - 3. Found o = op.origin -> set op.left = op.origin, and send it to the user. start again from 1. (set op = o) - 4. Found some o -> set o.right = op, o.left = o.origin, send it to the user, continue - */ - getOperations (startSS) { - // TODO: use bounds here! - if (startSS == null) { - startSS = new Map() - } - var send = [] - - var endSV = this.getStateVector() - for (let endState of endSV) { - let user = endState.user - if (user === 0xFFFFFF) { - continue - } - let startPos = startSS.get(user) || 0 - if (startPos > 0) { - // There is a change that [user, startPos] is in a composed Insertion (with a smaller counter) - // find out if that is the case - let firstMissing = this.getInsertion([user, startPos]) - if (firstMissing != null) { - // update startPos - startPos = firstMissing.id[1] - } - } - startSS.set(user, startPos) - } - for (let endState of endSV) { - let user = endState.user - let startPos = startSS.get(user) - if (user === 0xFFFFFF) { - continue - } - this.os.iterate(this, [user, startPos], [user, Number.MAX_VALUE], function (op) { - op = Y.Struct[op.struct].encode(op) - if (op.struct !== 'Insert') { - send.push(op) - } else if (op.right == null || op.right[1] < (startSS.get(op.right[0]) || 0)) { - // case 1. op.right is known - // this case is only reached if op.right is known. - // => this is not called for op.left, as op.right is unknown - let o = op - // Remember: ? - // -> set op.right - // 1. to the first operation that is known (according to startSS) - // 2. or to the first operation that has an origin that is not to the - // right of op. - // For this we maintain a list of ops which origins are not found yet. - var missingOrigins = [op] - var newright = op.right - while (true) { - if (o.left == null) { - op.left = null - send.push(op) - /* not necessary, as o is already sent.. - if (!Y.utils.compareIds(o.id, op.id) && o.id[1] >= (startSS.get(o.id[0]) || 0)) { - // o is not op && o is unknown - o = Y.Struct[op.struct].encode(o) - o.right = missingOrigins[missingOrigins.length - 1].id - send.push(o) - } - */ - break - } - o = this.getInsertion(o.left) - // we set another o, check if we can reduce $missingOrigins - while (missingOrigins.length > 0 && Y.utils.matchesId(o, missingOrigins[missingOrigins.length - 1].origin)) { - missingOrigins.pop() - } - if (o.id[1] < (startSS.get(o.id[0]) || 0)) { - // case 2. o is known - op.left = Y.utils.getLastId(o) - send.push(op) - break - } else if (Y.utils.matchesId(o, op.origin)) { - // case 3. o is op.origin - op.left = op.origin - send.push(op) - op = Y.Struct[op.struct].encode(o) - op.right = newright - if (missingOrigins.length > 0) { - throw new Error( - 'Reached inconsistent OS state.' + - 'Operations are not correctly connected.' - ) - } - missingOrigins = [op] - } else { - // case 4. send o, continue to find op.origin - var s = Y.Struct[op.struct].encode(o) - s.right = missingOrigins[missingOrigins.length - 1].id - s.left = s.origin - send.push(s) - missingOrigins.push(o) - } - } - } - }) - } - return send.reverse() - } - - writeOperations (encoder, decoder) { - let ss = new Map() - let ssLength = decoder.readUint32() - for (let i = 0; i < ssLength; i++) { - let user = decoder.readVarUint() - let clock = decoder.readVarUint() - ss.set(user, clock) - } - let ops = this.getOperations(ss) - encoder.writeUint32(ops.length) - for (let i = 0; i < ops.length; i++) { - let op = ops[i] - Y.Struct[op.struct].binaryEncode(encoder, Y.Struct[op.struct].encode(op)) - } - } - - toBinary () { - let encoder = new BinaryEncoder() - this.writeOperationsUntransformed(encoder) - this.writeDeleteSet(encoder) - return encoder.createBuffer() - } - - fromBinary (buffer) { - let decoder = new BinaryDecoder(buffer) - this.applyOperationsUntransformed(decoder) - this.applyDeleteSet(decoder) - } - - /* - * Get the plain untransformed operations from the database. - * You can apply these operations using .applyOperationsUntransformed(ops) - * - */ - writeOperationsUntransformed (encoder) { - let lenPosition = encoder.pos - let len = 0 - encoder.writeUint32(0) // placeholder - this.os.iterate(this, null, null, function (op) { - if (op.id[0] !== 0xFFFFFF) { - len++ - Y.Struct[op.struct].binaryEncode(encoder, Y.Struct[op.struct].encode(op)) - } - }) - encoder.setUint32(lenPosition, len) - this.writeStateSet(encoder) - } - applyOperationsUntransformed (decoder) { - let len = decoder.readUint32() - for (let i = 0; i < len; i++) { - let op = Y.Struct.binaryDecodeOperation(decoder) - this.os.put(op) - } - this.os.iterate(this, null, null, function (op) { - if (op.parent != null) { - if (op.struct === 'Insert') { - // update parents .map/start/end properties - if (op.parentSub != null && op.left == null) { - // op is child of Map - let parent = this.getOperation(op.parent) - parent.map[op.parentSub] = op.id - this.setOperation(parent) - } else if (op.right == null || op.left == null) { - let parent = this.getOperation(op.parent) - if (op.right == null) { - parent.end = Y.utils.getLastId(op) - } - if (op.left == null) { - parent.start = op.id - } - this.setOperation(parent) - } - } - } - }) - let stateSetLength = decoder.readUint32() - for (let i = 0; i < stateSetLength; i++) { - let user = decoder.readVarUint() - let clock = decoder.readVarUint() - this.ss.put({ - id: [user], - clock: clock - }) - } - } - /* this is what we used before.. use this as a reference.. - makeOperationReady (startSS, op) { - op = Y.Struct[op.struct].encode(op) - op = Y.utils.copyObject(op) -- use copyoperation instead now! - var o = op - var ids = [op.id] - // search for the new op.right - // it is either the first known op (according to startSS) - // or the o that has no origin to the right of op - // (this is why we use the ids array) - while (o.right != null) { - var right = this.getOperation(o.right) - if (o.right[1] < (startSS[o.right[0]] || 0) || !ids.some(function (id) { - return Y.utils.compareIds(id, right.origin) - })) { - break - } - ids.push(o.right) - o = right - } - op.right = o.right - op.left = op.origin - return op - } - */ - flush () { - this.os.flush() - this.ss.flush() - this.ds.flush() - } - } - Y.Transaction = TransactionInterface -} diff --git a/src/Type/YArray.js b/src/Type/YArray.js new file mode 100644 index 00000000..9894ca33 --- /dev/null +++ b/src/Type/YArray.js @@ -0,0 +1,87 @@ +import Type from '../Struct/Type' +import ItemJSON from '../Struct/ItemJSON' + +export default class YArray extends Type { + forEach (f) { + let pos = 0 + let n = this._start + while (n !== null) { + let content = n._getContent() + for (let i = 0; i < content.length; i++) { + pos++ + let c = content[i] + if (!c._deleted) { + f(content[i], pos, this) + } + } + n = n._right + } + } + [Symbol.iterator] () { + return { + next: function () { + while (this._item !== null && (this._item._deleted || this._item._content.length <= this._itemElement)) { + // item is deleted or itemElement does not exist (is deleted) + this._item = this._item._right + this._itemElement = 0 + } + if (this._item === null) { + return { + done: true + } + } else { + return { + value: [this._count, this._item._content[this._itemElement++]], + done: false + } + } + }, + _item: this._start, + _itemElement: 0, + _count: 0 + } + } + insert (pos, content) { + let left = this._start + let right + let count = 0 + while (left !== null && !left._deleted) { + if (pos < count + left._content.length) { + [left, right] = left._splitAt(pos - count) + break + } + left = left.right + } + if (pos > count) { + throw new Error('Position exceeds array range!') + } + let prevJsonIns = null + for (let i = 0; i < content.length; i++) { + let c = content[i] + if (c instanceof Type) { + if (prevJsonIns === null) { + prevJsonIns._integrate(this._y) + prevJsonIns = null + } + c._left = left + c._origin = left + c._right = right + c._parent = this + } else { + if (prevJsonIns === null) { + prevJsonIns = new ItemJSON() + prevJsonIns._origin = left + prevJsonIns._left = left + prevJsonIns._right = right + prevJsonIns._parent = this + prevJsonIns._content = [] + } + prevJsonIns._content.push(c) + } + } + } + _logString () { + let s = super._logString() + return 'YArray: ' + s + } +} diff --git a/src/Type/YMap.js b/src/Type/YMap.js new file mode 100644 index 00000000..4e170d92 --- /dev/null +++ b/src/Type/YMap.js @@ -0,0 +1,32 @@ +import Type from '../Struct/Type' +import Item from '../Struct/Item' +import ItemJSON from '../Struct/ItemJSON' + +export default class YMap extends Type { + set (key, value) { + let old = this._map.get(key) + let v + if (value instanceof Item) { + v = value + } else { + let v = new ItemJSON() + v._content = JSON.stringify(value) + } + v._right = old + v._parent = this + v._parentSub = key + v._integrate() + } + get (key) { + let v = this._map.get(key) + if (v instanceof Type) { + return v + } else { + return v._content[v._content.length - 1] + } + } + _logString () { + let s = super._logString() + return 'YMap: ' + s + } +} diff --git a/src/Type/YText.js b/src/Type/YText.js new file mode 100644 index 00000000..e69de29b diff --git a/src/Type/YXml.js b/src/Type/YXml.js new file mode 100644 index 00000000..e69de29b diff --git a/src/Util/EventHandler.js b/src/Util/EventHandler.js new file mode 100644 index 00000000..d92d3184 --- /dev/null +++ b/src/Util/EventHandler.js @@ -0,0 +1,34 @@ + +export default class EventHandler { + constructor () { + this.eventListeners = [] + } + destroy () { + this.eventListeners = null + } + addEventListener (f) { + this.eventListeners.push(f) + } + removeEventListener (f) { + this.eventListeners = this.eventListeners.filter(function (g) { + return f !== g + }) + } + removeAllEventListeners () { + this.eventListeners = [] + } + callEventListeners (event) { + for (var i = 0; i < this.eventListeners.length; i++) { + try { + this.eventListeners[i](event) + } catch (e) { + /* + Your observer threw an error. This error was caught so that Yjs + can ensure data consistency! In order to debug this error you + have to check "Pause On Caught Exceptions" in developer tools. + */ + console.error(e) + } + } + } +} diff --git a/src/Util/ID.js b/src/Util/ID.js new file mode 100644 index 00000000..17a68571 --- /dev/null +++ b/src/Util/ID.js @@ -0,0 +1,32 @@ + +import StructManager from './StructManager' + +export class ID { + constructor (user, clock) { + this.user = user + this.clock = clock + } + clone () { + return new ID(this.user, this.clock) + } + equals (id) { + return id !== null && id.user === this.user && id.clock === this.user + } + lessThan (id) { + return this.user < id.user || (this.user === id.user && this.clock < id.clock) + } +} + +export class RootID { + constructor (name, typeConstructor) { + this.user = -1 + this.name = name + this.type = StructManager.getReference(typeConstructor) + } + equals (id) { + return id !== null && id.user === this.user && id.name === this.name && id.type === this.type + } + lessThan (id) { + return this.user < id.user || (this.user === id.user && (this.name < id.name || (this.name === id.name && this.type < id.type))) + } +} diff --git a/src/Util/NamedEventHandler.js b/src/Util/NamedEventHandler.js new file mode 100644 index 00000000..c69631c7 --- /dev/null +++ b/src/Util/NamedEventHandler.js @@ -0,0 +1,28 @@ +export default class NamedEventHandler { + constructor () { + this._eventListener = {} + } + on (name, f) { + if (this._eventListener[name] == null) { + this._eventListener[name] = [] + } + this._eventListener[name].push(f) + } + off (name, f) { + if (name == null || f == null) { + throw new Error('You must specify event name and function!') + } + let listener = this._eventListener[name] || [] + this._eventListener[name] = listener.filter(e => e !== f) + } + emit (name, value) { + let listener = this._eventListener[name] || [] + if (name === 'error' && listener.length === 0) { + console.error(value) + } + listener.forEach(l => l(value)) + } + destroy () { + this._eventListener = null + } +} diff --git a/src/Util/Tree.js b/src/Util/Tree.js new file mode 100644 index 00000000..71b58842 --- /dev/null +++ b/src/Util/Tree.js @@ -0,0 +1,474 @@ + +class N { + // A created node is always red! + constructor (val) { + this.val = val + this.color = true + this._left = null + this._right = null + this._parent = null + } + isRed () { return this.color } + isBlack () { return !this.color } + redden () { this.color = true; return this } + blacken () { this.color = false; return this } + get grandparent () { + return this.parent.parent + } + get parent () { + return this._parent + } + get sibling () { + return (this === this.parent.left) + ? this.parent.right : this.parent.left + } + get left () { + return this._left + } + get right () { + return this._right + } + set left (n) { + if (n !== null) { + n._parent = this + } + this._left = n + } + set right (n) { + if (n !== null) { + n._parent = this + } + this._right = n + } + rotateLeft (tree) { + var parent = this.parent + var newParent = this.right + var newRight = this.right.left + newParent.left = this + this.right = newRight + if (parent === null) { + tree.root = newParent + newParent._parent = null + } else if (parent.left === this) { + parent.left = newParent + } else if (parent.right === this) { + parent.right = newParent + } else { + throw new Error('The elements are wrongly connected!') + } + } + next () { + if (this.right !== null) { + // search the most left node in the right tree + var o = this.right + while (o.left !== null) { + o = o.left + } + return o + } else { + var p = this + while (p.parent !== null && p !== p.parent.left) { + p = p.parent + } + return p.parent + } + } + prev () { + if (this.left !== null) { + // search the most right node in the left tree + var o = this.left + while (o.right !== null) { + o = o.right + } + return o + } else { + var p = this + while (p.parent !== null && p !== p.parent.right) { + p = p.parent + } + return p.parent + } + } + rotateRight (tree) { + var parent = this.parent + var newParent = this.left + var newLeft = this.left.right + newParent.right = this + this.left = newLeft + if (parent === null) { + tree.root = newParent + newParent._parent = null + } else if (parent.left === this) { + parent.left = newParent + } else if (parent.right === this) { + parent.right = newParent + } else { + throw new Error('The elements are wrongly connected!') + } + } + getUncle () { + // we can assume that grandparent exists when this is called! + if (this.parent === this.parent.parent.left) { + return this.parent.parent.right + } else { + return this.parent.parent.left + } + } +} + +/* + * This is a Red Black Tree implementation + */ +export default class Tree { + constructor () { + this.root = null + this.length = 0 + } + findNext (id) { + var nextID = id.clone() + nextID.clock += 1 + return this.findWithLowerBound(nextID) + } + findPrev (id) { + let prevID = id.clone() + prevID.clock -= 1 + return this.findWithUpperBound(prevID) + } + findNodeWithLowerBound (from) { + var o = this.root + if (o === null) { + return null + } else { + while (true) { + if (from === null || (from.lessThan(o.val.id) && o.left !== null)) { + // o is included in the bound + // try to find an element that is closer to the bound + o = o.left + } else if (from !== null && o.val.id.lessThan(from)) { + // o is not within the bound, maybe one of the right elements is.. + if (o.right !== null) { + o = o.right + } else { + // there is no right element. Search for the next bigger element, + // this should be within the bounds + return o.next() + } + } else { + return o + } + } + } + } + findNodeWithUpperBound (to) { + if (to === void 0) { + throw new Error('You must define from!') + } + var o = this.root + if (o === null) { + return null + } else { + while (true) { + if ((to === null || o.val.id.lessThan(to)) && o.right !== null) { + // o is included in the bound + // try to find an element that is closer to the bound + o = o.right + } else if (to !== null && to.lessThan(o.val.id)) { + // o is not within the bound, maybe one of the left elements is.. + if (o.left !== null) { + o = o.left + } else { + // there is no left element. Search for the prev smaller element, + // this should be within the bounds + return o.prev() + } + } else { + return o + } + } + } + } + findSmallestNode () { + var o = this.root + while (o != null && o.left != null) { + o = o.left + } + return o + } + findWithLowerBound (from) { + var n = this.findNodeWithLowerBound(from) + return n == null ? null : n.val + } + findWithUpperBound (to) { + var n = this.findNodeWithUpperBound(to) + return n == null ? null : n.val + } + iterate (from, to, f) { + var o + if (from === null) { + o = this.findSmallestNode() + } else { + o = this.findNodeWithLowerBound(from) + } + while ( + o !== null && + ( + to === null || // eslint-disable-line no-unmodified-loop-condition + o.val.id.lessThan(to) || + o.val.id.equals(to) + ) + ) { + f(o.val) + o = o.next() + } + } + find (id) { + let n = this.findNode(id) + if (n !== null) { + return n.val + } else { + return null + } + } + findNode (id) { + var o = this.root + if (o === null) { + return false + } else { + while (true) { + if (o === null) { + return false + } + if (id.lessThan(o.val.id)) { + o = o.left + } else if (o.val.id.lessThan(id)) { + o = o.right + } else { + return o + } + } + } + } + delete (id) { + if (id == null || id.constructor !== Array) { + throw new Error('id is expected to be an Array!') + } + var d = this.findNode(id) + if (d == null) { + // throw new Error('Element does not exist!') + return + } + this.length-- + if (d.left !== null && d.right !== null) { + // switch d with the greates element in the left subtree. + // o should have at most one child. + var o = d.left + // find + while (o.right !== null) { + o = o.right + } + // switch + d.val = o.val + d = o + } + // d has at most one child + // let n be the node that replaces d + var isFakeChild + var child = d.left || d.right + if (child === null) { + isFakeChild = true + child = new N({id: 0}) + child.blacken() + d.right = child + } else { + isFakeChild = false + } + + if (d.parent === null) { + if (!isFakeChild) { + this.root = child + child.blacken() + child._parent = null + } else { + this.root = null + } + return + } else if (d.parent.left === d) { + d.parent.left = child + } else if (d.parent.right === d) { + d.parent.right = child + } else { + throw new Error('Impossible!') + } + if (d.isBlack()) { + if (child.isRed()) { + child.blacken() + } else { + this._fixDelete(child) + } + } + this.root.blacken() + if (isFakeChild) { + if (child.parent.left === child) { + child.parent.left = null + } else if (child.parent.right === child) { + child.parent.right = null + } else { + throw new Error('Impossible #3') + } + } + } + _fixDelete (n) { + function isBlack (node) { + return node !== null ? node.isBlack() : true + } + function isRed (node) { + return node !== null ? node.isRed() : false + } + if (n.parent === null) { + // this can only be called after the first iteration of fixDelete. + return + } + // d was already replaced by the child + // d is not the root + // d and child are black + var sibling = n.sibling + if (isRed(sibling)) { + // make sibling the grandfather + n.parent.redden() + sibling.blacken() + if (n === n.parent.left) { + n.parent.rotateLeft(this) + } else if (n === n.parent.right) { + n.parent.rotateRight(this) + } else { + throw new Error('Impossible #2') + } + sibling = n.sibling + } + // parent, sibling, and children of n are black + if (n.parent.isBlack() && + sibling.isBlack() && + isBlack(sibling.left) && + isBlack(sibling.right) + ) { + sibling.redden() + this._fixDelete(n.parent) + } else if (n.parent.isRed() && + sibling.isBlack() && + isBlack(sibling.left) && + isBlack(sibling.right) + ) { + sibling.redden() + n.parent.blacken() + } else { + if (n === n.parent.left && + sibling.isBlack() && + isRed(sibling.left) && + isBlack(sibling.right) + ) { + sibling.redden() + sibling.left.blacken() + sibling.rotateRight(this) + sibling = n.sibling + } else if (n === n.parent.right && + sibling.isBlack() && + isRed(sibling.right) && + isBlack(sibling.left) + ) { + sibling.redden() + sibling.right.blacken() + sibling.rotateLeft(this) + sibling = n.sibling + } + sibling.color = n.parent.color + n.parent.blacken() + if (n === n.parent.left) { + sibling.right.blacken() + n.parent.rotateLeft(this) + } else { + sibling.left.blacken() + n.parent.rotateRight(this) + } + } + } + put (v) { + var node = new N(v) + if (this.root !== null) { + var p = this.root // p abbrev. parent + while (true) { + if (node.val.id.lessThan(p.val.id)) { + if (p.left === null) { + p.left = node + break + } else { + p = p.left + } + } else if (p.val.id.lessThan(node.val.id)) { + if (p.right === null) { + p.right = node + break + } else { + p = p.right + } + } else { + p.val = node.val + return p + } + } + this._fixInsert(node) + } else { + this.root = node + } + this.length++ + this.root.blacken() + return node + } + _fixInsert (n) { + if (n.parent === null) { + n.blacken() + return + } else if (n.parent.isBlack()) { + return + } + var uncle = n.getUncle() + if (uncle !== null && uncle.isRed()) { + // Note: parent: red, uncle: red + n.parent.blacken() + uncle.blacken() + n.grandparent.redden() + this._fixInsert(n.grandparent) + } else { + // Note: parent: red, uncle: black or null + // Now we transform the tree in such a way that + // either of these holds: + // 1) grandparent.left.isRed + // and grandparent.left.left.isRed + // 2) grandparent.right.isRed + // and grandparent.right.right.isRed + if (n === n.parent.right && n.parent === n.grandparent.left) { + n.parent.rotateLeft(this) + // Since we rotated and want to use the previous + // cases, we need to set n in such a way that + // n.parent.isRed again + n = n.left + } else if (n === n.parent.left && n.parent === n.grandparent.right) { + n.parent.rotateRight(this) + // see above + n = n.right + } + // Case 1) or 2) hold from here on. + // Now traverse grandparent, make parent a black node + // on the highest level which holds two red nodes. + n.parent.blacken() + n.grandparent.redden() + if (n === n.parent.left) { + // Case 1 + n.grandparent.rotateRight(this) + } else { + // Case 2 + n.grandparent.rotateLeft(this) + } + } + } + flush () {} +} diff --git a/src/Util/deleteItemRange.js b/src/Util/deleteItemRange.js new file mode 100644 index 00000000..804f7958 --- /dev/null +++ b/src/Util/deleteItemRange.js @@ -0,0 +1,9 @@ +import Delete from '../Struct/Delete' +import ID from './ID' + +export default function deleteItemRange (y, user, clock, length) { + let del = new Delete() + del._target = new ID(user, clock) + del._length = length + del._integrate(y) +} diff --git a/src/Util/generateUserID.js b/src/Util/generateUserID.js new file mode 100644 index 00000000..e0bc4993 --- /dev/null +++ b/src/Util/generateUserID.js @@ -0,0 +1,16 @@ +/* global crypto */ + +export default function generateUserID () { + if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) { + // browser + let arr = new Uint32Array(1) + crypto.getRandomValues(arr) + return arr[0] + } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) { + // node + let buf = crypto.randomBytes(4) + return new Uint32Array(buf.buffer)[0] + } else { + return Math.ceil(Math.random() * 0xFFFFFFFF) + } +} diff --git a/src/Util/relativePosition.js b/src/Util/relativePosition.js new file mode 100644 index 00000000..11aab809 --- /dev/null +++ b/src/Util/relativePosition.js @@ -0,0 +1,45 @@ +import ID from './ID' + +export function getRelativePosition (type, offset) { + if (offset === 0) { + return ['startof', type._id.user, type._id.clock] + } else { + let t = type.start + while (t !== null && t.length < offset) { + if (!t._deleted) { + offset -= t.length + } + t = t._right + } + return [t._id.user, t._id.clock + offset - 1] + } +} + +export function fromRelativePosition (y, rpos) { + if (rpos[0] === 'startof') { + return { + type: y.os.get(new ID(rpos[1], rpos[2])), + offset: 0 + } + } else { + let offset = 0 + let struct = y.os.findNodeWithUpperBound(new ID(rpos[0], rpos[1])) + let parent = struct._parent + if (parent._deleted) { + return null + } + if (!struct.deleted) { + offset = rpos[1] - struct._id.clock + } + while (struct.left !== null) { + struct = struct.left + if (!struct.deleted) { + offset += struct._length + } + } + return { + type: parent, + offset: offset + } + } +} diff --git a/src/Util/structReferences.js b/src/Util/structReferences.js new file mode 100644 index 00000000..4396234e --- /dev/null +++ b/src/Util/structReferences.js @@ -0,0 +1,30 @@ +import YArray from '../Type/YArray' +import YMap from '../Type/YMap' +import YText from '../Type/YText' +import YXml from '../Type/YXml' + +import ItemJSON from '../Struct/ItemJSON' +import ItemString from '../Struct/ItemString' + +const structs = new Map() +const references = new Map() + +function addStruct (reference, structConstructor) { + structs.set(reference, structConstructor) + references.set(structConstructor, reference) +} + +export function getStruct (reference) { + return structs.get(reference) +} + +export function getReference (typeConstructor) { + return references.get(typeConstructor) +} + +addStruct(0, YArray) +addStruct(1, YMap) +addStruct(2, YText) +addStruct(3, YXml) +addStruct(4, ItemJSON) +addStruct(5, ItemString) diff --git a/src/Util/writeJSONToType.js b/src/Util/writeJSONToType.js new file mode 100644 index 00000000..58ad3049 --- /dev/null +++ b/src/Util/writeJSONToType.js @@ -0,0 +1,33 @@ + +import YMap from '../Type/YMap' +import YArray from '../Type/YArray' + +export function writeObjectToYMap (object, type) { + for (var key in object) { + var val = object[key] + if (Array.isArray(val)) { + type.set(key, YArray) + writeArrayToYArray(val, type.get(key)) + } else if (typeof val === 'object') { + type.set(key, YMap) + writeObjectToYMap(val, type.get(key)) + } else { + type.set(key, val) + } + } +} + +export function writeArrayToYArray (array, type) { + for (var i = array.length - 1; i >= 0; i--) { + var val = array[i] + if (Array.isArray(val)) { + type.insert(0, [YArray]) + writeArrayToYArray(val, type.get(0)) + } else if (typeof val === 'object') { + type.insert(0, [YMap]) + writeObjectToYMap(val, type.get(0)) + } else { + type.insert(0, [val]) + } + } +} diff --git a/src/Utils.js b/src/Utils.js deleted file mode 100644 index e0d3aa15..00000000 --- a/src/Utils.js +++ /dev/null @@ -1,935 +0,0 @@ -/* globals crypto */ - -import { BinaryDecoder, BinaryEncoder } from './Encoding.js' - -/* - EventHandler is an helper class for constructing custom types. - - Why: When constructing custom types, you sometimes want your types to work - synchronous: E.g. - ``` Synchronous - mytype.setSomething("yay") - mytype.getSomething() === "yay" - ``` - versus - ``` Asynchronous - mytype.setSomething("yay") - mytype.getSomething() === undefined - mytype.waitForSomething().then(function(){ - mytype.getSomething() === "yay" - }) - ``` - - The structures usually work asynchronously (you have to wait for the - database request to finish). EventHandler helps you to make your type - synchronous. -*/ - -export default function Utils (Y) { - Y.utils = { - BinaryDecoder: BinaryDecoder, - BinaryEncoder: BinaryEncoder - } - - Y.utils.bubbleEvent = function (type, event) { - type.eventHandler.callEventListeners(event) - event.path = [] - while (type != null && type._deepEventHandler != null) { - type._deepEventHandler.callEventListeners(event) - var parent = null - if (type._parent != null) { - parent = type.os.getType(type._parent) - } - if (parent != null && parent._getPathToChild != null) { - event.path = [parent._getPathToChild(type._model)].concat(event.path) - type = parent - } else { - type = null - } - } - } - - Y.utils.getRelativePosition = function (type, offset) { - if (type == null) { - return null - } else { - if (type._content.length <= offset) { - return ['endof', type._model[0], type._model[1]] - } else { - return type._content[offset].id - } - } - } - - Y.utils.fromRelativePosition = function (y, id) { - var offset = 0 - var op - if (id[0] === 'endof') { - id = y.db.os.find(id.slice(1)).end - op = y.db.os.findNodeWithUpperBound(id).val - if (!op.deleted) { - offset = op.content != null ? op.content.length : 1 - } - } else { - op = y.db.os.findNodeWithUpperBound(id).val - if (!op.deleted) { - offset = id[1] - op.id[1] - } - } - - var type = y.db.getType(op.parent) - if (type == null || y.db.os.find(op.parent).deleted) { - return null - } - - while (op.left != null) { - op = y.db.os.findNodeWithUpperBound(op.left).val - if (!op.deleted) { - offset += op.content != null ? op.content.length : 1 - } - } - return { - type: type, - offset: offset - } - } - - class NamedEventHandler { - constructor () { - this._eventListener = {} - } - on (name, f) { - if (this._eventListener[name] == null) { - this._eventListener[name] = [] - } - this._eventListener[name].push(f) - } - off (name, f) { - if (name == null || f == null) { - throw new Error('You must specify event name and function!') - } - let listener = this._eventListener[name] || [] - this._eventListener[name] = listener.filter(e => e !== f) - } - emit (name, value) { - let listener = this._eventListener[name] || [] - if (name === 'error' && listener.length === 0) { - console.error(value) - } - listener.forEach(l => l(value)) - } - destroy () { - this._eventListener = null - } - } - Y.utils.NamedEventHandler = NamedEventHandler - - class EventListenerHandler { - constructor () { - this.eventListeners = [] - } - destroy () { - this.eventListeners = null - } - /* - Basic event listener boilerplate... - */ - addEventListener (f) { - this.eventListeners.push(f) - } - removeEventListener (f) { - this.eventListeners = this.eventListeners.filter(function (g) { - return f !== g - }) - } - removeAllEventListeners () { - this.eventListeners = [] - } - callEventListeners (event) { - for (var i = 0; i < this.eventListeners.length; i++) { - try { - var _event = {} - for (var name in event) { - _event[name] = event[name] - } - this.eventListeners[i](_event) - } catch (e) { - /* - Your observer threw an error. This error was caught so that Yjs - can ensure data consistency! In order to debug this error you - have to check "Pause On Caught Exceptions" in developer tools. - */ - console.error(e) - } - } - } - } - Y.utils.EventListenerHandler = EventListenerHandler - - class EventHandler extends EventListenerHandler { - /* :: - waiting: Array; - awaiting: number; - onevent: Function; - eventListeners: Array; - */ - /* - onevent: is called when the structure changes. - - Note: "awaiting opertations" is used to denote operations that were - prematurely called. Events for received operations can not be executed until - all prematurely called operations were executed ("waiting operations") - */ - constructor (onevent /* : Function */) { - super() - this.waiting = [] - this.awaiting = 0 - this.onevent = onevent - } - destroy () { - super.destroy() - this.waiting = null - this.onevent = null - } - /* - Call this when a new operation arrives. It will be executed right away if - there are no waiting operations, that you prematurely executed - */ - receivedOp (op) { - if (this.awaiting <= 0) { - this.onevent(op) - } else if (op.struct === 'Delete') { - var self = this - var checkDelete = function checkDelete (d) { - if (d.length == null) { - throw new Error('This shouldn\'t happen! d.length must be defined!') - } - // we check if o deletes something in self.waiting - // if so, we remove the deleted operation - for (var w = 0; w < self.waiting.length; w++) { - var i = self.waiting[w] - if (i.struct === 'Insert' && i.id[0] === d.target[0]) { - var iLength = i.hasOwnProperty('content') ? i.content.length : 1 - var dStart = d.target[1] - var dEnd = d.target[1] + (d.length || 1) - var iStart = i.id[1] - var iEnd = i.id[1] + iLength - // Check if they don't overlap - if (iEnd <= dStart || dEnd <= iStart) { - // no overlapping - continue - } - // we check all overlapping cases. All cases: - /* - 1) iiiii - ddddd - --> modify i and d - 2) iiiiiii - ddddd - --> modify i, remove d - 3) iiiiiii - ddd - --> remove d, modify i, and create another i (for the right hand side) - 4) iiiii - ddddddd - --> remove i, modify d - 5) iiiiiii - ddddddd - --> remove both i and d (**) - 6) iiiiiii - ddddd - --> modify i, remove d - 7) iii - ddddddd - --> remove i, create and apply two d with checkDelete(d) (**) - 8) iiiii - ddddddd - --> remove i, modify d (**) - 9) iiiii - ddddd - --> modify i and d - (**) (also check if i contains content or type) - */ - // TODO: I left some debugger statements, because I want to debug all cases once in production. REMEMBER END TODO - if (iStart < dStart) { - if (dStart < iEnd) { - if (iEnd < dEnd) { - // Case 1 - // remove the right part of i's content - i.content.splice(dStart - iStart) - // remove the start of d's deletion - d.length = dEnd - iEnd - d.target = [d.target[0], iEnd] - continue - } else if (iEnd === dEnd) { - // Case 2 - i.content.splice(dStart - iStart) - // remove d, we do that by simply ending this function - return - } else { // (dEnd < iEnd) - // Case 3 - var newI = { - id: [i.id[0], dEnd], - content: i.content.slice(dEnd - iStart), - struct: 'Insert' - } - self.waiting.push(newI) - i.content.splice(dStart - iStart) - return - } - } - } else if (dStart === iStart) { - if (iEnd < dEnd) { - // Case 4 - d.length = dEnd - iEnd - d.target = [d.target[0], iEnd] - i.content = [] - continue - } else if (iEnd === dEnd) { - // Case 5 - self.waiting.splice(w, 1) - return - } else { // (dEnd < iEnd) - // Case 6 - i.content = i.content.slice(dEnd - iStart) - i.id = [i.id[0], dEnd] - return - } - } else { // (dStart < iStart) - if (iStart < dEnd) { - // they overlap - /* - 7) iii - ddddddd - --> remove i, create and apply two d with checkDelete(d) (**) - 8) iiiii - ddddddd - --> remove i, modify d (**) - 9) iiiii - ddddd - --> modify i and d - */ - if (iEnd < dEnd) { - // Case 7 - // debugger // TODO: You did not test this case yet!!!! (add the debugger here) - self.waiting.splice(w, 1) - checkDelete({ - target: [d.target[0], dStart], - length: iStart - dStart, - struct: 'Delete' - }) - checkDelete({ - target: [d.target[0], iEnd], - length: iEnd - dEnd, - struct: 'Delete' - }) - return - } else if (iEnd === dEnd) { - // Case 8 - self.waiting.splice(w, 1) - w-- - d.length -= iLength - continue - } else { // dEnd < iEnd - // Case 9 - d.length = iStart - dStart - i.content.splice(0, dEnd - iStart) - i.id = [i.id[0], dEnd] - continue - } - } - } - } - } - // finished with remaining operations - self.waiting.push(d) - } - if (op.key == null) { - // deletes in list - checkDelete(op) - } else { - // deletes in map - this.waiting.push(op) - } - } else { - this.waiting.push(op) - } - } - /* - You created some operations, and you want the `onevent` function to be - called right away. Received operations will not be executed untill all - prematurely called operations are executed - */ - awaitAndPrematurelyCall (ops) { - this.awaiting++ - ops.map(Y.utils.copyOperation).forEach(this.onevent) - } - awaitOps (transaction, f, args) { - function notSoSmartSort (array) { - // this function sorts insertions in a executable order - var result = [] - while (array.length > 0) { - for (var i = 0; i < array.length; i++) { - var independent = true - for (var j = 0; j < array.length; j++) { - if (Y.utils.matchesId(array[j], array[i].left)) { - // array[i] depends on array[j] - independent = false - break - } - } - if (independent) { - result.push(array.splice(i, 1)[0]) - i-- - } - } - } - return result - } - var before = this.waiting.length - // somehow create new operations - f.apply(transaction, args) - // remove all appended ops / awaited ops - this.waiting.splice(before) - if (this.awaiting > 0) this.awaiting-- - // if there are no awaited ops anymore, we can update all waiting ops, and send execute them (if there are still no awaited ops) - if (this.awaiting === 0 && this.waiting.length > 0) { - // update all waiting ops - for (let i = 0; i < this.waiting.length; i++) { - var o = this.waiting[i] - if (o.struct === 'Insert') { - var _o = transaction.getInsertion(o.id) - if (_o.parentSub != null && _o.left != null) { - // if o is an insertion of a map struc (parentSub is defined), then it shouldn't be necessary to compute left - this.waiting.splice(i, 1) - i-- // update index - } else if (!Y.utils.compareIds(_o.id, o.id)) { - // o got extended - o.left = [o.id[0], o.id[1] - 1] - } else if (_o.left == null) { - o.left = null - } else { - // find next undeleted op - var left = transaction.getInsertion(_o.left) - while (left.deleted != null) { - if (left.left != null) { - left = transaction.getInsertion(left.left) - } else { - left = null - break - } - } - o.left = left != null ? Y.utils.getLastId(left) : null - } - } - } - // the previous stuff was async, so we have to check again! - // We also pull changes from the bindings, if there exists such a method, this could increase awaiting too - if (this._pullChanges != null) { - this._pullChanges() - } - if (this.awaiting === 0) { - // sort by type, execute inserts first - var ins = [] - var dels = [] - this.waiting.forEach(function (o) { - if (o.struct === 'Delete') { - dels.push(o) - } else { - ins.push(o) - } - }) - this.waiting = [] - // put in executable order - ins = notSoSmartSort(ins) - // this.onevent can trigger the creation of another operation - // -> check if this.awaiting increased & stop computation if it does - for (var i = 0; i < ins.length; i++) { - if (this.awaiting === 0) { - this.onevent(ins[i]) - } else { - this.waiting = this.waiting.concat(ins.slice(i)) - break - } - } - for (i = 0; i < dels.length; i++) { - if (this.awaiting === 0) { - this.onevent(dels[i]) - } else { - this.waiting = this.waiting.concat(dels.slice(i)) - break - } - } - } - } - } - // TODO: Remove awaitedInserts and awaitedDeletes in favor of awaitedOps, as they are deprecated and do not always work - // Do this in one of the coming releases that are breaking anyway - /* - Call this when you successfully awaited the execution of n Insert operations - */ - awaitedInserts (n) { - var ops = this.waiting.splice(this.waiting.length - n) - for (var oid = 0; oid < ops.length; oid++) { - var op = ops[oid] - if (op.struct === 'Insert') { - for (var i = this.waiting.length - 1; i >= 0; i--) { - let w = this.waiting[i] - // TODO: do I handle split operations correctly here? Super unlikely, but yeah.. - // Also: can this case happen? Can op be inserted in the middle of a larger op that is in $waiting? - if (w.struct === 'Insert') { - if (Y.utils.matchesId(w, op.left)) { - // include the effect of op in w - w.right = op.id - // exclude the effect of w in op - op.left = w.left - } else if (Y.utils.compareIds(w.id, op.right)) { - // similar.. - w.left = Y.utils.getLastId(op) - op.right = w.right - } - } - } - } else { - throw new Error('Expected Insert Operation!') - } - } - this._tryCallEvents(n) - } - /* - Call this when you successfully awaited the execution of n Delete operations - */ - awaitedDeletes (n, newLeft) { - var ops = this.waiting.splice(this.waiting.length - n) - for (var j = 0; j < ops.length; j++) { - var del = ops[j] - if (del.struct === 'Delete') { - if (newLeft != null) { - for (var i = 0; i < this.waiting.length; i++) { - let w = this.waiting[i] - // We will just care about w.left - if (w.struct === 'Insert' && Y.utils.compareIds(del.target, w.left)) { - w.left = newLeft - } - } - } - } else { - throw new Error('Expected Delete Operation!') - } - } - this._tryCallEvents(n) - } - /* (private) - Try to execute the events for the waiting operations - */ - _tryCallEvents () { - function notSoSmartSort (array) { - var result = [] - while (array.length > 0) { - for (var i = 0; i < array.length; i++) { - var independent = true - for (var j = 0; j < array.length; j++) { - if (Y.utils.matchesId(array[j], array[i].left)) { - // array[i] depends on array[j] - independent = false - break - } - } - if (independent) { - result.push(array.splice(i, 1)[0]) - i-- - } - } - } - return result - } - if (this.awaiting > 0) this.awaiting-- - if (this.awaiting === 0 && this.waiting.length > 0) { - var ins = [] - var dels = [] - this.waiting.forEach(function (o) { - if (o.struct === 'Delete') { - dels.push(o) - } else { - ins.push(o) - } - }) - ins = notSoSmartSort(ins) - ins.forEach(this.onevent) - dels.forEach(this.onevent) - this.waiting = [] - } - } - } - Y.utils.EventHandler = EventHandler - - /* - Default class of custom types! - */ - class CustomType { - getPath () { - var parent = null - if (this._parent != null) { - parent = this.os.getType(this._parent) - } - if (parent != null && parent._getPathToChild != null) { - var firstKey = parent._getPathToChild(this._model) - var parentKeys = parent.getPath() - parentKeys.push(firstKey) - return parentKeys - } else { - return [] - } - } - } - Y.utils.CustomType = CustomType - - /* - A wrapper for the definition of a custom type. - Every custom type must have three properties: - - * struct - - Structname of this type - * initType - - Given a model, creates a custom type - * class - - the constructor of the custom type (e.g. in order to inherit from a type) - */ - class CustomTypeDefinition { // eslint-disable-line - /* :: - struct: any; - initType: any; - class: Function; - name: String; - */ - constructor (def) { - if (def.struct == null || - def.initType == null || - def.class == null || - def.name == null || - def.createType == null - ) { - throw new Error('Custom type was not initialized correctly!') - } - this.struct = def.struct - this.initType = def.initType - this.createType = def.createType - this.class = def.class - this.name = def.name - if (def.appendAdditionalInfo != null) { - this.appendAdditionalInfo = def.appendAdditionalInfo - } - this.parseArguments = (def.parseArguments || function () { - return [this] - }).bind(this) - this.parseArguments.typeDefinition = this - } - } - Y.utils.CustomTypeDefinition = CustomTypeDefinition - - Y.utils.isTypeDefinition = function isTypeDefinition (v) { - if (v != null) { - if (v instanceof Y.utils.CustomTypeDefinition) return [v] - else if (v.constructor === Array && v[0] instanceof Y.utils.CustomTypeDefinition) return v - else if (v instanceof Function && v.typeDefinition instanceof Y.utils.CustomTypeDefinition) return [v.typeDefinition] - } - return false - } - - /* - Make a flat copy of an object - (just copy properties) - */ - function copyObject (o) { - var c = {} - for (var key in o) { - c[key] = o[key] - } - return c - } - Y.utils.copyObject = copyObject - - /* - Copy an operation, so that it can be manipulated. - Note: You must not change subproperties (except o.content)! - */ - function copyOperation (o) { - o = copyObject(o) - if (o.content != null) { - o.content = o.content.map(function (c) { return c }) - } - return o - } - - Y.utils.copyOperation = copyOperation - - /* - Defines a smaller relation on Id's - */ - function smaller (a, b) { - return a[0] < b[0] || (a[0] === b[0] && (a[1] < b[1] || typeof a[1] < typeof b[1])) - } - Y.utils.smaller = smaller - - function inDeletionRange (del, ins) { - return del.target[0] === ins[0] && del.target[1] <= ins[1] && ins[1] < del.target[1] + (del.length || 1) - } - Y.utils.inDeletionRange = inDeletionRange - - function compareIds (id1, id2) { - if (id1 == null || id2 == null) { - return id1 === id2 - } else { - return id1[0] === id2[0] && id1[1] === id2[1] - } - } - Y.utils.compareIds = compareIds - - function matchesId (op, id) { - if (id == null || op == null) { - return id === op - } else { - if (id[0] === op.id[0]) { - if (op.content == null) { - return id[1] === op.id[1] - } else { - return id[1] >= op.id[1] && id[1] < op.id[1] + op.content.length - } - } - } - return false - } - Y.utils.matchesId = matchesId - - function getLastId (op) { - if (op.content == null || op.content.length === 1) { - return op.id - } else { - return [op.id[0], op.id[1] + op.content.length - 1] - } - } - Y.utils.getLastId = getLastId - - function createEmptyOpsArray (n) { - var a = new Array(n) - for (var i = 0; i < a.length; i++) { - a[i] = { - id: [null, null] - } - } - return a - } - - function createSmallLookupBuffer (Store) { - /* - This buffer implements a very small buffer that temporarily stores operations - after they are read / before they are written. - The buffer basically implements FIFO. Often requested lookups will be re-queued every time they are looked up / written. - - It can speed up lookups on Operation Stores and State Stores. But it does not require notable use of memory or processing power. - - Good for os and ss, bot not for ds (because it often uses methods that require a flush) - - I tried to optimize this for performance, therefore no highlevel operations. - */ - class SmallLookupBuffer extends Store { - constructor (arg1, arg2) { - // super(...arguments) -- do this when this is supported by stable nodejs - super(arg1, arg2) - this.writeBuffer = createEmptyOpsArray(5) - this.readBuffer = createEmptyOpsArray(10) - } - find (id, noSuperCall) { - var i, r - for (i = this.readBuffer.length - 1; i >= 0; i--) { - r = this.readBuffer[i] - // we don't have to use compareids, because id is always defined! - if (r.id[1] === id[1] && r.id[0] === id[0]) { - // found r - // move r to the end of readBuffer - for (; i < this.readBuffer.length - 1; i++) { - this.readBuffer[i] = this.readBuffer[i + 1] - } - this.readBuffer[this.readBuffer.length - 1] = r - return r - } - } - var o - for (i = this.writeBuffer.length - 1; i >= 0; i--) { - r = this.writeBuffer[i] - if (r.id[1] === id[1] && r.id[0] === id[0]) { - o = r - break - } - } - if (i < 0 && noSuperCall === undefined) { - // did not reach break in last loop - // read id and put it to the end of readBuffer - o = super.find(id) - } - if (o != null) { - for (i = 0; i < this.readBuffer.length - 1; i++) { - this.readBuffer[i] = this.readBuffer[i + 1] - } - this.readBuffer[this.readBuffer.length - 1] = o - } - return o - } - put (o) { - var id = o.id - var i, r // helper variables - for (i = this.writeBuffer.length - 1; i >= 0; i--) { - r = this.writeBuffer[i] - if (r.id[1] === id[1] && r.id[0] === id[0]) { - // is already in buffer - // forget r, and move o to the end of writeBuffer - for (; i < this.writeBuffer.length - 1; i++) { - this.writeBuffer[i] = this.writeBuffer[i + 1] - } - this.writeBuffer[this.writeBuffer.length - 1] = o - break - } - } - if (i < 0) { - // did not reach break in last loop - // write writeBuffer[0] - var write = this.writeBuffer[0] - if (write.id[0] !== null) { - super.put(write) - } - // put o to the end of writeBuffer - for (i = 0; i < this.writeBuffer.length - 1; i++) { - this.writeBuffer[i] = this.writeBuffer[i + 1] - } - this.writeBuffer[this.writeBuffer.length - 1] = o - } - // check readBuffer for every occurence of o.id, overwrite if found - // whether found or not, we'll append o to the readbuffer - for (i = 0; i < this.readBuffer.length - 1; i++) { - r = this.readBuffer[i + 1] - if (r.id[1] === id[1] && r.id[0] === id[0]) { - this.readBuffer[i] = o - } else { - this.readBuffer[i] = r - } - } - this.readBuffer[this.readBuffer.length - 1] = o - } - delete (id) { - var i, r - for (i = 0; i < this.readBuffer.length; i++) { - r = this.readBuffer[i] - if (r.id[1] === id[1] && r.id[0] === id[0]) { - this.readBuffer[i] = { - id: [null, null] - } - } - } - this.flush() - super.delete(id) - } - findWithLowerBound (id) { - var o = this.find(id, true) - if (o != null) { - return o - } else { - this.flush() - return super.findWithLowerBound.apply(this, arguments) - } - } - findWithUpperBound (id) { - var o = this.find(id, true) - if (o != null) { - return o - } else { - this.flush() - return super.findWithUpperBound.apply(this, arguments) - } - } - findNext () { - this.flush() - return super.findNext.apply(this, arguments) - } - findPrev () { - this.flush() - return super.findPrev.apply(this, arguments) - } - iterate () { - this.flush() - super.iterate.apply(this, arguments) - } - flush () { - for (var i = 0; i < this.writeBuffer.length; i++) { - var write = this.writeBuffer[i] - if (write.id[0] !== null) { - super.put(write) - this.writeBuffer[i] = { - id: [null, null] - } - } - } - } - } - return SmallLookupBuffer - } - Y.utils.createSmallLookupBuffer = createSmallLookupBuffer - - function generateUserId () { - if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) { - // browser - let arr = new Uint32Array(1) - crypto.getRandomValues(arr) - return arr[0] - } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) { - // node - let buf = crypto.randomBytes(4) - return new Uint32Array(buf.buffer)[0] - } else { - return Math.ceil(Math.random() * 0xFFFFFFFF) - } - } - Y.utils.generateUserId = generateUserId - - Y.utils.parseTypeDefinition = function parseTypeDefinition (type, typeArgs) { - var args = [] - try { - args = JSON.parse('[' + typeArgs + ']') - } catch (e) { - throw new Error('Was not able to parse type definition!') - } - if (type.typeDefinition.parseArguments != null) { - args = type.typeDefinition.parseArguments(args[0])[1] - } - return args - } - - Y.utils.writeObjectToYMap = function writeObjectToYMap (object, type) { - for (var key in object) { - var val = object[key] - if (Array.isArray(val)) { - type.set(key, Y.Array) - Y.utils.writeArrayToYArray(val, type.get(key)) - } else if (typeof val === 'object') { - type.set(key, Y.Map) - Y.utils.writeObjectToYMap(val, type.get(key)) - } else { - type.set(key, val) - } - } - } - - Y.utils.writeArrayToYArray = function writeArrayToYArray (array, type) { - for (var i = array.length - 1; i >= 0; i--) { - var val = array[i] - if (Array.isArray(val)) { - type.insert(0, [Y.Array]) - Y.utils.writeArrayToYArray(val, type.get(0)) - } else if (typeof val === 'object') { - type.insert(0, [Y.Map]) - Y.utils.writeObjectToYMap(val, type.get(0)) - } else { - type.insert(0, [val]) - } - } - } -} diff --git a/src/Y.js b/src/Y.js new file mode 100644 index 00000000..f5b6316e --- /dev/null +++ b/src/Y.js @@ -0,0 +1,101 @@ + +import debug from 'debug' + +import DeleteStore from './Store/DeleteStore' +import OperationStore from './Store/OperationStore' +import StateStore from './Store/StateStore' +import generateUserID from './Function/generateUserID' +import { RootID } from './Util/ID.js' + +import { formatYjsMessage, formatYjsMessageType } from './MessageHandler' + +import Connector from './Connector' +import Persistence from './Persistence' +import YArray from './Type/YArray' +import YMap from './Type/YMap' +import YText from './Type/YText' +import YXml from './Type/YXml' + +export default class Y { + constructor (opts) { + this.userID = generateUserID() + this.ds = new DeleteStore(this) + this.os = new OperationStore(this) + this.ss = new StateStore(this) + this.connector = new Y[opts.connector.name](this, opts.connector) + if (opts.persistence != null) { + this.persistence = new Y[opts.persistence.name](this, opts.persistence) + this.persistence.retrieveContent() + } else { + this.persistence = null + } + this.connected = true + this._missingStructs = new Map() + this._readyToIntegrate = new Map() + } + get room () { + return this.connector.opts.room + } + get (name, TypeConstructor) { + let id = new RootID(name, TypeConstructor) + let type = this.os.get(id) + if (type === null) { + type = new TypeConstructor() + type._id = id + type._integrate(this) + } + return type + } + disconnect () { + if (this.connected) { + this.connected = false + return this.connector.disconnect() + } else { + return Promise.resolve() + } + } + reconnect () { + if (!this.connected) { + this.connected = true + return this.connector.reconnect() + } else { + return Promise.resolve() + } + } + destroy () { + this.share = null + if (this.connector.destroy != null) { + this.connector.destroy() + } else { + this.connector.disconnect() + } + this.os.iterate(null, null, function (struct) { + struct.destroy() + }) + this.os = null + this.ds = null + this.ss = null + } +} + +Y.extend = function extendYjs () { + for (var i = 0; i < arguments.length; i++) { + var f = arguments[i] + if (typeof f === 'function') { + f(Y) + } else { + throw new Error('Expected a function!') + } + } +} + +Y.Connector = Connector +Y.Persisence = Persistence +Y.Array = YArray +Y.Map = YMap +Y.Text = YText +Y.Xml = YXml + +Y.debug = debug +debug.formatters.Y = formatYjsMessage +debug.formatters.y = formatYjsMessageType diff --git a/src/y-memory.js b/src/y-memory.js deleted file mode 100644 index 7744814d..00000000 --- a/src/y-memory.js +++ /dev/null @@ -1,67 +0,0 @@ -import extendRBTree from './RedBlackTree' - -export default function extend (Y) { - extendRBTree(Y) - - class Transaction extends Y.Transaction { - constructor (store) { - super(store) - this.store = store - this.ss = store.ss - this.os = store.os - this.ds = store.ds - } - } - var Store = Y.utils.RBTree - var BufferedStore = Y.utils.createSmallLookupBuffer(Store) - - class Database extends Y.AbstractDatabase { - constructor (y, opts) { - super(y, opts) - this.os = new BufferedStore() - this.ds = new Store() - this.ss = new BufferedStore() - } - logTable () { - var self = this - self.requestTransaction(function () { - console.log('User: ', this.store.y.connector.userId, "==============================") // eslint-disable-line - console.log("State Set (SS):", this.getStateSet()) // eslint-disable-line - console.log("Operation Store (OS):") // eslint-disable-line - this.os.logTable() // eslint-disable-line - console.log("Deletion Store (DS):") //eslint-disable-line - this.ds.logTable() // eslint-disable-line - if (this.store.gc1.length > 0 || this.store.gc2.length > 0) { - console.warn('GC1|2 not empty!', this.store.gc1, this.store.gc2) - } - if (JSON.stringify(this.store.listenersById) !== '{}') { - console.warn('listenersById not empty!') - } - if (JSON.stringify(this.store.listenersByIdExecuteNow) !== '[]') { - console.warn('listenersByIdExecuteNow not empty!') - } - if (this.store.transactionInProgress) { - console.warn('Transaction still in progress!') - } - }, true) - } - transact (makeGen) { - const t = new Transaction(this) - try { - while (makeGen != null) { - makeGen.call(t) - makeGen = this.getNextRequest() - } - } catch (e) { - this.y.emit('error', e) - } - } - destroy () { - super.destroy() - delete this.os - delete this.ss - delete this.ds - } - } - Y.memory = Database -} diff --git a/src/y.js b/src/y.js deleted file mode 100644 index 359afbaf..00000000 --- a/src/y.js +++ /dev/null @@ -1,258 +0,0 @@ -import extendConnector from './Connector.js' -import extendPersistence from './Persistence.js' -import extendDatabase from './Database.js' -import extendTransaction from './Transaction.js' -import extendStruct from './Struct.js' -import extendUtils from './Utils.js' -import extendMemory from './y-memory.js' -import debug from 'debug' -import { formatYjsMessage, formatYjsMessageType } from './MessageHandler.js' - -extendConnector(Y) -extendPersistence(Y) -extendDatabase(Y) -extendTransaction(Y) -extendStruct(Y) -extendUtils(Y) -extendMemory(Y) - -Y.debug = debug -debug.formatters.Y = formatYjsMessage -debug.formatters.y = formatYjsMessageType - -var requiringModules = {} - -Y.requiringModules = requiringModules - -Y.extend = function (name, value) { - if (arguments.length === 2 && typeof name === 'string') { - if (value instanceof Y.utils.CustomTypeDefinition) { - Y[name] = value.parseArguments - } else { - Y[name] = value - } - if (requiringModules[name] != null) { - requiringModules[name].resolve() - delete requiringModules[name] - } - } else { - for (var i = 0; i < arguments.length; i++) { - var f = arguments[i] - if (typeof f === 'function') { - f(Y) - } else { - throw new Error('Expected function!') - } - } - } -} - -Y.requestModules = requestModules -function requestModules (modules) { - var sourceDir - if (Y.sourceDir === null) { - sourceDir = null - } else { - sourceDir = Y.sourceDir || '/bower_components' - } - // determine if this module was compiled for es5 or es6 (y.js vs. y.es6) - // if Insert.execute is a Function, then it isnt a generator.. - // then load the es5(.js) files.. - var extention = typeof regeneratorRuntime !== 'undefined' ? '.js' : '.es6' - var promises = [] - for (var i = 0; i < modules.length; i++) { - var module = modules[i].split('(')[0] - var modulename = 'y-' + module.toLowerCase() - if (Y[module] == null) { - if (requiringModules[module] == null) { - // module does not exist - if (typeof window !== 'undefined' && window.Y !== 'undefined') { - if (sourceDir != null) { - var imported = document.createElement('script') - imported.src = sourceDir + '/' + modulename + '/' + modulename + extention - document.head.appendChild(imported) - } - let requireModule = {} - requiringModules[module] = requireModule - requireModule.promise = new Promise(function (resolve) { - requireModule.resolve = resolve - }) - promises.push(requireModule.promise) - } else { - console.info('YJS: Please do not depend on automatic requiring of modules anymore! Extend modules as follows `require(\'y-modulename\')(Y)`') - require(modulename)(Y) - } - } else { - promises.push(requiringModules[modules[i]].promise) - } - } - } - return Promise.all(promises) -} - -/* :: -type MemoryOptions = { - name: 'memory' -} -type IndexedDBOptions = { - name: 'indexeddb', - namespace: string -} -type DbOptions = MemoryOptions | IndexedDBOptions - -type WebRTCOptions = { - name: 'webrtc', - room: string -} -type WebsocketsClientOptions = { - name: 'websockets-client', - room: string -} -type ConnectionOptions = WebRTCOptions | WebsocketsClientOptions - -type YOptions = { - connector: ConnectionOptions, - db: DbOptions, - types: Array, - sourceDir: string, - share: {[key: string]: TypeName} -} -*/ - -export default function Y (opts/* :YOptions */) /* :Promise */ { - if (opts.hasOwnProperty('sourceDir')) { - Y.sourceDir = opts.sourceDir - } - opts.types = opts.types != null ? opts.types : [] - var modules = [opts.db.name, opts.connector.name].concat(opts.types) - for (var name in opts.share) { - modules.push(opts.share[name]) - } - return new Promise(function (resolve, reject) { - if (opts == null) reject(new Error('An options object is expected!')) - else if (opts.connector == null) reject(new Error('You must specify a connector! (missing connector property)')) - else if (opts.connector.name == null) reject(new Error('You must specify connector name! (missing connector.name property)')) - else if (opts.db == null) reject(new Error('You must specify a database! (missing db property)')) - else if (opts.connector.name == null) reject(new Error('You must specify db name! (missing db.name property)')) - else { - opts = Y.utils.copyObject(opts) - opts.connector = Y.utils.copyObject(opts.connector) - opts.db = Y.utils.copyObject(opts.db) - opts.share = Y.utils.copyObject(opts.share) - Y.requestModules(modules).then(function () { - var yconfig = new YConfig(opts) - let resolved = false - if (opts.timeout != null && opts.timeout >= 0) { - setTimeout(function () { - if (!resolved) { - reject(new Error('Yjs init timeout')) - yconfig.destroy() - } - }, opts.timeout) - } - if (yconfig.persistence != null) { - yconfig.persistence.retrieveContent() - } - yconfig.db.whenUserIdSet(function () { - yconfig.init(function () { - resolved = true - resolve(yconfig) - }, reject) - }) - }).catch(reject) - } - }) -} - -class YConfig extends Y.utils.NamedEventHandler { - /* :: - db: Y.AbstractDatabase; - connector: Y.AbstractConnector; - share: {[key: string]: any}; - options: Object; - */ - constructor (opts, callback) { - super() - this.options = opts - this.db = new Y[opts.db.name](this, opts.db) - this.connector = new Y[opts.connector.name](this, opts.connector) - if (opts.persistence != null) { - this.persistence = new Y[opts.persistence.name](this, opts.persistence) - } else { - this.persistence = null - } - this.connected = true - } - init (callback) { - var opts = this.options - var share = {} - this.share = share - this.db.requestTransaction(function requestTransaction () { - // create shared object - for (var propertyname in opts.share) { - var typeConstructor = opts.share[propertyname].split('(') - let typeArgs = '' - if (typeConstructor.length === 2) { - typeArgs = typeConstructor[1].split(')')[0] || '' - } - var typeName = typeConstructor.splice(0, 1) - var type = Y[typeName] - var typedef = type.typeDefinition - var id = [0xFFFFFF, typedef.struct + '_' + typeName + '_' + propertyname + '_' + typeArgs] - let args = Y.utils.parseTypeDefinition(type, typeArgs) - share[propertyname] = this.store.initType.call(this, id, args) - } - }) - this.db.whenTransactionsFinished() - .then(callback) - } - isConnected () { - return this.connector.isSynced - } - disconnect () { - if (this.connected) { - this.connected = false - return this.connector.disconnect() - } else { - return Promise.resolve() - } - } - reconnect () { - if (!this.connected) { - this.connected = true - return this.connector.reconnect() - } else { - return Promise.resolve() - } - } - destroy () { - var self = this - return this.close().then(function () { - if (self.db.deleteDB != null) { - return self.db.deleteDB() - } else { - return Promise.resolve() - } - }).then(() => { - // remove existing event listener - super.destroy() - }) - } - close () { - var self = this - this.share = null - if (this.connector.destroy != null) { - this.connector.destroy() - } else { - this.connector.disconnect() - } - return this.db.whenTransactionsFinished().then(function () { - self.db.destroyTypes() - // make sure to wait for all transactions before destroying the db - self.db.requestTransaction(function () { - self.db.destroy() - }) - return self.db.whenTransactionsFinished() - }) - } -}