refactor the whole damn thing

examples/chat/index.html

@@ -13,10 +13,6 @@
       <input type="submit" value="Send">
     </form>
     <script src="../../y.js"></script>
-    <script src="../../../y-array/y-array.js"></script>
-    <script src="../../../y-map/dist/y-map.js"></script>
-    <script src="../../../y-text/dist/y-text.js"></script>
-    <script src="../../../y-memory/y-memory.js"></script>
     <script src="../../../y-websockets-client/dist/y-websockets-client.js"></script>
     <script src="./index.js"></script>
   </body>

examples/chat/index.js

@@ -1,73 +1,71 @@
-/* global Y, chat */
+/* global Y */
 
 // initialize a shared object. This function call returns a promise!
-Y({
+var y = new Y({
-  db: {
-    name: 'memory'
-  },
   connector: {
     name: 'websockets-client',
     room: 'chat-example'
-  },
-  sourceDir: '/bower_components',
-  share: {
-    chat: 'Array'
   }
-}).then(function (y) {
+})
-  window.yChat = y
+
-  // This functions inserts a message at the specified position in the DOM
+window.yChat = y
-  function appendMessage (message, position) {
+
+let chatprotocol = y.get('chatprotocol', Y.Array)
+
+let chatcontainer = document.querySelector('#chat')
+
+// This functions inserts a message at the specified position in the DOM
+function appendMessage (message, position) {
   var p = document.createElement('p')
   var uname = document.createElement('span')
   uname.appendChild(document.createTextNode(message.username + ': '))
   p.appendChild(uname)
   p.appendChild(document.createTextNode(message.message))
-    document.querySelector('#chat').insertBefore(p, chat.children[position] || null)
+  chatcontainer.insertBefore(p, chatcontainer.children[position] || null)
+}
+// This function makes sure that only 7 messages exist in the chat history.
+// The rest is deleted
+function cleanupChat () {
+  if (chatprotocol.length > 7) {
+    chatprotocol.delete(0, chatprotocol.length - 7)
   }
-  // This function makes sure that only 7 messages exist in the chat history.
+}
-  // The rest is deleted
+// Insert the initial content
-  function cleanupChat () {
+chatprotocol.toArray().forEach(appendMessage)
-    if (y.share.chat.length > 7) {
+cleanupChat()
-      y.share.chat.delete(0, y.chat.length - 7)
-    }
-  }
-  // Insert the initial content
-  y.share.chat.toArray().forEach(appendMessage)
-  cleanupChat()
 
-  // whenever content changes, make sure to reflect the changes in the DOM
+// whenever content changes, make sure to reflect the changes in the DOM
-  y.share.chat.observe(function (event) {
+chatprotocol.observe(function (event) {
   if (event.type === 'insert') {
     for (let i = 0; i < event.length; i++) {
       appendMessage(event.values[i], event.index + i)
     }
   } else if (event.type === 'delete') {
     for (let i = 0; i < event.length; i++) {
-        chat.children[event.index].remove()
+      chatcontainer.children[event.index].remove()
     }
   }
   // concurrent insertions may result in a history > 7, so cleanup here
   cleanupChat()
-  })
+})
-  document.querySelector('#chatform').onsubmit = function (event) {
+document.querySelector('#chatform').onsubmit = function (event) {
   // the form is submitted
   var message = {
     username: this.querySelector('[name=username]').value,
     message: this.querySelector('[name=message]').value
   }
   if (message.username.length > 0 && message.message.length > 0) {
-      if (y.share.chat.length > 6) {
+    if (chatprotocol.length > 6) {
       // If we are goint to insert the 8th element, make sure to delete first.
-        y.share.chat.delete(0)
+      chatprotocol.delete(0)
     }
     // Here we insert a message in the shared chat type.
     // This will call the observe function (see line 40)
     // and reflect the change in the DOM
-      y.share.chat.push([message])
+    chatprotocol.push([message])
     this.querySelector('[name=message]').value = ''
   }
   // Do not send this form!
   event.preventDefault()
   return false
-  }
+}
-})

src/Binary/Decoder.js

@@ -1,85 +1,6 @@
 import utf8 from 'utf-8'
 
-const bits7 = 0b1111111
+export default class BinaryDecoder {
-const bits8 = 0b11111111
-
-export class BinaryEncoder {
-  constructor () {
-    this.data = []
-  }
-
-  get length () {
-    return this.data.length
-  }
-
-  get pos () {
-    return this.data.length
-  }
-
-  createBuffer () {
-    return Uint8Array.from(this.data).buffer
-  }
-
-  writeUint8 (num) {
-    this.data.push(num & bits8)
-  }
-
-  setUint8 (pos, num) {
-    this.data[pos] = num & bits8
-  }
-
-  writeUint16 (num) {
-    this.data.push(num & bits8, (num >>> 8) & bits8)
-  }
-
-  setUint16 (pos, num) {
-    this.data[pos] = num & bits8
-    this.data[pos + 1] = (num >>> 8) & bits8
-  }
-
-  writeUint32 (num) {
-    for (let i = 0; i < 4; i++) {
-      this.data.push(num & bits8)
-      num >>>= 8
-    }
-  }
-
-  setUint32 (pos, num) {
-    for (let i = 0; i < 4; i++) {
-      this.data[pos + i] = num & bits8
-      num >>>= 8
-    }
-  }
-
-  writeVarUint (num) {
-    while (num >= 0b10000000) {
-      this.data.push(0b10000000 | (bits7 & num))
-      num >>>= 7
-    }
-    this.data.push(bits7 & num)
-  }
-
-  writeVarString (str) {
-    let bytes = utf8.setBytesFromString(str)
-    let len = bytes.length
-    this.writeVarUint(len)
-    for (let i = 0; i < len; i++) {
-      this.data.push(bytes[i])
-    }
-  }
-
-  writeOpID (id) {
-    let user = id[0]
-    this.writeVarUint(user)
-    if (user !== 0xFFFFFF) {
-      this.writeVarUint(id[1])
-    } else {
-      this.writeVarString(id[1])
-    }
-  }
-}
-
-export class BinaryDecoder {
   constructor (buffer) {
     if (buffer instanceof ArrayBuffer) {
       this.uint8arr = new Uint8Array(buffer)
@@ -91,6 +12,16 @@ export class BinaryDecoder {
     this.pos = 0
   }
 
+  clone (newPos = this.pos) {
+    let decoder = new BinaryDecoder(this.uint8arr)
+    decoder.pos = newPos
+    return decoder
+  }
+
+  get length () {
+    return this.uint8arr.length
+  }
+
   skip8 () {
     this.pos++
   }
@@ -118,7 +49,7 @@ export class BinaryDecoder {
     let len = 0
     while (true) {
       let r = this.uint8arr[this.pos++]
-      num = num | ((r & bits7) << len)
+      num = num | ((r & 0b1111111) << len)
       len += 7
       if (r < 1 << 7) {
         return num >>> 0 // return unsigned number!

src/Binary/Encoder.js

@@ -0,0 +1,80 @@
+import utf8 from 'utf-8'
+
+const bits7 = 0b1111111
+const bits8 = 0b11111111
+
+export default class BinaryEncoder {
+  constructor () {
+    this.data = []
+  }
+
+  get length () {
+    return this.data.length
+  }
+
+  get pos () {
+    return this.data.length
+  }
+
+  createBuffer () {
+    return Uint8Array.from(this.data).buffer
+  }
+
+  writeUint8 (num) {
+    this.data.push(num & bits8)
+  }
+
+  setUint8 (pos, num) {
+    this.data[pos] = num & bits8
+  }
+
+  writeUint16 (num) {
+    this.data.push(num & bits8, (num >>> 8) & bits8)
+  }
+
+  setUint16 (pos, num) {
+    this.data[pos] = num & bits8
+    this.data[pos + 1] = (num >>> 8) & bits8
+  }
+
+  writeUint32 (num) {
+    for (let i = 0; i < 4; i++) {
+      this.data.push(num & bits8)
+      num >>>= 8
+    }
+  }
+
+  setUint32 (pos, num) {
+    for (let i = 0; i < 4; i++) {
+      this.data[pos + i] = num & bits8
+      num >>>= 8
+    }
+  }
+
+  writeVarUint (num) {
+    while (num >= 0b10000000) {
+      this.data.push(0b10000000 | (bits7 & num))
+      num >>>= 7
+    }
+    this.data.push(bits7 & num)
+  }
+
+  writeVarString (str) {
+    let bytes = utf8.setBytesFromString(str)
+    let len = bytes.length
+    this.writeVarUint(len)
+    for (let i = 0; i < len; i++) {
+      this.data.push(bytes[i])
+    }
+  }
+
+  writeOpID (id) {
+    let user = id[0]
+    this.writeVarUint(user)
+    if (user !== 0xFFFFFF) {
+      this.writeVarUint(id[1])
+    } else {
+      this.writeVarString(id[1])
+    }
+  }
+}

src/Connector.js

@@ -1,23 +1,15 @@
 import { BinaryEncoder, BinaryDecoder } from './Encoding.js'
-import { sendSyncStep1, computeMessageSyncStep1, computeMessageSyncStep2, computeMessageUpdate } from './MessageHandler.js'
 
-export default function extendConnector (Y/* :any */) {
+import { sendSyncStep1, readSyncStep1 } from './MessageHandler/syncStep1'
-  class AbstractConnector {
+import { readSyncStep2 } from './MessageHandler/syncStep2'
-    /*
+import { readUpdate } from './MessageHandler/update.js'
-      opts contains the following information:
+
-       role : String Role of this client ("master" or "slave")
+import debug from 'debug'
-    */
+
+export default class AbstractConnector {
   constructor (y, opts) {
     this.y = y
-      if (opts == null) {
-        opts = {}
-      }
     this.opts = opts
-      // Prefer to receive untransformed operations. This does only work if
-      // this client receives operations from only one other client.
-      // In particular, this does not work with y-webrtc.
-      // It will work with y-websockets-client
-      this.preferUntransformed = opts.preferUntransformed || false
     if (opts.role == null || opts.role === 'master') {
       this.role = 'master'
     } else if (opts.role === 'slave') {
@@ -25,9 +17,9 @@ export default function extendConnector (Y/* :any */) {
     } else {
       throw new Error("Role must be either 'master' or 'slave'!")
     }
-      this.log = Y.debug('y:connector')
+    this.log = debug('y:connector')
-      this.logMessage = Y.debug('y:connector-message')
+    this.logMessage = debug('y:connector-message')
-      this.y.db.forwardAppliedOperations = opts.forwardAppliedOperations || false
+    this._forwardAppliedStructs = opts.forwardAppliedOperations || false // TODO: rename
     this.role = opts.role
     this.connections = new Map()
     this.isSynced = false
@@ -35,13 +27,10 @@ export default function extendConnector (Y/* :any */) {
     this.whenSyncedListeners = []
     this.currentSyncTarget = null
     this.debug = opts.debug === true
-      this.broadcastOpBuffer = []
+    this.broadcastBuffer = new BinaryEncoder()
     this.protocolVersion = 11
     this.authInfo = opts.auth || null
     this.checkAuth = opts.checkAuth || function () { return Promise.resolve('write') } // default is everyone has write access
-      if (opts.generateUserId !== false) {
-        this.setUserId(Y.utils.generateUserId())
-      }
     if (opts.maxBufferLength == null) {
       this.maxBufferLength = -1
     } else {
@@ -60,32 +49,7 @@ export default function extendConnector (Y/* :any */) {
     this.isSynced = false
     this.currentSyncTarget = null
     this.whenSyncedListeners = []
-      this.y.db.stopGarbageCollector()
+    return Promise.resolve()
-      return this.y.db.whenTransactionsFinished()
-    }
-
-    repair () {
-      this.log('Repairing the state of Yjs. This can happen if messages get lost, and Yjs detects that something is wrong. If this happens often, please report an issue here: https://github.com/y-js/yjs/issues')
-      this.isSynced = false
-      this.connections.forEach((user, userId) => {
-        user.isSynced = false
-        this._syncWithUser(userId)
-      })
-    }
-
-    setUserId (userId) {
-      if (this.userId == null) {
-        if (!Number.isInteger(userId)) {
-          let err = new Error('UserId must be an integer!')
-          this.y.emit('error', err)
-          throw err
-        }
-        this.log('Set userId to "%s"', userId)
-        this.userId = userId
-        return this.y.db.setUserId(userId)
-      } else {
-        return null
-      }
   }
 
   onUserEvent (f) {
@@ -158,7 +122,7 @@ export default function extendConnector (Y/* :any */) {
   }
 
   _fireIsSyncedListeners () {
-      this.y.db.whenTransactionsFinished().then(() => {
+    new Promise().then(() => {
       if (!this.isSynced) {
         this.isSynced = true
         // It is safer to remove this!
@@ -174,7 +138,7 @@ export default function extendConnector (Y/* :any */) {
 
   send (uid, buffer) {
     if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
-        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages')
+      throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - don\'t use this method to send custom messages')
     }
     this.log('%s: Send \'%y\' to %s', this.userId, buffer, uid)
     this.logMessage('Message: %Y', buffer)
@@ -182,7 +146,7 @@ export default function extendConnector (Y/* :any */) {
 
   broadcast (buffer) {
     if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
-        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages')
+      throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - don\'t use this method to send custom messages')
     }
     this.log('%s: Broadcast \'%y\'', this.userId, buffer)
     this.logMessage('Message: %Y', buffer)
@@ -191,37 +155,26 @@ export default function extendConnector (Y/* :any */) {
   /*
     Buffer operations, and broadcast them when ready.
   */
-    broadcastOps (ops) {
+  broadcastStruct (struct) {
-      ops = ops.map(function (op) {
+    let firstContent = this.broadcastBuffer.length === 0
-        return Y.Struct[op.struct].encode(op)
+    struct._toBinary(this.broadcastBuffer)
+    if (this.maxBufferLength > 0 && this.broadcastBuffer.length > this.maxBufferLength) {
+      // it is necessary to send the buffer now
+      // cache the buffer and check if server is responsive
+      let buffer = this.broadcastBuffer
+      this.broadcastBuffer = new BinaryEncoder()
+      this.whenRemoteResponsive().then(() => {
+        this.broadcast(buffer)
       })
-      var self = this
+    } else if (firstContent) {
-      function broadcastOperations () {
+      // send the buffer when all transactions are finished
-        if (self.broadcastOpBuffer.length > 0) {
+      // (or buffer exceeds maxBufferLength)
-          let encoder = new BinaryEncoder()
+      setTimeout(() => {
-          encoder.writeVarString(self.opts.room)
+        if (this.broadcastBuffer.length > 0) {
-          encoder.writeVarString('update')
+          this.broadcast(this.broadcastBuffer)
-          let ops = self.broadcastOpBuffer
+          this.broadcastBuffer = new BinaryEncoder()
-          let length = ops.length
-          let encoderPosLen = encoder.pos
-          encoder.writeUint32(0)
-          for (var i = 0; i < length && (self.maxBufferLength < 0 || encoder.length < self.maxBufferLength); i++) {
-            let op = ops[i]
-            Y.Struct[op.struct].binaryEncode(encoder, op)
         }
-          encoder.setUint32(encoderPosLen, i)
+      })
-          self.broadcastOpBuffer = ops.slice(i)
-          self.broadcast(encoder.createBuffer())
-          if (i !== length) {
-            self.whenRemoteResponsive().then(broadcastOperations)
-          }
-        }
-      }
-      if (this.broadcastOpBuffer.length === 0) {
-        this.broadcastOpBuffer = ops
-        this.y.db.whenTransactionsFinished().then(broadcastOperations)
-      } else {
-        this.broadcastOpBuffer = this.broadcastOpBuffer.concat(ops)
     }
   }
 
@@ -293,14 +246,13 @@ export default function extendConnector (Y/* :any */) {
   computeMessage (messageType, senderConn, decoder, encoder, sender, skipAuth) {
     if (messageType === 'sync step 1' && (senderConn.auth === 'write' || senderConn.auth === 'read')) {
       // cannot wait for sync step 1 to finish, because we may wait for sync step 2 in sync step 1 (->lock)
-        computeMessageSyncStep1(decoder, encoder, this, senderConn, sender)
+      readSyncStep1()(decoder, encoder, this.y, senderConn, sender)
-        return this.y.db.whenTransactionsFinished()
     } else if (messageType === 'sync step 2' && senderConn.auth === 'write') {
-        return computeMessageSyncStep2(decoder, encoder, this, senderConn, sender)
+      readSyncStep2(decoder, encoder, this.y, senderConn, sender)
     } else if (messageType === 'update' && (skipAuth || senderConn.auth === 'write')) {
-        return computeMessageUpdate(decoder, encoder, this, senderConn, sender)
+      readUpdate(decoder, encoder, this.y, senderConn, sender)
     } else {
-        return Promise.reject(new Error('Unable to receive message'))
+      throw new Error('Unable to receive message')
     }
   }
 
@@ -313,6 +265,4 @@ export default function extendConnector (Y/* :any */) {
       this._fireIsSyncedListeners()
     }
   }
-  }
-  Y.AbstractConnector = AbstractConnector
 }

src/Database.js

@@ -1,609 +0,0 @@
-/* @flow */
-'use strict'
-
-export default function extendDatabase (Y /* :any */) {
-  /*
-    Partial definition of an OperationStore.
-    TODO: name it Database, operation store only holds operations.
-
-    A database definition must alse define the following methods:
-    * logTable() (optional)
-      - show relevant information information in a table
-    * requestTransaction(makeGen)
-      - request a transaction
-    * destroy()
-      - destroy the database
-  */
-  class AbstractDatabase {
-    /* ::
-    y: YConfig;
-    forwardAppliedOperations: boolean;
-    listenersById: Object;
-    listenersByIdExecuteNow: Array<Object>;
-    listenersByIdRequestPending: boolean;
-    initializedTypes: Object;
-    whenUserIdSetListener: ?Function;
-    waitingTransactions: Array<Transaction>;
-    transactionInProgress: boolean;
-    executeOrder: Array<Object>;
-    gc1: Array<Struct>;
-    gc2: Array<Struct>;
-    gcTimeout: number;
-    gcInterval: any;
-    garbageCollect: Function;
-    executeOrder: Array<any>; // for debugging only
-    userId: UserId;
-    opClock: number;
-    transactionsFinished: ?{promise: Promise, resolve: any};
-    transact: (x: ?Generator) => any;
-    */
-    constructor (y, opts) {
-      this.y = y
-      opts.gc = opts.gc === true
-      this.dbOpts = opts
-      var os = this
-      this.userId = null
-      var resolve_
-      this.userIdPromise = new Promise(function (resolve) {
-        resolve_ = resolve
-      })
-      this.userIdPromise.resolve = resolve_
-      // whether to broadcast all applied operations (insert & delete hook)
-      this.forwardAppliedOperations = false
-      // E.g. this.listenersById[id] : Array<Listener>
-      this.listenersById = {}
-      // Execute the next time a transaction is requested
-      this.listenersByIdExecuteNow = []
-      // A transaction is requested
-      this.listenersByIdRequestPending = false
-      /* To make things more clear, the following naming conventions:
-         * ls : we put this.listenersById on ls
-         * l : Array<Listener>
-         * id : Id (can't use as property name)
-         * sid : String (converted from id via JSON.stringify
-                         so we can use it as a property name)
-
-        Always remember to first overwrite
-        a property before you iterate over it!
-      */
-      // TODO: Use ES7 Weak Maps. This way types that are no longer user,
-      // wont be kept in memory.
-      this.initializedTypes = {}
-      this.waitingTransactions = []
-      this.transactionInProgress = false
-      this.transactionIsFlushed = false
-      if (typeof YConcurrencyTestingMode !== 'undefined') {
-        this.executeOrder = []
-      }
-      this.gc1 = [] // first stage
-      this.gc2 = [] // second stage -> after that, remove the op
-
-      function garbageCollect () {
-        return os.whenTransactionsFinished().then(function () {
-          if (os.gcTimeout > 0 && (os.gc1.length > 0 || os.gc2.length > 0)) {
-            if (!os.y.connector.isSynced) {
-              console.warn('gc should be empty when not synced!')
-            }
-            return new Promise((resolve) => {
-              os.requestTransaction(function () {
-                if (os.y.connector != null && os.y.connector.isSynced) {
-                  for (var i = 0; i < os.gc2.length; i++) {
-                    var oid = os.gc2[i]
-                    this.garbageCollectOperation(oid)
-                  }
-                  os.gc2 = os.gc1
-                  os.gc1 = []
-                }
-                // TODO: Use setInterval here instead (when garbageCollect is called several times there will be several timeouts..)
-                if (os.gcTimeout > 0) {
-                  os.gcInterval = setTimeout(garbageCollect, os.gcTimeout)
-                }
-                resolve()
-              })
-            })
-          } else {
-            // TODO: see above
-            if (os.gcTimeout > 0) {
-              os.gcInterval = setTimeout(garbageCollect, os.gcTimeout)
-            }
-            return Promise.resolve()
-          }
-        })
-      }
-      this.garbageCollect = garbageCollect
-      this.startGarbageCollector()
-
-      this.repairCheckInterval = !opts.repairCheckInterval ? 6000 : opts.repairCheckInterval
-      this.opsReceivedTimestamp = new Date()
-      this.startRepairCheck()
-    }
-    startGarbageCollector () {
-      this.gc = this.dbOpts.gc
-      if (this.gc) {
-        this.gcTimeout = !this.dbOpts.gcTimeout ? 30000 : this.dbOpts.gcTimeout
-      } else {
-        this.gcTimeout = -1
-      }
-      if (this.gcTimeout > 0) {
-        this.garbageCollect()
-      }
-    }
-    startRepairCheck () {
-      var os = this
-      if (this.repairCheckInterval > 0) {
-        this.repairCheckIntervalHandler = setInterval(function repairOnMissingOperations () {
-          /*
-            Case 1. No ops have been received in a while (new Date() - os.opsReceivedTimestamp > os.repairCheckInterval)
-              - 1.1 os.listenersById is empty. Then the state was correct the whole time. -> Nothing to do (nor to update)
-              - 1.2 os.listenersById is not empty.
-                      * Then the state was incorrect for at least {os.repairCheckInterval} seconds.
-                      * -> Remove everything in os.listenersById and sync again (connector.repair())
-            Case 2. An op has been received in the last {os.repairCheckInterval } seconds.
-                    It is not yet necessary to check for faulty behavior. Everything can still resolve itself. Wait for more messages.
-                    If nothing was received for a while and os.listenersById is still not emty, we are in case 1.2
-                    -> Do nothing
-
-            Baseline here is: we really only have to catch case 1.2..
-          */
-          if (
-            new Date() - os.opsReceivedTimestamp > os.repairCheckInterval &&
-            Object.keys(os.listenersById).length > 0 // os.listenersById is not empty
-          ) {
-            // haven't received operations for over {os.repairCheckInterval} seconds, resend state vector
-            os.listenersById = {}
-            os.opsReceivedTimestamp = new Date() // update so you don't send repair several times in a row
-            os.y.connector.repair()
-          }
-        }, this.repairCheckInterval)
-      }
-    }
-    stopRepairCheck () {
-      clearInterval(this.repairCheckIntervalHandler)
-    }
-    queueGarbageCollector (id) {
-      if (this.y.connector.isSynced && this.gc) {
-        this.gc1.push(id)
-      }
-    }
-    emptyGarbageCollector () {
-      return new Promise(resolve => {
-        var check = () => {
-          if (this.gc1.length > 0 || this.gc2.length > 0) {
-            this.garbageCollect().then(check)
-          } else {
-            resolve()
-          }
-        }
-        setTimeout(check, 0)
-      })
-    }
-    addToDebug () {
-      if (typeof YConcurrencyTestingMode !== 'undefined') {
-        var command /* :string */ = Array.prototype.map.call(arguments, function (s) {
-          if (typeof s === 'string') {
-            return s
-          } else {
-            return JSON.stringify(s)
-          }
-        }).join('').replace(/"/g, "'").replace(/,/g, ', ').replace(/:/g, ': ')
-        this.executeOrder.push(command)
-      }
-    }
-    getDebugData () {
-      console.log(this.executeOrder.join('\n'))
-    }
-    stopGarbageCollector () {
-      var self = this
-      this.gc = false
-      this.gcTimeout = -1
-      return new Promise(function (resolve) {
-        self.requestTransaction(function () {
-          var ungc /* :Array<Struct> */ = self.gc1.concat(self.gc2)
-          self.gc1 = []
-          self.gc2 = []
-          for (var i = 0; i < ungc.length; i++) {
-            var op = this.getOperation(ungc[i])
-            if (op != null) {
-              delete op.gc
-              this.setOperation(op)
-            }
-          }
-          resolve()
-        })
-      })
-    }
-    /*
-      Try to add to GC.
-
-      TODO: rename this function
-
-      Rulez:
-      * Only gc if this user is online & gc turned on
-      * The most left element in a list must not be gc'd.
-        => There is at least one element in the list
-
-      returns true iff op was added to GC
-    */
-    addToGarbageCollector (op, left) {
-      if (
-        op.gc == null &&
-        op.deleted === true &&
-        this.store.gc &&
-        this.store.y.connector.isSynced
-      ) {
-        var gc = false
-        if (left != null && left.deleted === true) {
-          gc = true
-        } else if (op.content != null && op.content.length > 1) {
-          op = this.getInsertionCleanStart([op.id[0], op.id[1] + 1])
-          gc = true
-        }
-        if (gc) {
-          op.gc = true
-          this.setOperation(op)
-          this.store.queueGarbageCollector(op.id)
-          return true
-        }
-      }
-      return false
-    }
-    removeFromGarbageCollector (op) {
-      function filter (o) {
-        return !Y.utils.compareIds(o, op.id)
-      }
-      this.gc1 = this.gc1.filter(filter)
-      this.gc2 = this.gc2.filter(filter)
-      delete op.gc
-    }
-    destroyTypes () {
-      for (var key in this.initializedTypes) {
-        var type = this.initializedTypes[key]
-        if (type._destroy != null) {
-          type._destroy()
-        } else {
-          console.error('The type you included does not provide destroy functionality, it will remain in memory (updating your packages will help).')
-        }
-      }
-    }
-    destroy () {
-      clearTimeout(this.gcInterval)
-      this.gcInterval = null
-      this.stopRepairCheck()
-    }
-    setUserId (userId) {
-      if (!this.userIdPromise.inProgress) {
-        this.userIdPromise.inProgress = true
-        var self = this
-        self.requestTransaction(function () {
-          self.userId = userId
-          var state = this.getState(userId)
-          self.opClock = state.clock
-          self.userIdPromise.resolve(userId)
-        })
-      }
-      return this.userIdPromise
-    }
-    whenUserIdSet (f) {
-      this.userIdPromise.then(f)
-    }
-    getNextOpId (numberOfIds) {
-      if (numberOfIds == null) {
-        throw new Error('getNextOpId expects the number of created ids to create!')
-      } else if (this.userId == null) {
-        throw new Error('OperationStore not yet initialized!')
-      } else {
-        var id = [this.userId, this.opClock]
-        this.opClock += numberOfIds
-        return id
-      }
-    }
-    /*
-      Apply a list of operations.
-
-      * we save a timestamp, because we received new operations that could resolve ops in this.listenersById (see this.startRepairCheck)
-      * get a transaction
-      * check whether all Struct.*.requiredOps are in the OS
-      * check if it is an expected op (otherwise wait for it)
-      * check if was deleted, apply a delete operation after op was applied
-    */
-    applyOperations (decoder) {
-      this.opsReceivedTimestamp = new Date()
-      let length = decoder.readUint32()
-
-      for (var i = 0; i < length; i++) {
-        let o = Y.Struct.binaryDecodeOperation(decoder)
-        if (o.id == null || o.id[0] !== this.y.connector.userId) {
-          var required = Y.Struct[o.struct].requiredOps(o)
-          if (o.requires != null) {
-            required = required.concat(o.requires)
-          }
-          this.whenOperationsExist(required, o)
-        }
-      }
-    }
-    /*
-      op is executed as soon as every operation requested is available.
-      Note that Transaction can (and should) buffer requests.
-    */
-    whenOperationsExist (ids, op) {
-      if (ids.length > 0) {
-        let listener = {
-          op: op,
-          missing: ids.length
-        }
-
-        for (let i = 0; i < ids.length; i++) {
-          let id = ids[i]
-          let sid = JSON.stringify(id)
-          let l = this.listenersById[sid]
-          if (l == null) {
-            l = []
-            this.listenersById[sid] = l
-          }
-          l.push(listener)
-        }
-      } else {
-        this.listenersByIdExecuteNow.push({
-          op: op
-        })
-      }
-
-      if (this.listenersByIdRequestPending) {
-        return
-      }
-
-      this.listenersByIdRequestPending = true
-      var store = this
-
-      this.requestTransaction(function () {
-        var exeNow = store.listenersByIdExecuteNow
-        store.listenersByIdExecuteNow = []
-
-        var ls = store.listenersById
-        store.listenersById = {}
-
-        store.listenersByIdRequestPending = false
-
-        for (let key = 0; key < exeNow.length; key++) {
-          let o = exeNow[key].op
-          store.tryExecute.call(this, o)
-        }
-
-        for (var sid in ls) {
-          var l = ls[sid]
-          var id = JSON.parse(sid)
-          var op
-          if (typeof id[1] === 'string') {
-            op = this.getOperation(id)
-          } else {
-            op = this.getInsertion(id)
-          }
-          if (op == null) {
-            store.listenersById[sid] = l
-          } else {
-            for (let i = 0; i < l.length; i++) {
-              let listener = l[i]
-              let o = listener.op
-              if (--listener.missing === 0) {
-                store.tryExecute.call(this, o)
-              }
-            }
-          }
-        }
-      })
-    }
-    /*
-      Actually execute an operation, when all expected operations are available.
-    */
-    /* :: // TODO: this belongs somehow to transaction
-    store: Object;
-    getOperation: any;
-    isGarbageCollected: any;
-    addOperation: any;
-    whenOperationsExist: any;
-    */
-    tryExecute (op) {
-      this.store.addToDebug('this.store.tryExecute.call(this, ', JSON.stringify(op), ')')
-      if (op.struct === 'Delete') {
-        Y.Struct.Delete.execute.call(this, op)
-        // this is now called in Transaction.deleteOperation!
-        // this.store.operationAdded(this, op)
-      } else {
-        // check if this op was defined
-        var defined = this.getInsertion(op.id)
-        while (defined != null && defined.content != null) {
-          // check if this op has a longer content in the case it is defined
-          if (defined.id[1] + defined.content.length < op.id[1] + op.content.length) {
-            var overlapSize = defined.content.length - (op.id[1] - defined.id[1])
-            op.content.splice(0, overlapSize)
-            op.id = [op.id[0], op.id[1] + overlapSize]
-            op.left = Y.utils.getLastId(defined)
-            op.origin = op.left
-            defined = this.getOperation(op.id) // getOperation suffices here
-          } else {
-            break
-          }
-        }
-        if (defined == null) {
-          var opid = op.id
-          var isGarbageCollected = this.isGarbageCollected(opid)
-          if (!isGarbageCollected) {
-            // TODO: reduce number of get / put calls for op ..
-            Y.Struct[op.struct].execute.call(this, op)
-            this.addOperation(op)
-            this.store.operationAdded(this, op)
-            // operationAdded can change op..
-            op = this.getOperation(opid)
-            // if insertion, try to combine with left
-            this.tryCombineWithLeft(op)
-          }
-        }
-      }
-    }
-    /*
-     * Called by a transaction when an operation is added.
-     * This function is especially important for y-indexeddb, where several instances may share a single database.
-     * Every time an operation is created by one instance, it is send to all other instances and operationAdded is called
-     *
-     * If it's not a Delete operation:
-     *   * Checks if another operation is executable (listenersById)
-     *   * Update state, if possible
-     *
-     * Always:
-     *   * Call type
-     */
-    operationAdded (transaction, op) {
-      if (op.struct === 'Delete') {
-        var type = this.initializedTypes[JSON.stringify(op.targetParent)]
-        if (type != null) {
-          type._changed(transaction, op)
-        }
-      } else {
-        // increase SS
-        transaction.updateState(op.id[0])
-        var opLen = op.content != null ? op.content.length : 1
-        for (let i = 0; i < opLen; i++) {
-          // notify whenOperation listeners (by id)
-          var sid = JSON.stringify([op.id[0], op.id[1] + i])
-          var l = this.listenersById[sid]
-          delete this.listenersById[sid]
-          if (l != null) {
-            for (var key in l) {
-              var listener = l[key]
-              if (--listener.missing === 0) {
-                this.whenOperationsExist([], listener.op)
-              }
-            }
-          }
-        }
-        var t = this.initializedTypes[JSON.stringify(op.parent)]
-
-        // if parent is deleted, mark as gc'd and return
-        if (op.parent != null) {
-          var parentIsDeleted = transaction.isDeleted(op.parent)
-          if (parentIsDeleted) {
-            transaction.deleteList(op.id)
-            return
-          }
-        }
-
-        // notify parent, if it was instanciated as a custom type
-        if (t != null) {
-          let o = Y.utils.copyOperation(op)
-          t._changed(transaction, o)
-        }
-        if (!op.deleted) {
-          // Delete if DS says this is actually deleted
-          var len = op.content != null ? op.content.length : 1
-          var startId = op.id // You must not use op.id in the following loop, because op will change when deleted
-            // TODO: !! console.log('TODO: change this before commiting')
-          for (let i = 0; i < len; i++) {
-            var id = [startId[0], startId[1] + i]
-            var opIsDeleted = transaction.isDeleted(id)
-            if (opIsDeleted) {
-              var delop = {
-                struct: 'Delete',
-                target: id
-              }
-              this.tryExecute.call(transaction, delop)
-            }
-          }
-        }
-      }
-    }
-
-    whenTransactionsFinished () {
-      if (this.transactionInProgress) {
-        if (this.transactionsFinished == null) {
-          var resolve_
-          var promise = new Promise(function (resolve) {
-            resolve_ = resolve
-          })
-          this.transactionsFinished = {
-            resolve: resolve_,
-            promise: promise
-          }
-        }
-        return this.transactionsFinished.promise
-      } else {
-        return Promise.resolve()
-      }
-    }
-
-    // Check if there is another transaction request.
-    // * the last transaction is always a flush :)
-    getNextRequest () {
-      if (this.waitingTransactions.length === 0) {
-        if (this.transactionIsFlushed) {
-          this.transactionInProgress = false
-          this.transactionIsFlushed = false
-          if (this.transactionsFinished != null) {
-            this.transactionsFinished.resolve()
-            this.transactionsFinished = null
-          }
-          return null
-        } else {
-          this.transactionIsFlushed = true
-          return function () {
-            this.flush()
-          }
-        }
-      } else {
-        this.transactionIsFlushed = false
-        return this.waitingTransactions.shift()
-      }
-    }
-    requestTransaction (makeGen/* :any */, callImmediately) {
-      this.waitingTransactions.push(makeGen)
-      if (!this.transactionInProgress) {
-        this.transactionInProgress = true
-        setTimeout(() => {
-          this.transact(this.getNextRequest())
-        }, 0)
-      }
-    }
-    /*
-      Get a created/initialized type.
-    */
-    getType (id) {
-      return this.initializedTypes[JSON.stringify(id)]
-    }
-    /*
-      Init type. This is called when a remote operation is retrieved, and transformed to a type
-      TODO: delete type from store.initializedTypes[id] when corresponding id was deleted!
-    */
-    initType (id, args) {
-      var sid = JSON.stringify(id)
-      var t = this.store.initializedTypes[sid]
-      if (t == null) {
-        var op/* :MapStruct | ListStruct */ = this.getOperation(id)
-        if (op != null) {
-          t = Y[op.type].typeDefinition.initType.call(this, this.store, op, args)
-          this.store.initializedTypes[sid] = t
-        }
-      }
-      return t
-    }
-    /*
-     Create type. This is called when the local user creates a type (which is a synchronous action)
-    */
-    createType (typedefinition, id) {
-      var structname = typedefinition[0].struct
-      id = id || this.getNextOpId(1)
-      var op = Y.Struct[structname].create(id, typedefinition[1])
-      op.type = typedefinition[0].name
-
-      this.requestTransaction(function () {
-        if (op.id[0] === 0xFFFFFF) {
-          this.setOperation(op)
-        } else {
-          this.applyCreatedOperations([op])
-        }
-      })
-      var t = Y[op.type].typeDefinition.createType(this, op, typedefinition[1])
-      this.initializedTypes[JSON.stringify(op.id)] = t
-      return t
-    }
-  }
-  Y.AbstractDatabase = AbstractDatabase
-}

src/MessageHandler.js

@@ -1,193 +0,0 @@
-
-import Y from './y.js'
-import { BinaryDecoder, BinaryEncoder } from './Encoding.js'
-
-export function formatYjsMessage (buffer) {
-  let decoder = new BinaryDecoder(buffer)
-  decoder.readVarString() // read roomname
-  let type = decoder.readVarString()
-  let strBuilder = []
-  strBuilder.push('\n === ' + type + ' ===\n')
-  if (type === 'update') {
-    logMessageUpdate(decoder, strBuilder)
-  } else if (type === 'sync step 1') {
-    logMessageSyncStep1(decoder, strBuilder)
-  } else if (type === 'sync step 2') {
-    logMessageSyncStep2(decoder, strBuilder)
-  } else {
-    strBuilder.push('-- Unknown message type - probably an encoding issue!!!')
-  }
-  return strBuilder.join('')
-}
-
-export function formatYjsMessageType (buffer) {
-  let decoder = new BinaryDecoder(buffer)
-  decoder.readVarString() // roomname
-  return decoder.readVarString()
-}
-
-export function logMessageUpdate (decoder, strBuilder) {
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    strBuilder.push(JSON.stringify(Y.Struct.binaryDecodeOperation(decoder)) + '\n')
-  }
-}
-
-export function computeMessageUpdate (decoder, encoder, conn) {
-  if (conn.y.db.forwardAppliedOperations || conn.y.persistence != null) {
-    let messagePosition = decoder.pos
-    let len = decoder.readUint32()
-    let delops = []
-    for (let i = 0; i < len; i++) {
-      let op = Y.Struct.binaryDecodeOperation(decoder)
-      if (op.struct === 'Delete') {
-        delops.push(op)
-      }
-    }
-    if (delops.length > 0) {
-      if (conn.y.db.forwardAppliedOperations) {
-        conn.broadcastOps(delops)
-      }
-      if (conn.y.persistence) {
-        conn.y.persistence.saveOperations(delops)
-      }
-    }
-    decoder.pos = messagePosition
-  }
-  conn.y.db.applyOperations(decoder)
-}
-
-export function sendSyncStep1 (conn, syncUser) {
-  conn.y.db.requestTransaction(function () {
-    let encoder = new BinaryEncoder()
-    encoder.writeVarString(conn.opts.room || '')
-    encoder.writeVarString('sync step 1')
-    encoder.writeVarString(conn.authInfo || '')
-    encoder.writeVarUint(conn.protocolVersion)
-    let preferUntransformed = conn.preferUntransformed && this.os.length === 0 // TODO: length may not be defined
-    encoder.writeUint8(preferUntransformed ? 1 : 0)
-    this.writeStateSet(encoder)
-    conn.send(syncUser, encoder.createBuffer())
-  })
-}
-
-export function logMessageSyncStep1 (decoder, strBuilder) {
-  let auth = decoder.readVarString()
-  let protocolVersion = decoder.readVarUint()
-  let preferUntransformed = decoder.readUint8() === 1
-  strBuilder.push(`
-  - auth: "${auth}"
-  - protocolVersion: ${protocolVersion}
-  - preferUntransformed: ${preferUntransformed}
-`)
-  logSS(decoder, strBuilder)
-}
-
-export function computeMessageSyncStep1 (decoder, encoder, conn, senderConn, sender) {
-  let protocolVersion = decoder.readVarUint()
-  let preferUntransformed = decoder.readUint8() === 1
-
-  // check protocol version
-  if (protocolVersion !== conn.protocolVersion) {
-    console.warn(
-      `You tried to sync with a yjs instance that has a different protocol version
-      (You: ${protocolVersion}, Client: ${protocolVersion}).
-      The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)!
-      `)
-    conn.y.destroy()
-  }
-
-  return conn.y.db.whenTransactionsFinished().then(() => {
-    // send sync step 2
-    conn.y.db.requestTransaction(function () {
-      encoder.writeVarString('sync step 2')
-      encoder.writeVarString(conn.authInfo || '')
-
-      if (preferUntransformed) {
-        encoder.writeUint8(1)
-        this.writeOperationsUntransformed(encoder)
-      } else {
-        encoder.writeUint8(0)
-        this.writeOperations(encoder, decoder)
-      }
-
-      this.writeDeleteSet(encoder)
-      conn.send(senderConn.uid, encoder.createBuffer())
-      senderConn.receivedSyncStep2 = true
-    })
-    return conn.y.db.whenTransactionsFinished().then(() => {
-      if (conn.role === 'slave') {
-        sendSyncStep1(conn, sender)
-      }
-    })
-  })
-}
-
-export function logSS (decoder, strBuilder) {
-  strBuilder.push('  == SS: \n')
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    let user = decoder.readVarUint()
-    let clock = decoder.readVarUint()
-    strBuilder.push(`     ${user}: ${clock}\n`)
-  }
-}
-
-export function logOS (decoder, strBuilder) {
-  strBuilder.push('  == OS: \n')
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    let op = Y.Struct.binaryDecodeOperation(decoder)
-    strBuilder.push(JSON.stringify(op) + '\n')
-  }
-}
-
-export function logDS (decoder, strBuilder) {
-  strBuilder.push('  == DS: \n')
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    let user = decoder.readVarUint()
-    strBuilder.push(`    User: ${user}: `)
-    let len2 = decoder.readVarUint()
-    for (let j = 0; j < len2; j++) {
-      let from = decoder.readVarUint()
-      let to = decoder.readVarUint()
-      let gc = decoder.readUint8() === 1
-      strBuilder.push(`[${from}, ${to}, ${gc}]`)
-    }
-  }
-}
-
-export function logMessageSyncStep2 (decoder, strBuilder) {
-  strBuilder.push('     - auth: ' + decoder.readVarString() + '\n')
-  let osTransformed = decoder.readUint8() === 1
-  strBuilder.push('     - osUntransformed: ' + osTransformed + '\n')
-  logOS(decoder, strBuilder)
-  if (osTransformed) {
-    logSS(decoder, strBuilder)
-  }
-  logDS(decoder, strBuilder)
-}
-
-export function computeMessageSyncStep2 (decoder, encoder, conn, senderConn, sender) {
-  var db = conn.y.db
-  let defer = senderConn.syncStep2
-
-  // apply operations first
-  db.requestTransaction(function () {
-    let osUntransformed = decoder.readUint8()
-    if (osUntransformed === 1) {
-      this.applyOperationsUntransformed(decoder)
-    } else {
-      this.store.applyOperations(decoder)
-    }
-  })
-  // then apply ds
-  db.requestTransaction(function () {
-    this.applyDeleteSet(decoder)
-  })
-  return db.whenTransactionsFinished().then(() => {
-    conn._setSyncedWith(sender)
-    defer.resolve()
-  })
-}

src/MessageHandler/deleteSet.js

@@ -0,0 +1,117 @@
+import deleteItemRange from 'deleteItemRange'
+
+export function stringifyDeleteSet (y, decoder, strBuilder) {
+  let dsLength = decoder.readUint32()
+  for (let i = 0; i < dsLength; i++) {
+    let user = decoder.readVarUint()
+    strBuilder.push(' -' + user + ':')
+    let dvLength = decoder.readVarUint()
+    for (let j = 0; j < dvLength; j++) {
+      let from = decoder.readVarUint()
+      let len = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      strBuilder.push(`clock: ${from}, length: ${len}, gc: ${gc}`)
+    }
+  }
+  return strBuilder
+}
+
+export function writeDeleteSet (y, encoder) {
+  let currentUser = null
+  let currentLength = 0
+  let lastLenPos
+
+  let numberOfUsers = 0
+  let laterDSLenPus = encoder.pos
+  encoder.writeUint32(0)
+
+  y.ds.iterate(null, null, function (n) {
+    var user = n._id.user
+    var clock = n._id.clock
+    var len = n.len
+    var gc = n.gc
+    if (currentUser !== user) {
+      numberOfUsers++
+      // a new user was found
+      if (currentUser !== null) { // happens on first iteration
+        encoder.setUint32(lastLenPos, currentLength)
+      }
+      encoder.writeVarUint(user)
+      // pseudo-fill pos
+      lastLenPos = encoder.pos
+      encoder.writeUint32(0)
+    }
+    encoder.writeVarUint(clock)
+    encoder.writeVarUint(len)
+    encoder.writeUint8(gc ? 1 : 0)
+  })
+  if (currentUser !== null) { // happens on first iteration
+    encoder.setUint32(lastLenPos, currentLength)
+  }
+  encoder.writeUint32(laterDSLenPus, numberOfUsers)
+}
+
+export function readDeleteSet (y, decoder) {
+  let dsLength = decoder.readUint32()
+  for (let i = 0; i < dsLength; i++) {
+    let user = decoder.readVarUint()
+    let dv = []
+    let dvLength = decoder.readVarUint()
+    for (let j = 0; j < dvLength; j++) {
+      let from = decoder.readVarUint()
+      let len = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      dv.push([from, len, gc])
+    }
+    var pos = 0
+    var d = dv[pos]
+    y.ds.iterate(this, [user, 0], [user, Number.MAX_VALUE], function (n) {
+      // cases:
+      // 1. d deletes something to the right of n
+      //  => go to next n (break)
+      // 2. d deletes something to the left of n
+      //  => create deletions
+      //  => reset d accordingly
+      //  *)=> if d doesn't delete anything anymore, go to next d (continue)
+      // 3. not 2) and d deletes something that also n deletes
+      //  => reset d so that it doesn't contain n's deletion
+      //  *)=> if d does not delete anything anymore, go to next d (continue)
+      while (d != null) {
+        var diff = 0 // describe the diff of length in 1) and 2)
+        if (n.id[1] + n.len <= d[0]) {
+          // 1)
+          break
+        } else if (d[0] < n.id[1]) {
+          // 2)
+          // delete maximum the len of d
+          // else delete as much as possible
+          diff = Math.min(n.id[1] - d[0], d[1])
+          deleteItemRange(y, user, d[0], diff)
+          // deletions.push([user, d[0], diff, d[2]])
+        } else {
+          // 3)
+          diff = n.id[1] + n.len - d[0] // never null (see 1)
+          if (d[2] && !n.gc) {
+            // d marks as gc'd but n does not
+            // then delete either way
+            deleteItemRange(y, user, d[0], Math.min(diff, d[1]))
+            // deletions.push([user, d[0], Math.min(diff, d[1]), d[2]])
+          }
+        }
+        if (d[1] <= diff) {
+          // d doesn't delete anything anymore
+          d = dv[++pos]
+        } else {
+          d[0] = d[0] + diff // reset pos
+          d[1] = d[1] - diff // reset length
+        }
+      }
+    })
+    // for the rest.. just apply it
+    for (; pos < dv.length; pos++) {
+      d = dv[pos]
+      deleteItemRange(y, user, d[0], d[1])
+      // deletions.push([user, d[0], d[1], d[2]])
+    }
+  }
+}

src/MessageHandler/integrateRemoteStructs.js

@@ -0,0 +1,73 @@
+import { getStruct } from '../Util/StructReferences'
+import BinaryDecoder from '../Util/Binary/Decoder'
+
+class MissingEntry {
+  constructor (decoder, missing, struct) {
+    this.decoder = decoder
+    this.missing = missing.length
+    this.struct = struct
+  }
+}
+
+/**
+ * Integrate remote struct
+ * When a remote struct is integrated, other structs might be ready to ready to
+ * integrate.
+ */
+function _integrateRemoteStructHelper (y, struct) {
+  struct._integrate(y)
+  let msu = y._missingStructs.get(struct._id.user)
+  if (msu != null) {
+    let len = struct._length
+    for (let i = 0; i < len; i++) {
+      if (msu.has(struct._id.clock + i)) {
+        let msuc = msu.get(struct._id.clock + i)
+        msuc.forEach(missingDef => {
+          missingDef.missing--
+          if (missingDef.missing === 0) {
+            let missing = missingDef.struct._fromBinary(y, missingDef.decoder)
+            if (missing.length > 0) {
+              console.error('Missing should be empty!')
+            } else {
+              y._readyToIntegrate.push(missingDef.struct)
+            }
+          }
+        })
+        msu.delete(struct._id.clock)
+      }
+    }
+  }
+}
+
+export default function integrateRemoteStructs (decoder, encoder, y) {
+  while (decoder.length !== decoder.pos) {
+    let decoderPos = decoder.pos
+    let reference = decoder.readVarUint()
+    let Constr = getStruct(reference)
+    let struct = new Constr()
+    let missing = struct._fromBinary(decoder)
+    if (missing.length === 0) {
+      while (struct != null) {
+        _integrateRemoteStructHelper(y, struct)
+        struct = y._readyToIntegrate.shift()
+      }
+    } else {
+      let _decoder = new BinaryDecoder(decoder.uint8arr)
+      _decoder.pos = decoderPos
+      let missingEntry = new MissingEntry(_decoder, missing, struct)
+      let missingStructs = y._missingStructs
+      for (let i = missing.length - 1; i >= 0; i--) {
+        let m = missing[i]
+        if (!missingStructs.has(m.user)) {
+          missingStructs.set(m.user, new Map())
+        }
+        let msu = missingStructs.get(m.user)
+        if (!msu.has(m.clock)) {
+          msu.set(m.clock, [])
+        }
+        let mArray = msu = msu.get(m.clock)
+        mArray.push(missingEntry)
+      }
+    }
+  }
+}

src/MessageHandler/messageToString.js

@@ -0,0 +1,28 @@
+import BinaryDecoder from '../Utily/Binary/Decoder'
+import { stringifyUpdate } from './update'
+import { stringifySyncStep1 } from './syncStep1'
+import { stringifySyncStep2 } from './syncStep2'
+
+export function messageToString (buffer) {
+  let decoder = new BinaryDecoder(buffer)
+  decoder.readVarString() // read roomname
+  let type = decoder.readVarString()
+  let strBuilder = []
+  strBuilder.push('\n === ' + type + ' ===\n')
+  if (type === 'update') {
+    stringifyUpdate(decoder, strBuilder)
+  } else if (type === 'sync step 1') {
+    stringifySyncStep1(decoder, strBuilder)
+  } else if (type === 'sync step 2') {
+    stringifySyncStep2(decoder, strBuilder)
+  } else {
+    strBuilder.push('-- Unknown message type - probably an encoding issue!!!')
+  }
+  return strBuilder.join('\n')
+}
+
+export function messageToRoomname (buffer) {
+  let decoder = new BinaryDecoder(buffer)
+  decoder.readVarString() // roomname
+  return decoder.readVarString() // messageType
+}

src/MessageHandler/stateSet.js

@@ -0,0 +1,24 @@
+
+export function readStateSet (decoder) {
+  let ss = new Map()
+  let ssLength = decoder.readUint32()
+  for (let i = 0; i < ssLength; i++) {
+    let user = decoder.readVarUint()
+    let clock = decoder.readVarUint()
+    ss.set(user, clock)
+  }
+  return ss
+}
+
+export function writeStateSet (encoder) {
+  let lenPosition = encoder.pos
+  let len = 0
+  encoder.writeUint32(0)
+  this.ss.iterate(this, null, null, function (n) {
+    encoder.writeVarUint(n.id[0])
+    encoder.writeVarUint(n.clock)
+    len++
+  })
+  encoder.setUint32(lenPosition, len)
+  return len === 0
+}

src/MessageHandler/syncStep1.js

@@ -0,0 +1,53 @@
+import BinaryEncoder from './Util/Binary/Encoder.js'
+
+export function stringifySyncStep1 (decoder, strBuilder) {
+  let auth = decoder.readVarString()
+  let protocolVersion = decoder.readVarUint()
+  strBuilder.push(`
+  - auth: "${auth}"
+  - protocolVersion: ${protocolVersion}
+`)
+  // write SS
+  strBuilder.push('  == SS: \n')
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let user = decoder.readVarUint()
+    let clock = decoder.readVarUint()
+    strBuilder.push(`     ${user}: ${clock}\n`)
+  }
+}
+
+export function sendSyncStep1 (y, syncUser) {
+  let encoder = new BinaryEncoder()
+  encoder.writeVarString(y.room)
+  encoder.writeVarString('sync step 1')
+  encoder.writeVarString(y.connector.authInfo || '')
+  encoder.writeVarUint(y.connector.protocolVersion)
+  y.ss.writeStateSet(encoder)
+  y.connector.send(syncUser, encoder.createBuffer())
+}
+
+export function readSyncStep1 (decoder, encoder, y, senderConn, sender) {
+  let protocolVersion = decoder.readVarUint()
+  // check protocol version
+  if (protocolVersion !== y.connector.protocolVersion) {
+    console.warn(
+      `You tried to sync with a yjs instance that has a different protocol version
+      (You: ${protocolVersion}, Client: ${protocolVersion}).
+      The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)!
+      `)
+    y.destroy()
+  }
+
+  // send sync step 2
+  encoder.writeVarString('sync step 2')
+  encoder.writeVarString(y.connector.authInfo || '')
+  writeDeleteSet(encoder)
+  // reads ss and writes os
+  writeOperations(encoder, decoder)
+  y.connector.send(senderConn.uid, encoder.createBuffer())
+  senderConn.receivedSyncStep2 = true
+  if (y.connector.role === 'slave') {
+    sendSyncStep1(y, sender)
+  }
+}

src/MessageHandler/syncStep2.js

@@ -0,0 +1,48 @@
+import integrateRemoteStructs from './integrateRemoteStructs'
+import { stringifyUpdate } from './update.js'
+import ID from '../Util/ID'
+
+export function stringifySyncStep2 (decoder, strBuilder) {
+  strBuilder.push('     - auth: ' + decoder.readVarString() + '\n')
+  strBuilder.push('  == OS: \n')
+  stringifyUpdate(decoder, strBuilder)
+  // write DS to string
+  strBuilder.push('  == DS: \n')
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let user = decoder.readVarUint()
+    strBuilder.push(`    User: ${user}: `)
+    let len2 = decoder.readVarUint()
+    for (let j = 0; j < len2; j++) {
+      let from = decoder.readVarUint()
+      let to = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      strBuilder.push(`[${from}, ${to}, ${gc}]`)
+    }
+  }
+}
+
+export function writeSyncStep2 () {
+  // TODO
+}
+
+export default function writeStructs (encoder, decoder, y, ss) {
+  let lenPos = encoder.pos
+  let len = 0
+  encoder.writeUint32(0)
+  for (let [user, clock] of ss) {
+    y.os.iterate(new ID(user, clock), null, function (struct) {
+      struct._toBinary(y, encoder)
+      len++
+    })
+  }
+  encoder.setUint32(lenPos, len)
+}
+
+export function readSyncStep2 (decoder, encoder, y, senderConn, sender) {
+  // apply operations first
+  applyDeleteSet(decoder)
+  integrateRemoteStructs(decoder, encoder, y)
+  // then apply ds
+  y.connector._setSyncedWith(sender)
+}

src/MessageHandler/update.js

@@ -0,0 +1,19 @@
+
+import { getStruct } from '../Util/StructReferences'
+
+export function stringifyUpdate (decoder, strBuilder) {
+  while (decoder.length !== decoder.pos) {
+    let reference = decoder.readVarUint()
+    let Constr = getStruct(reference)
+    let struct = new Constr()
+    let missing = struct._fromBinary(decoder)
+    let logMessage = struct._logString()
+    if (missing.length > 0) {
+      logMessage += missing.map(m => m._logString()).join(', ')
+    }
+    logMessage += '\n'
+    strBuilder.push(logMessage)
+  }
+}
+
+export { integrateRemoteStructs as readUpdate } from './integrateRemoteStructs'

src/Notes.md

@@ -1,12 +0,0 @@
-
-# Notes
-
-### Terminology
-
-* DB: DataBase that holds all the information of the shared object. It is devided into the OS, DS, and SS. This can be a persistent database or an in-memory database. Depending on the type of database, it could make sense to store OS, DS, and SS in different tables, or maybe different databases.
-* OS: OperationStore holds all the operations. An operation is a js object with a fixed number of name fields.
-* DS: DeleteStore holds the information about which operations are deleted and which operations were garbage collected (no longer available in the OS).
-* SS: StateSet holds the current state of the OS. SS.getState(username) refers to the amount of operations that were received by that respective user.
-* Op: Operation defines an action on a shared type. But it is also the format in which we store the model of a type. This is why it is also called a Struct/Structure.
-* Type and Structure: We crearly distinguish between type and structure. Short explanation: A type (e.g. Strings, Numbers) have a number of functions that you can apply on them. (+) is well defined on both of them. They are *modeled* by a structure - the functions really change the structure of a type. Types can be implemented differently but still provide the same functionality. In Yjs, almost all types are realized as a doubly linked list (on which Yjs can provide eventual convergence)
-*

src/RedBlackTree.js

@@ -1,506 +0,0 @@
-
-export default function extendRBTree (Y) {
-  class N {
-    // A created node is always red!
-    constructor (val) {
-      this.val = val
-      this.color = true
-      this._left = null
-      this._right = null
-      this._parent = null
-      if (val.id === null) {
-        throw new Error('You must define id!')
-      }
-    }
-    isRed () { return this.color }
-    isBlack () { return !this.color }
-    redden () { this.color = true; return this }
-    blacken () { this.color = false; return this }
-    get grandparent () {
-      return this.parent.parent
-    }
-    get parent () {
-      return this._parent
-    }
-    get sibling () {
-      return (this === this.parent.left)
-        ? this.parent.right : this.parent.left
-    }
-    get left () {
-      return this._left
-    }
-    get right () {
-      return this._right
-    }
-    set left (n) {
-      if (n !== null) {
-        n._parent = this
-      }
-      this._left = n
-    }
-    set right (n) {
-      if (n !== null) {
-        n._parent = this
-      }
-      this._right = n
-    }
-    rotateLeft (tree) {
-      var parent = this.parent
-      var newParent = this.right
-      var newRight = this.right.left
-      newParent.left = this
-      this.right = newRight
-      if (parent === null) {
-        tree.root = newParent
-        newParent._parent = null
-      } else if (parent.left === this) {
-        parent.left = newParent
-      } else if (parent.right === this) {
-        parent.right = newParent
-      } else {
-        throw new Error('The elements are wrongly connected!')
-      }
-    }
-    next () {
-      if (this.right !== null) {
-        // search the most left node in the right tree
-        var o = this.right
-        while (o.left !== null) {
-          o = o.left
-        }
-        return o
-      } else {
-        var p = this
-        while (p.parent !== null && p !== p.parent.left) {
-          p = p.parent
-        }
-        return p.parent
-      }
-    }
-    prev () {
-      if (this.left !== null) {
-        // search the most right node in the left tree
-        var o = this.left
-        while (o.right !== null) {
-          o = o.right
-        }
-        return o
-      } else {
-        var p = this
-        while (p.parent !== null && p !== p.parent.right) {
-          p = p.parent
-        }
-        return p.parent
-      }
-    }
-    rotateRight (tree) {
-      var parent = this.parent
-      var newParent = this.left
-      var newLeft = this.left.right
-      newParent.right = this
-      this.left = newLeft
-      if (parent === null) {
-        tree.root = newParent
-        newParent._parent = null
-      } else if (parent.left === this) {
-        parent.left = newParent
-      } else if (parent.right === this) {
-        parent.right = newParent
-      } else {
-        throw new Error('The elements are wrongly connected!')
-      }
-    }
-    getUncle () {
-      // we can assume that grandparent exists when this is called!
-      if (this.parent === this.parent.parent.left) {
-        return this.parent.parent.right
-      } else {
-        return this.parent.parent.left
-      }
-    }
-  }
-
-  class RBTree {
-    constructor () {
-      this.root = null
-      this.length = 0
-    }
-    findNext (id) {
-      return this.findWithLowerBound([id[0], id[1] + 1])
-    }
-    findPrev (id) {
-      return this.findWithUpperBound([id[0], id[1] - 1])
-    }
-    findNodeWithLowerBound (from) {
-      if (from === void 0) {
-        throw new Error('You must define from!')
-      }
-      var o = this.root
-      if (o === null) {
-        return null
-      } else {
-        while (true) {
-          if ((from === null || Y.utils.smaller(from, o.val.id)) && o.left !== null) {
-            // o is included in the bound
-            // try to find an element that is closer to the bound
-            o = o.left
-          } else if (from !== null && Y.utils.smaller(o.val.id, from)) {
-            // o is not within the bound, maybe one of the right elements is..
-            if (o.right !== null) {
-              o = o.right
-            } else {
-              // there is no right element. Search for the next bigger element,
-              // this should be within the bounds
-              return o.next()
-            }
-          } else {
-            return o
-          }
-        }
-      }
-    }
-    findNodeWithUpperBound (to) {
-      if (to === void 0) {
-        throw new Error('You must define from!')
-      }
-      var o = this.root
-      if (o === null) {
-        return null
-      } else {
-        while (true) {
-          if ((to === null || Y.utils.smaller(o.val.id, to)) && o.right !== null) {
-            // o is included in the bound
-            // try to find an element that is closer to the bound
-            o = o.right
-          } else if (to !== null && Y.utils.smaller(to, o.val.id)) {
-            // o is not within the bound, maybe one of the left elements is..
-            if (o.left !== null) {
-              o = o.left
-            } else {
-              // there is no left element. Search for the prev smaller element,
-              // this should be within the bounds
-              return o.prev()
-            }
-          } else {
-            return o
-          }
-        }
-      }
-    }
-    findSmallestNode () {
-      var o = this.root
-      while (o != null && o.left != null) {
-        o = o.left
-      }
-      return o
-    }
-    findWithLowerBound (from) {
-      var n = this.findNodeWithLowerBound(from)
-      return n == null ? null : n.val
-    }
-    findWithUpperBound (to) {
-      var n = this.findNodeWithUpperBound(to)
-      return n == null ? null : n.val
-    }
-    iterate (t, from, to, f) {
-      var o
-      if (from === null) {
-        o = this.findSmallestNode()
-      } else {
-        o = this.findNodeWithLowerBound(from)
-      }
-      while (
-        o !== null &&
-        (
-          to === null || // eslint-disable-line no-unmodified-loop-condition
-          Y.utils.smaller(o.val.id, to) ||
-          Y.utils.compareIds(o.val.id, to)
-        )
-      ) {
-        f.call(t, o.val)
-        o = o.next()
-      }
-      return true
-    }
-    logTable (from, to, filter) {
-      if (filter == null) {
-        filter = function () {
-          return true
-        }
-      }
-      if (from == null) { from = null }
-      if (to == null) { to = null }
-      var os = []
-      this.iterate(this, from, to, function (o) {
-        if (filter(o)) {
-          var o_ = {}
-          for (var key in o) {
-            if (typeof o[key] === 'object') {
-              o_[key] = JSON.stringify(o[key])
-            } else {
-              o_[key] = o[key]
-            }
-          }
-          os.push(o_)
-        }
-      })
-      if (console.table != null) {
-        console.table(os)
-      }
-    }
-    find (id) {
-      var n
-      return (n = this.findNode(id)) ? n.val : null
-    }
-    findNode (id) {
-      if (id == null || id.constructor !== Array) {
-        throw new Error('Expect id to be an array!')
-      }
-      var o = this.root
-      if (o === null) {
-        return false
-      } else {
-        while (true) {
-          if (o === null) {
-            return false
-          }
-          if (Y.utils.smaller(id, o.val.id)) {
-            o = o.left
-          } else if (Y.utils.smaller(o.val.id, id)) {
-            o = o.right
-          } else {
-            return o
-          }
-        }
-      }
-    }
-    delete (id) {
-      if (id == null || id.constructor !== Array) {
-        throw new Error('id is expected to be an Array!')
-      }
-      var d = this.findNode(id)
-      if (d == null) {
-        // throw new Error('Element does not exist!')
-        return
-      }
-      this.length--
-      if (d.left !== null && d.right !== null) {
-        // switch d with the greates element in the left subtree.
-        // o should have at most one child.
-        var o = d.left
-        // find
-        while (o.right !== null) {
-          o = o.right
-        }
-        // switch
-        d.val = o.val
-        d = o
-      }
-      // d has at most one child
-      // let n be the node that replaces d
-      var isFakeChild
-      var child = d.left || d.right
-      if (child === null) {
-        isFakeChild = true
-        child = new N({id: 0})
-        child.blacken()
-        d.right = child
-      } else {
-        isFakeChild = false
-      }
-
-      if (d.parent === null) {
-        if (!isFakeChild) {
-          this.root = child
-          child.blacken()
-          child._parent = null
-        } else {
-          this.root = null
-        }
-        return
-      } else if (d.parent.left === d) {
-        d.parent.left = child
-      } else if (d.parent.right === d) {
-        d.parent.right = child
-      } else {
-        throw new Error('Impossible!')
-      }
-      if (d.isBlack()) {
-        if (child.isRed()) {
-          child.blacken()
-        } else {
-          this._fixDelete(child)
-        }
-      }
-      this.root.blacken()
-      if (isFakeChild) {
-        if (child.parent.left === child) {
-          child.parent.left = null
-        } else if (child.parent.right === child) {
-          child.parent.right = null
-        } else {
-          throw new Error('Impossible #3')
-        }
-      }
-    }
-    _fixDelete (n) {
-      function isBlack (node) {
-        return node !== null ? node.isBlack() : true
-      }
-      function isRed (node) {
-        return node !== null ? node.isRed() : false
-      }
-      if (n.parent === null) {
-        // this can only be called after the first iteration of fixDelete.
-        return
-      }
-      // d was already replaced by the child
-      // d is not the root
-      // d and child are black
-      var sibling = n.sibling
-      if (isRed(sibling)) {
-        // make sibling the grandfather
-        n.parent.redden()
-        sibling.blacken()
-        if (n === n.parent.left) {
-          n.parent.rotateLeft(this)
-        } else if (n === n.parent.right) {
-          n.parent.rotateRight(this)
-        } else {
-          throw new Error('Impossible #2')
-        }
-        sibling = n.sibling
-      }
-      // parent, sibling, and children of n are black
-      if (n.parent.isBlack() &&
-        sibling.isBlack() &&
-        isBlack(sibling.left) &&
-        isBlack(sibling.right)
-      ) {
-        sibling.redden()
-        this._fixDelete(n.parent)
-      } else if (n.parent.isRed() &&
-        sibling.isBlack() &&
-        isBlack(sibling.left) &&
-        isBlack(sibling.right)
-      ) {
-        sibling.redden()
-        n.parent.blacken()
-      } else {
-        if (n === n.parent.left &&
-          sibling.isBlack() &&
-          isRed(sibling.left) &&
-          isBlack(sibling.right)
-        ) {
-          sibling.redden()
-          sibling.left.blacken()
-          sibling.rotateRight(this)
-          sibling = n.sibling
-        } else if (n === n.parent.right &&
-          sibling.isBlack() &&
-          isRed(sibling.right) &&
-          isBlack(sibling.left)
-        ) {
-          sibling.redden()
-          sibling.right.blacken()
-          sibling.rotateLeft(this)
-          sibling = n.sibling
-        }
-        sibling.color = n.parent.color
-        n.parent.blacken()
-        if (n === n.parent.left) {
-          sibling.right.blacken()
-          n.parent.rotateLeft(this)
-        } else {
-          sibling.left.blacken()
-          n.parent.rotateRight(this)
-        }
-      }
-    }
-    put (v) {
-      if (v == null || v.id == null || v.id.constructor !== Array) {
-        throw new Error('v is expected to have an id property which is an Array!')
-      }
-      var node = new N(v)
-      if (this.root !== null) {
-        var p = this.root // p abbrev. parent
-        while (true) {
-          if (Y.utils.smaller(node.val.id, p.val.id)) {
-            if (p.left === null) {
-              p.left = node
-              break
-            } else {
-              p = p.left
-            }
-          } else if (Y.utils.smaller(p.val.id, node.val.id)) {
-            if (p.right === null) {
-              p.right = node
-              break
-            } else {
-              p = p.right
-            }
-          } else {
-            p.val = node.val
-            return p
-          }
-        }
-        this._fixInsert(node)
-      } else {
-        this.root = node
-      }
-      this.length++
-      this.root.blacken()
-      return node
-    }
-    _fixInsert (n) {
-      if (n.parent === null) {
-        n.blacken()
-        return
-      } else if (n.parent.isBlack()) {
-        return
-      }
-      var uncle = n.getUncle()
-      if (uncle !== null && uncle.isRed()) {
-        // Note: parent: red, uncle: red
-        n.parent.blacken()
-        uncle.blacken()
-        n.grandparent.redden()
-        this._fixInsert(n.grandparent)
-      } else {
-        // Note: parent: red, uncle: black or null
-        // Now we transform the tree in such a way that
-        // either of these holds:
-        //   1) grandparent.left.isRed
-        //     and grandparent.left.left.isRed
-        //   2) grandparent.right.isRed
-        //     and grandparent.right.right.isRed
-        if (n === n.parent.right && n.parent === n.grandparent.left) {
-          n.parent.rotateLeft(this)
-          // Since we rotated and want to use the previous
-          // cases, we need to set n in such a way that
-          // n.parent.isRed again
-          n = n.left
-        } else if (n === n.parent.left && n.parent === n.grandparent.right) {
-          n.parent.rotateRight(this)
-          // see above
-          n = n.right
-        }
-        // Case 1) or 2) hold from here on.
-        // Now traverse grandparent, make parent a black node
-        // on the highest level which holds two red nodes.
-        n.parent.blacken()
-        n.grandparent.redden()
-        if (n === n.parent.left) {
-          // Case 1
-          n.grandparent.rotateRight(this)
-        } else {
-          // Case 2
-          n.grandparent.rotateLeft(this)
-        }
-      }
-    }
-    flush () {}
-  }
-
-  Y.utils.RBTree = RBTree
-}

src/Store/DeleteStore.js

@@ -0,0 +1,113 @@
+import Tree from '../Util/Tree'
+import ID from '../Util/ID'
+
+class DSNode {
+  constructor (id, len, gc) {
+    this.id = id
+    this.len = len
+    this.gc = gc
+  }
+  clone () {
+    return new DSNode(this.id, this.len, this.gc)
+  }
+}
+
+export default class DeleteStore extends Tree {
+  isDeleted (id) {
+    var n = this.ds.findWithUpperBound(id)
+    return n != null && n.id[0] === id[0] && id[1] < n.id[1] + n.len
+  }
+  /*
+   * Mark an operation as deleted. returns the deleted node
+   */
+  markDeleted (id, length) {
+    if (length == null) {
+      throw new Error('length must be defined')
+    }
+    var n = this.findWithUpperBound(id)
+    if (n != null && n.id.user === id.user) {
+      if (n.id.clock <= id.clock && id.clock <= n.id.clock + n.len) {
+        // id is in n's range
+        var diff = id.clock + length - (n.id.clock + n.len) // overlapping right
+        if (diff > 0) {
+          // id+length overlaps n
+          if (!n.gc) {
+            n.len += diff
+          } else {
+            diff = n.id.clock + n.len - id.clock // overlapping left (id till n.end)
+            if (diff < length) {
+              // a partial deletion
+              let nId = id.clone()
+              nId.clock += diff
+              n = new DSNode(nId, length - diff, false)
+              this.ds.put(n)
+            } else {
+              // already gc'd
+              throw new Error(
+                'DS reached an inconsistent state. Please report this issue!'
+              )
+            }
+          }
+        } else {
+          // no overlapping, already deleted
+          return n
+        }
+      } else {
+        // cannot extend left (there is no left!)
+        n = new DSNode(id, length, false)
+        this.ds.put(n) // TODO: you double-put !!
+      }
+    } else {
+      // cannot extend left
+      n = new DSNode(id, length, false)
+      this.ds.put(n)
+    }
+    // can extend right?
+    var next = this.ds.findNext(n.id)
+    if (
+      next != null &&
+      n.id.user === next.id.user &&
+      n.id.clock + n.len >= next.id.clock
+    ) {
+      diff = n.id.clock + n.len - next.id.clock // from next.start to n.end
+      while (diff >= 0) {
+        // n overlaps with next
+        if (next.gc) {
+          // gc is stronger, so reduce length of n
+          n.len -= diff
+          if (diff >= next.len) {
+            // delete the missing range after next
+            diff = diff - next.len // missing range after next
+            if (diff > 0) {
+              this.put(n) // unneccessary? TODO!
+              this.markDeleted(new ID(next.id.user, next.id.clock + next.len), diff)
+            }
+          }
+          break
+        } else {
+          // we can extend n with next
+          if (diff > next.len) {
+            // n is even longer than next
+            // get next.next, and try to extend it
+            var _next = this.findNext(next.id)
+            this.delete(next.id)
+            if (_next == null || n.id.user !== _next.id.user) {
+              break
+            } else {
+              next = _next
+              diff = n.id.clock + n.len - next.id.clock // from next.start to n.end
+              // continue!
+            }
+          } else {
+            // n just partially overlaps with next. extend n, delete next, and break this loop
+            n.len += next.len - diff
+            this.delete(next.id)
+            break
+          }
+        }
+      }
+    }
+    this.put(n)
+    return n
+  }
+}

src/Store/OperationStore.js

@@ -0,0 +1,88 @@
+import Tree from '../Util/Tree'
+import RootID from '../Util/ID'
+import { getStruct } from '../Util/structReferences'
+
+export default class OperationStore extends Tree {
+  constructor () {
+
+  }
+  get (id) {
+    let struct = this.find(id)
+    if (struct === null && id instanceof RootID) {
+      let Constr = getStruct(id.type)
+      struct = new Constr()
+      struct._id = id
+      this.put(struct)
+    }
+    return struct
+  }
+  getItem (id) {
+    var item = this.findWithUpperBound(id)
+    if (item == null) {
+      return null
+    }
+    var len = item.content != null ? item.content.length : 1 // in case of opContent
+    if (id[0] === item.id[0] && id[1] < item.id[1] + len) {
+      return item
+    } else {
+      return null
+    }
+    
+  }
+  // Return an insertion such that id is the first element of content
+  // This function manipulates an operation, if necessary
+  getInsertionCleanStart (id) {
+    var ins = this.getInsertion(id)
+    if (ins != null) {
+      if (ins.id[1] === id[1]) {
+        return ins
+      } else {
+        var left = Y.utils.copyObject(ins)
+        ins.content = left.content.splice(id[1] - ins.id[1])
+        ins.id = id
+        var leftLid = Y.utils.getLastId(left)
+        ins.origin = leftLid
+        left.originOf = [ins.id]
+        left.right = ins.id
+        ins.left = leftLid
+        // debugger // check
+        this.setOperation(left)
+        this.setOperation(ins)
+        if (left.gc) {
+          this.store.queueGarbageCollector(ins.id)
+        }
+        return ins
+      }
+    } else {
+      return null
+    }
+  }
+  // Return an insertion such that id is the last element of content
+  // This function manipulates an operation, if necessary
+  getInsertionCleanEnd (id) {
+    var ins = this.getInsertion(id)
+    if (ins != null) {
+      if (ins.content == null || (ins.id[1] + ins.content.length - 1 === id[1])) {
+        return ins
+      } else {
+        var right = Y.utils.copyObject(ins)
+        right.content = ins.content.splice(id[1] - ins.id[1] + 1) // cut off remainder
+        right.id = [id[0], id[1] + 1]
+        var insLid = Y.utils.getLastId(ins)
+        right.origin = insLid
+        ins.originOf = [right.id]
+        ins.right = right.id
+        right.left = insLid
+        // debugger // check
+        this.setOperation(right)
+        this.setOperation(ins)
+        if (ins.gc) {
+          this.store.queueGarbageCollector(right.id)
+        }
+        return ins
+      }
+    } else {
+      return null
+    }
+  }
+}

src/Store/StateStore.js

@@ -0,0 +1,30 @@
+import ID from '../Util/ID'
+
+export default class StateStore {
+  constructor (y) {
+    this.y = y
+    this.state = new Map()
+    this.currentClock = 0
+  }
+  getNextID (len) {
+    let id = new ID(this.y.userID, this.currentClock)
+    this.currentClock += len
+    return id
+  }
+  updateRemoteState (struct) {
+    let user = struct._id.user
+    let userState = this.state.get(user)
+    while (struct !== null && struct._id.clock === userState) {
+      userState += struct._length
+      struct = this.y.os.get(new ID(user, userState))
+    }
+    this.state.set(user, userState)
+  }
+  getState (user) {
+    let state = this.state.get(user)
+    if (state == null) {
+      return 0
+    }
+    return state
+  }
+}

src/Struct.js

@@ -1,619 +0,0 @@
-const CDELETE = 0
-const CINSERT = 1
-const CLIST = 2
-const CMAP = 3
-const CXML = 4
-
-/*
- An operation also defines the structure of a type. This is why operation and
- structure are used interchangeably here.
-
- It must be of the type Object. I hope to achieve some performance
- improvements when working on databases that support the json format.
-
- An operation must have the following properties:
-
- * encode
-     - Encode the structure in a readable format (preferably string- todo)
- * decode (todo)
-     - decode structure to json
- * execute
-     - Execute the semantics of an operation.
- * requiredOps
-     - Operations that are required to execute this operation.
-*/
-export default function extendStruct (Y) {
-  let Struct = {}
-  Y.Struct = Struct
-  Struct.binaryDecodeOperation = function (decoder) {
-    let code = decoder.peekUint8()
-    if (code === CDELETE) {
-      return Struct.Delete.binaryDecode(decoder)
-    } else if (code === CINSERT) {
-      return Struct.Insert.binaryDecode(decoder)
-    } else if (code === CLIST) {
-      return Struct.List.binaryDecode(decoder)
-    } else if (code === CMAP) {
-      return Struct.Map.binaryDecode(decoder)
-    } else if (code === CXML) {
-      return Struct.Xml.binaryDecode(decoder)
-    } else {
-      throw new Error('Unable to decode operation!')
-    }
-  }
-
-  /* This is the only operation that is actually not a structure, because
-  it is not stored in the OS. This is why it _does not_ have an id
-
-  op = {
-    target: Id
-  }
-  */
-  Struct.Delete = {
-    encode: function (op) {
-      return {
-        target: op.target,
-        length: op.length || 0,
-        struct: 'Delete'
-      }
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CDELETE)
-      encoder.writeOpID(op.target)
-      encoder.writeVarUint(op.length || 0)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      return {
-        target: decoder.readOpID(),
-        length: decoder.readVarUint(),
-        struct: 'Delete'
-      }
-    },
-    requiredOps: function (op) {
-      return [] // [op.target]
-    },
-    execute: function (op) {
-      return this.deleteOperation(op.target, op.length || 1)
-    }
-  }
-
-  /* {
-      content: [any],
-      opContent: Id,
-      id: Id,
-      left: Id,
-      origin: Id,
-      right: Id,
-      parent: Id,
-      parentSub: string (optional), // child of Map type
-    }
-  */
-  Struct.Insert = {
-    encode: function (op/* :Insertion */) /* :Insertion */ {
-      // TODO: you could not send the "left" property, then you also have to
-      // "op.left = null" in $execute or $decode
-      var e/* :any */ = {
-        id: op.id,
-        left: op.left,
-        right: op.right,
-        origin: op.origin,
-        parent: op.parent,
-        struct: op.struct
-      }
-      if (op.parentSub != null) {
-        e.parentSub = op.parentSub
-      }
-      if (op.hasOwnProperty('opContent')) {
-        e.opContent = op.opContent
-      } else {
-        e.content = op.content.slice()
-      }
-
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CINSERT)
-      // compute info property
-      let contentIsText = op.content != null && op.content.every(c => typeof c === 'string' && c.length === 1)
-      let originIsLeft = Y.utils.compareIds(op.left, op.origin)
-      let info =
-        (op.parentSub != null ? 1 : 0) |
-        (op.opContent != null ? 2 : 0) |
-        (contentIsText ? 4 : 0) |
-        (originIsLeft ? 8 : 0) |
-        (op.left != null ? 16 : 0) |
-        (op.right != null ? 32 : 0) |
-        (op.origin != null ? 64 : 0)
-      encoder.writeUint8(info)
-      encoder.writeOpID(op.id)
-      encoder.writeOpID(op.parent)
-      if (info & 16) {
-        encoder.writeOpID(op.left)
-      }
-      if (info & 32) {
-        encoder.writeOpID(op.right)
-      }
-      if (!originIsLeft && info & 64) {
-        encoder.writeOpID(op.origin)
-      }
-      if (info & 1) {
-        // write parentSub
-        encoder.writeVarString(op.parentSub)
-      }
-      if (info & 2) {
-        // write opContent
-        encoder.writeOpID(op.opContent)
-      } else if (info & 4) {
-        // write text
-        encoder.writeVarString(op.content.join(''))
-      } else {
-        // convert to JSON and write
-        encoder.writeVarString(JSON.stringify(op.content))
-      }
-    },
-    binaryDecode: function (decoder) {
-      let op = {
-        struct: 'Insert'
-      }
-      decoder.skip8()
-      // get info property
-      let info = decoder.readUint8()
-
-      op.id = decoder.readOpID()
-      op.parent = decoder.readOpID()
-      if (info & 16) {
-        op.left = decoder.readOpID()
-      } else {
-        op.left = null
-      }
-      if (info & 32) {
-        op.right = decoder.readOpID()
-      } else {
-        op.right = null
-      }
-      if (info & 8) {
-        // origin is left
-        op.origin = op.left
-      } else if (info & 64) {
-        op.origin = decoder.readOpID()
-      } else {
-        op.origin = null
-      }
-      if (info & 1) {
-        // has parentSub
-        op.parentSub = decoder.readVarString()
-      }
-      if (info & 2) {
-        // has opContent
-        op.opContent = decoder.readOpID()
-      } else if (info & 4) {
-        // has pure text content
-        op.content = decoder.readVarString().split('')
-      } else {
-        // has mixed content
-        let s = decoder.readVarString()
-        op.content = JSON.parse(s)
-      }
-      return op
-    },
-    requiredOps: function (op) {
-      var ids = []
-      if (op.left != null) {
-        ids.push(op.left)
-      }
-      if (op.right != null) {
-        ids.push(op.right)
-      }
-      if (op.origin != null && !Y.utils.compareIds(op.left, op.origin)) {
-        ids.push(op.origin)
-      }
-      // if (op.right == null && op.left == null) {
-      ids.push(op.parent)
-
-      if (op.opContent != null) {
-        ids.push(op.opContent)
-      }
-      return ids
-    },
-    getDistanceToOrigin: function (op) {
-      if (op.left == null) {
-        return 0
-      } else {
-        var d = 0
-        var o = this.getInsertion(op.left)
-        while (!Y.utils.matchesId(o, op.origin)) {
-          d++
-          if (o.left == null) {
-            break
-          } else {
-            o = this.getInsertion(o.left)
-          }
-        }
-        return d
-      }
-    },
-    /*
-    # $this has to find a unique position between origin and the next known character
-    # case 1: $origin equals $o.origin: the $creator parameter decides if left or right
-    #         let $OL= [o1,o2,o3,o4], whereby $this is to be inserted between o1 and o4
-    #         o2,o3 and o4 origin is 1 (the position of o2)
-    #         there is the case that $this.creator < o2.creator, but o3.creator < $this.creator
-    #         then o2 knows o3. Since on another client $OL could be [o1,o3,o4] the problem is complex
-    #         therefore $this would be always to the right of o3
-    # case 2: $origin < $o.origin
-    #         if current $this insert_position > $o origin: $this ins
-    #         else $insert_position will not change
-    #         (maybe we encounter case 1 later, then this will be to the right of $o)
-    # case 3: $origin > $o.origin
-    #         $this insert_position is to the left of $o (forever!)
-    */
-    execute: function (op) {
-      var i // loop counter
-
-      // during this function some ops may get split into two pieces (e.g. with getInsertionCleanEnd)
-      // We try to merge them later, if possible
-      var tryToRemergeLater = []
-
-      if (op.origin != null) { // TODO: !== instead of !=
-        // we save in origin that op originates in it
-        // we need that later when we eventually garbage collect origin (see transaction)
-        var origin = this.getInsertionCleanEnd(op.origin)
-        if (origin.originOf == null) {
-          origin.originOf = []
-        }
-        origin.originOf.push(op.id)
-        this.setOperation(origin)
-        if (origin.right != null) {
-          tryToRemergeLater.push(origin.right)
-        }
-      }
-      var distanceToOrigin = i = Struct.Insert.getDistanceToOrigin.call(this, op) // most cases: 0 (starts from 0)
-
-      // now we begin to insert op in the list of insertions..
-      var o
-      var parent
-      var start
-
-      // find o. o is the first conflicting operation
-      if (op.left != null) {
-        o = this.getInsertionCleanEnd(op.left)
-        if (!Y.utils.compareIds(op.left, op.origin) && o.right != null) {
-          // only if not added previously
-          tryToRemergeLater.push(o.right)
-        }
-        o = (o.right == null) ? null : this.getOperation(o.right)
-      } else { // left == null
-        parent = this.getOperation(op.parent)
-        let startId = op.parentSub ? parent.map[op.parentSub] : parent.start
-        start = startId == null ? null : this.getOperation(startId)
-        o = start
-      }
-
-      // make sure to split op.right if necessary (also add to tryCombineWithLeft)
-      if (op.right != null) {
-        tryToRemergeLater.push(op.right)
-        this.getInsertionCleanStart(op.right)
-      }
-
-      // handle conflicts
-      while (true) {
-        if (o != null && !Y.utils.compareIds(o.id, op.right)) {
-          var oOriginDistance = Struct.Insert.getDistanceToOrigin.call(this, o)
-          if (oOriginDistance === i) {
-            // case 1
-            if (o.id[0] < op.id[0]) {
-              op.left = Y.utils.getLastId(o)
-              distanceToOrigin = i + 1 // just ignore o.content.length, doesn't make a difference
-            }
-          } else if (oOriginDistance < i) {
-            // case 2
-            if (i - distanceToOrigin <= oOriginDistance) {
-              op.left = Y.utils.getLastId(o)
-              distanceToOrigin = i + 1 // just ignore o.content.length, doesn't make a difference
-            }
-          } else {
-            break
-          }
-          i++
-          if (o.right != null) {
-            o = this.getInsertion(o.right)
-          } else {
-            o = null
-          }
-        } else {
-          break
-        }
-      }
-
-      // reconnect..
-      var left = null
-      var right = null
-      if (parent == null) {
-        parent = this.getOperation(op.parent)
-      }
-
-      // reconnect left and set right of op
-      if (op.left != null) {
-        left = this.getInsertion(op.left)
-        // link left
-        op.right = left.right
-        left.right = op.id
-
-        this.setOperation(left)
-      } else {
-        // set op.right from parent, if necessary
-        op.right = op.parentSub ? parent.map[op.parentSub] || null : parent.start
-      }
-      // reconnect right
-      if (op.right != null) {
-        // TODO: wanna connect right too?
-        right = this.getOperation(op.right)
-        right.left = Y.utils.getLastId(op)
-
-        // if right exists, and it is supposed to be gc'd. Remove it from the gc
-        if (right.gc != null) {
-          if (right.content != null && right.content.length > 1) {
-            right = this.getInsertionCleanEnd(right.id)
-          }
-          this.store.removeFromGarbageCollector(right)
-        }
-        this.setOperation(right)
-      }
-
-      // update parents .map/start/end properties
-      if (op.parentSub != null) {
-        if (left == null) {
-          parent.map[op.parentSub] = op.id
-          this.setOperation(parent)
-        }
-        // is a child of a map struct.
-        // Then also make sure that only the most left element is not deleted
-        // We do not call the type in this case (this is what the third parameter is for)
-        if (op.right != null) {
-          this.deleteOperation(op.right, 1, true)
-        }
-        if (op.left != null) {
-          this.deleteOperation(op.id, 1, true)
-        }
-      } else {
-        if (right == null || left == null) {
-          if (right == null) {
-            parent.end = Y.utils.getLastId(op)
-          }
-          if (left == null) {
-            parent.start = op.id
-          }
-          this.setOperation(parent)
-        }
-      }
-
-      // try to merge original op.left and op.origin
-      for (i = 0; i < tryToRemergeLater.length; i++) {
-        var m = this.getOperation(tryToRemergeLater[i])
-        this.tryCombineWithLeft(m)
-      }
-    }
-  }
-
-  /*
-  {
-    start: null,
-    end: null,
-    struct: "List",
-    type: "",
-    id: this.os.getNextOpId(1)
-  }
-  */
-  Struct.List = {
-    create: function (id) {
-      return {
-        start: null,
-        end: null,
-        struct: 'List',
-        id: id
-      }
-    },
-    encode: function (op) {
-      var e = {
-        struct: 'List',
-        id: op.id,
-        type: op.type
-      }
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CLIST)
-      encoder.writeOpID(op.id)
-      encoder.writeVarString(op.type)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      let op = {
-        id: decoder.readOpID(),
-        type: decoder.readVarString(),
-        struct: 'List',
-        start: null,
-        end: null
-      }
-      return op
-    },
-    requiredOps: function () {
-      /*
-      var ids = []
-      if (op.start != null) {
-        ids.push(op.start)
-      }
-      if (op.end != null){
-        ids.push(op.end)
-      }
-      return ids
-      */
-      return []
-    },
-    execute: function (op) {
-      op.start = null
-      op.end = null
-    },
-    ref: function (op, pos) {
-      if (op.start == null) {
-        return null
-      }
-      var res = null
-      var o = this.getOperation(op.start)
-
-      while (true) {
-        if (!o.deleted) {
-          res = o
-          pos--
-        }
-        if (pos >= 0 && o.right != null) {
-          o = this.getOperation(o.right)
-        } else {
-          break
-        }
-      }
-      return res
-    },
-    map: function (o, f) {
-      o = o.start
-      var res = []
-      while (o != null) { // TODO: change to != (at least some convention)
-        var operation = this.getOperation(o)
-        if (!operation.deleted) {
-          res.push(f(operation))
-        }
-        o = operation.right
-      }
-      return res
-    }
-  }
-
-  /*
-    {
-      map: {},
-      struct: "Map",
-      type: "",
-      id: this.os.getNextOpId(1)
-    }
-  */
-  Struct.Map = {
-    create: function (id) {
-      return {
-        id: id,
-        map: {},
-        struct: 'Map'
-      }
-    },
-    encode: function (op) {
-      var e = {
-        struct: 'Map',
-        type: op.type,
-        id: op.id,
-        map: {} // overwrite map!!
-      }
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CMAP)
-      encoder.writeOpID(op.id)
-      encoder.writeVarString(op.type)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      let op = {
-        id: decoder.readOpID(),
-        type: decoder.readVarString(),
-        struct: 'Map',
-        map: {}
-      }
-      return op
-    },
-    requiredOps: function () {
-      return []
-    },
-    execute: function (op) {
-      op.start = null
-      op.end = null
-    },
-    /*
-      Get a property by name
-    */
-    get: function (op, name) {
-      var oid = op.map[name]
-      if (oid != null) {
-        var res = this.getOperation(oid)
-        if (res == null || res.deleted) {
-          return void 0
-        } else if (res.opContent == null) {
-          return res.content[0]
-        } else {
-          return this.getType(res.opContent)
-        }
-      }
-    }
-  }
-
-  /*
-    {
-      map: {},
-      start: null,
-      end: null,
-      struct: "Xml",
-      type: "",
-      id: this.os.getNextOpId(1)
-    }
-  */
-  Struct.Xml = {
-    create: function (id, args) {
-      let nodeName = args != null ? args.nodeName : null
-      return {
-        id: id,
-        map: {},
-        start: null,
-        end: null,
-        struct: 'Xml',
-        nodeName
-      }
-    },
-    encode: function (op) {
-      var e = {
-        struct: 'Xml',
-        type: op.type,
-        id: op.id,
-        map: {},
-        nodeName: op.nodeName
-      }
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CXML)
-      encoder.writeOpID(op.id)
-      encoder.writeVarString(op.type)
-      encoder.writeVarString(op.nodeName)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      let op = {
-        id: decoder.readOpID(),
-        type: decoder.readVarString(),
-        struct: 'Xml',
-        map: {},
-        start: null,
-        end: null,
-        nodeName: decoder.readVarString()
-      }
-      return op
-    },
-    requiredOps: function () {
-      return []
-    },
-    execute: function () {},
-    ref: Struct.List.ref,
-    map: Struct.List.map,
-    /*
-      Get a property by name
-    */
-    get: Struct.Map.get
-  }
-}

src/Struct/Delete.js

@@ -0,0 +1,31 @@
+import StructManager from '../Util/StructManager'
+
+export default class Delete {
+  constructor () {
+    this._target = null
+    this._length = null
+  }
+  _fromBinary (y, decoder) {
+    this._targetID = decoder.readOpID()
+    this._length = decoder.readVarUint()
+  }
+  _toBinary (y, encoder) {
+    encoder.writeUint8(StructManager.getReference(this.constructor))
+    encoder.writeOpID(this._targetID)
+    encoder.writeVarUint(this._length)
+  }
+  _integrate (y) {
+    let items = y.os.getItems(this._target, this._length)
+    for (let i = items.length - 1; i >= 0; i--) {
+      items[i]._delete()
+    }
+    // TODO: only broadcast if created by local user or if y.connector._forwardAppliedStructs..
+    y.connector.broadcastStruct(this)
+    if (y.persistence !== null) {
+      y.persistence.saveOperations(this)
+    }
+  }
+  _logString () {
+    return `Delete - target: ${this._target}, len: ${this._length}`
+  }
+}

src/Struct/Item.js

@@ -0,0 +1,159 @@
+import StructManager from '../Util/StructManager'
+
+export default class Item {
+  constructor () {
+    this._id = null
+    this._origin = null
+    this._left = null
+    this._right = null
+    this._right_origin = null
+    this._parent = null
+    this._parentSub = null
+    this._deleted = false
+  }
+  get _length () {
+    return 1
+  }
+  _getDistanceToOrigin () {
+    if (this.left == null) {
+      return 0
+    } else {
+      var d = 0
+      var o = this.left
+      while (o !== null && !this.origin.equals(o.id)) {
+        d++
+        o = o.left
+      }
+      return d
+    }
+  }
+  _delete (y) {
+    this._deleted = true
+    y.ds.markDeleted(this._id, this._length)
+  }
+  /*
+   * - Integrate the struct so that other types/structs can see it
+   * - Add this struct to y.os
+   * - Check if this is struct deleted
+   */
+  _integrate (y) {
+    if (this._id === null) {
+      this._id = y.ss.getNextID(this._length)
+    }
+    /*
+    # $this has to find a unique position between origin and the next known character
+    # case 1: $origin equals $o.origin: the $creator parameter decides if left or right
+    #         let $OL= [o1,o2,o3,o4], whereby $this is to be inserted between o1 and o4
+    #         o2,o3 and o4 origin is 1 (the position of o2)
+    #         there is the case that $this.creator < o2.creator, but o3.creator < $this.creator
+    #         then o2 knows o3. Since on another client $OL could be [o1,o3,o4] the problem is complex
+    #         therefore $this would be always to the right of o3
+    # case 2: $origin < $o.origin
+    #         if current $this insert_position > $o origin: $this ins
+    #         else $insert_position will not change
+    #         (maybe we encounter case 1 later, then this will be to the right of $o)
+    # case 3: $origin > $o.origin
+    #         $this insert_position is to the left of $o (forever!)
+    */
+    // handle conflicts
+    let o
+    // set o to the first conflicting item
+    if (this._left !== null) {
+      o = this._left._right
+    } else if (this._parentSub !== null) {
+      o = this._parent._map.get(this._parentSub)
+    } else {
+      o = this._parent._start
+    }
+    let conflictingItems = new Set()
+    let itemsBeforeOrigin = new Set()
+    // Let c in conflictingItems, b in itemsBeforeOrigin
+    // ***{origin}bbbb{this}{c,b}{c,b}{o}***
+    // Note that conflictingItems is a subset of itemsBeforeOrigin
+    while (o !== null && o !== this._right) {
+      itemsBeforeOrigin.add(o)
+      if (this.origin === o.origin) {
+        // case 1
+        if (o._id.user < this._id.user) {
+          this.left = o
+          conflictingItems = new Set()
+        }
+      } else if (itemsBeforeOrigin.has(o)) {
+        // case 2
+        if (conflictingItems.has(o)) {
+          this.left = o
+          conflictingItems = new Set()
+        }
+      } else {
+        break
+      }
+      o = o.right
+    }
+    y.os.set(this)
+    y.ds.checkIfDeleted(this)
+    if (y.connector._forwardAppliedStructs || this._id.user === y.userID) {
+      y.connector.broadcastStruct(this)
+    }
+    if (y.persistence !== null) {
+      y.persistence.saveOperations(this)
+    }
+  }
+  _toBinary (y, encoder) {
+    encoder.writeUint8(StructManager.getReference(this.constructor))
+    encoder.writeOpID(this._id)
+    encoder.writeOpID(this._parent._id)
+    encoder.writeVarString(this.parentSub === null ? '' : JSON.stringify(this.parentSub))
+    encoder.writeOpID(this._left === null ? null : this._left._id)
+    encoder.writeOpID(this._right_origin === null ? null : this._right_origin._id)
+    encoder.writeOpID(this._origin === null ? null : this._origin._id)
+  }
+  _fromBinary (y, decoder) {
+    let missing = []
+    this._id = decoder.readOpID()
+    let parent = decoder.readOpID()
+    let parentSub = decoder.readVarString()
+    if (parentSub.length > 0) {
+      this._parentSub = JSON.parse(parentSub)
+    }
+    let left = decoder.readOpID()
+    let right = decoder.readOpId()
+    let origin = decoder.readOpID()
+    if (parent !== null && this._parent === null) {
+      let _parent = y.os.get(parent)
+      if (_parent === null) {
+        missing.push(parent)
+      } else {
+        this._parent = _parent
+      }
+    }
+    if (origin !== null && this._origin === null) {
+      let _origin = y.os.getCleanStart(origin)
+      if (_origin === null) {
+        missing.push(origin)
+      } else {
+        this._origin = _origin
+      }
+    }
+    if (left !== null && this._left === null) {
+      let _left = y.os.getCleanEnd(left)
+      if (_left === null) {
+        // use origin instead
+        this._left = this._origin
+      } else {
+        this._left = _left
+      }
+    }
+    if (right !== null && this._right_origin === null) {
+      let _right = y.os.getCleanStart(right)
+      if (_right === null) {
+        missing.push(right)
+      } else {
+        this._right = _right
+        this._right_origin = _right
+      }
+    }
+  }
+  _logString () {
+    return `left: ${this._left}, origin: ${this._origin}, right: ${this._right}, parent: ${this._parent}, parentSub: ${this._parentSub}`
+  }
+}

src/Struct/ItemJSON.js

@@ -0,0 +1,32 @@
+import Item from './Item'
+
+export default class ItemJSON extends Item {
+  constructor () {
+    super()
+    this._content = null
+  }
+  get _length () {
+    return this._content.length
+  }
+  _fromBinary (y, decoder) {
+    let missing = super._fromBinary(y, decoder)
+    let len = decoder.readVarUint()
+    this._content = new Array(len)
+    for (let i = 0; i < len; i++) {
+      this._content[i] = JSON.parse(decoder.readVarString())
+    }
+    return missing
+  }
+  _toBinary (y, encoder) {
+    super._toBinary(y, encoder)
+    let len = this._content.length
+    encoder.writeVarUint(len)
+    for (let i = 0; i < len; i++) {
+      encoder.writeVarString(JSON.stringify(this._content[i]))
+    }
+  }
+  _logString () {
+    let s = super._logString()
+    return 'ItemJSON: ' + s
+  }
+}

src/Struct/ItemString.js

@@ -0,0 +1,24 @@
+import Item from './Item'
+
+export default class ItemString extends Item {
+  constructor () {
+    super()
+    this._content = null
+  }
+  get _length () {
+    return this._content.length
+  }
+  _fromBinary (y, decoder) {
+    let missing = super._fromBinary(y, decoder)
+    this._content = decoder.readVarString()
+    return missing
+  }
+  _toBinary (y, encoder) {
+    super._toBinary(y, encoder)
+    encoder.writeVarString(this._content)
+  }
+  _logString () {
+    let s = super._logString()
+    return 'ItemString: ' + s
+  }
+}

src/Struct/Type.js

@@ -0,0 +1,26 @@
+import Item from './Item'
+
+export default class Type extends Item {
+  constructor () {
+    super()
+    this._map = new Map()
+    this._start = null
+  }
+  _delete (y) {
+    super._delete(y)
+    // delete map types
+    for (let value of this._map.values()) {
+      if (value instanceof Item && !value._deleted) {
+        value._delete()
+      }
+    }
+    // delete array types
+    let t = this._start
+    while (t !== null) {
+      if (!t._deleted) {
+        t._delete()
+      }
+      t = t._right
+    }
+  }
+}

src/Type/YArray.js

@@ -0,0 +1,87 @@
+import Type from '../Struct/Type'
+import ItemJSON from '../Struct/ItemJSON'
+
+export default class YArray extends Type {
+  forEach (f) {
+    let pos = 0
+    let n = this._start
+    while (n !== null) {
+      let content = n._getContent()
+      for (let i = 0; i < content.length; i++) {
+        pos++
+        let c = content[i]
+        if (!c._deleted) {
+          f(content[i], pos, this)
+        }
+      }
+      n = n._right
+    }
+  }
+  [Symbol.iterator] () {
+    return {
+      next: function () {
+        while (this._item !== null && (this._item._deleted || this._item._content.length <= this._itemElement)) {
+          // item is deleted or itemElement does not exist (is deleted)
+          this._item = this._item._right
+          this._itemElement = 0
+        }
+        if (this._item === null) {
+          return {
+            done: true
+          }
+        } else {
+          return {
+            value: [this._count, this._item._content[this._itemElement++]],
+            done: false
+          }
+        }
+      },
+      _item: this._start,
+      _itemElement: 0,
+      _count: 0
+    }
+  }
+  insert (pos, content) {
+    let left = this._start
+    let right
+    let count = 0
+    while (left !== null && !left._deleted) {
+      if (pos < count + left._content.length) {
+        [left, right] = left._splitAt(pos - count)
+        break
+      }
+      left = left.right
+    }
+    if (pos > count) {
+      throw new Error('Position exceeds array range!')
+    }
+    let prevJsonIns = null
+    for (let i = 0; i < content.length; i++) {
+      let c = content[i]
+      if (c instanceof Type) {
+        if (prevJsonIns === null) {
+          prevJsonIns._integrate(this._y)
+          prevJsonIns = null
+        }
+        c._left = left
+        c._origin = left
+        c._right = right
+        c._parent = this
+      } else {
+        if (prevJsonIns === null) {
+          prevJsonIns = new ItemJSON()
+          prevJsonIns._origin = left
+          prevJsonIns._left = left
+          prevJsonIns._right = right
+          prevJsonIns._parent = this
+          prevJsonIns._content = []
+        }
+        prevJsonIns._content.push(c)
+      }
+    }
+  }
+  _logString () {
+    let s = super._logString()
+    return 'YArray: ' + s
+  }
+}

src/Type/YMap.js

@@ -0,0 +1,32 @@
+import Type from '../Struct/Type'
+import Item from '../Struct/Item'
+import ItemJSON from '../Struct/ItemJSON'
+
+export default class YMap extends Type {
+  set (key, value) {
+    let old = this._map.get(key)
+    let v
+    if (value instanceof Item) {
+      v = value
+    } else {
+      let v = new ItemJSON()
+      v._content = JSON.stringify(value)
+    }
+    v._right = old
+    v._parent = this
+    v._parentSub = key
+    v._integrate()
+  }
+  get (key) {
+    let v = this._map.get(key)
+    if (v instanceof Type) {
+      return v
+    } else {
+      return v._content[v._content.length - 1]
+    }
+  }
+  _logString () {
+    let s = super._logString()
+    return 'YMap: ' + s
+  }
+}

src/Util/EventHandler.js

@@ -0,0 +1,34 @@
+
+export default class EventHandler {
+  constructor () {
+    this.eventListeners = []
+  }
+  destroy () {
+    this.eventListeners = null
+  }
+  addEventListener (f) {
+    this.eventListeners.push(f)
+  }
+  removeEventListener (f) {
+    this.eventListeners = this.eventListeners.filter(function (g) {
+      return f !== g
+    })
+  }
+  removeAllEventListeners () {
+    this.eventListeners = []
+  }
+  callEventListeners (event) {
+    for (var i = 0; i < this.eventListeners.length; i++) {
+      try {
+        this.eventListeners[i](event)
+      } catch (e) {
+        /*
+          Your observer threw an error. This error was caught so that Yjs
+          can ensure data consistency! In order to debug this error you
+          have to check "Pause On Caught Exceptions" in developer tools.
+        */
+        console.error(e)
+      }
+    }
+  }
+}

src/Util/ID.js

@@ -0,0 +1,32 @@
+
+import StructManager from './StructManager'
+
+export class ID {
+  constructor (user, clock) {
+    this.user = user
+    this.clock = clock
+  }
+  clone () {
+    return new ID(this.user, this.clock)
+  }
+  equals (id) {
+    return id !== null && id.user === this.user && id.clock === this.user
+  }
+  lessThan (id) {
+    return this.user < id.user || (this.user === id.user && this.clock < id.clock)
+  }
+}
+
+export class RootID {
+  constructor (name, typeConstructor) {
+    this.user = -1
+    this.name = name
+    this.type = StructManager.getReference(typeConstructor)
+  }
+  equals (id) {
+    return id !== null && id.user === this.user && id.name === this.name && id.type === this.type
+  }
+  lessThan (id) {
+    return this.user < id.user || (this.user === id.user && (this.name < id.name || (this.name === id.name && this.type < id.type)))
+  }
+}

src/Util/NamedEventHandler.js

@@ -0,0 +1,28 @@
+export default class NamedEventHandler {
+  constructor () {
+    this._eventListener = {}
+  }
+  on (name, f) {
+    if (this._eventListener[name] == null) {
+      this._eventListener[name] = []
+    }
+    this._eventListener[name].push(f)
+  }
+  off (name, f) {
+    if (name == null || f == null) {
+      throw new Error('You must specify event name and function!')
+    }
+    let listener = this._eventListener[name] || []
+    this._eventListener[name] = listener.filter(e => e !== f)
+  }
+  emit (name, value) {
+    let listener = this._eventListener[name] || []
+    if (name === 'error' && listener.length === 0) {
+      console.error(value)
+    }
+    listener.forEach(l => l(value))
+  }
+  destroy () {
+    this._eventListener = null
+  }
+}

src/Util/Tree.js

@@ -0,0 +1,474 @@
+
+class N {
+  // A created node is always red!
+  constructor (val) {
+    this.val = val
+    this.color = true
+    this._left = null
+    this._right = null
+    this._parent = null
+  }
+  isRed () { return this.color }
+  isBlack () { return !this.color }
+  redden () { this.color = true; return this }
+  blacken () { this.color = false; return this }
+  get grandparent () {
+    return this.parent.parent
+  }
+  get parent () {
+    return this._parent
+  }
+  get sibling () {
+    return (this === this.parent.left)
+      ? this.parent.right : this.parent.left
+  }
+  get left () {
+    return this._left
+  }
+  get right () {
+    return this._right
+  }
+  set left (n) {
+    if (n !== null) {
+      n._parent = this
+    }
+    this._left = n
+  }
+  set right (n) {
+    if (n !== null) {
+      n._parent = this
+    }
+    this._right = n
+  }
+  rotateLeft (tree) {
+    var parent = this.parent
+    var newParent = this.right
+    var newRight = this.right.left
+    newParent.left = this
+    this.right = newRight
+    if (parent === null) {
+      tree.root = newParent
+      newParent._parent = null
+    } else if (parent.left === this) {
+      parent.left = newParent
+    } else if (parent.right === this) {
+      parent.right = newParent
+    } else {
+      throw new Error('The elements are wrongly connected!')
+    }
+  }
+  next () {
+    if (this.right !== null) {
+      // search the most left node in the right tree
+      var o = this.right
+      while (o.left !== null) {
+        o = o.left
+      }
+      return o
+    } else {
+      var p = this
+      while (p.parent !== null && p !== p.parent.left) {
+        p = p.parent
+      }
+      return p.parent
+    }
+  }
+  prev () {
+    if (this.left !== null) {
+      // search the most right node in the left tree
+      var o = this.left
+      while (o.right !== null) {
+        o = o.right
+      }
+      return o
+    } else {
+      var p = this
+      while (p.parent !== null && p !== p.parent.right) {
+        p = p.parent
+      }
+      return p.parent
+    }
+  }
+  rotateRight (tree) {
+    var parent = this.parent
+    var newParent = this.left
+    var newLeft = this.left.right
+    newParent.right = this
+    this.left = newLeft
+    if (parent === null) {
+      tree.root = newParent
+      newParent._parent = null
+    } else if (parent.left === this) {
+      parent.left = newParent
+    } else if (parent.right === this) {
+      parent.right = newParent
+    } else {
+      throw new Error('The elements are wrongly connected!')
+    }
+  }
+  getUncle () {
+    // we can assume that grandparent exists when this is called!
+    if (this.parent === this.parent.parent.left) {
+      return this.parent.parent.right
+    } else {
+      return this.parent.parent.left
+    }
+  }
+}
+
+/*
+ * This is a Red Black Tree implementation
+ */
+export default class Tree {
+  constructor () {
+    this.root = null
+    this.length = 0
+  }
+  findNext (id) {
+    var nextID = id.clone()
+    nextID.clock += 1
+    return this.findWithLowerBound(nextID)
+  }
+  findPrev (id) {
+    let prevID = id.clone()
+    prevID.clock -= 1
+    return this.findWithUpperBound(prevID)
+  }
+  findNodeWithLowerBound (from) {
+    var o = this.root
+    if (o === null) {
+      return null
+    } else {
+      while (true) {
+        if (from === null || (from.lessThan(o.val.id) && o.left !== null)) {
+          // o is included in the bound
+          // try to find an element that is closer to the bound
+          o = o.left
+        } else if (from !== null && o.val.id.lessThan(from)) {
+          // o is not within the bound, maybe one of the right elements is..
+          if (o.right !== null) {
+            o = o.right
+          } else {
+            // there is no right element. Search for the next bigger element,
+            // this should be within the bounds
+            return o.next()
+          }
+        } else {
+          return o
+        }
+      }
+    }
+  }
+  findNodeWithUpperBound (to) {
+    if (to === void 0) {
+      throw new Error('You must define from!')
+    }
+    var o = this.root
+    if (o === null) {
+      return null
+    } else {
+      while (true) {
+        if ((to === null || o.val.id.lessThan(to)) && o.right !== null) {
+          // o is included in the bound
+          // try to find an element that is closer to the bound
+          o = o.right
+        } else if (to !== null && to.lessThan(o.val.id)) {
+          // o is not within the bound, maybe one of the left elements is..
+          if (o.left !== null) {
+            o = o.left
+          } else {
+            // there is no left element. Search for the prev smaller element,
+            // this should be within the bounds
+            return o.prev()
+          }
+        } else {
+          return o
+        }
+      }
+    }
+  }
+  findSmallestNode () {
+    var o = this.root
+    while (o != null && o.left != null) {
+      o = o.left
+    }
+    return o
+  }
+  findWithLowerBound (from) {
+    var n = this.findNodeWithLowerBound(from)
+    return n == null ? null : n.val
+  }
+  findWithUpperBound (to) {
+    var n = this.findNodeWithUpperBound(to)
+    return n == null ? null : n.val
+  }
+  iterate (from, to, f) {
+    var o
+    if (from === null) {
+      o = this.findSmallestNode()
+    } else {
+      o = this.findNodeWithLowerBound(from)
+    }
+    while (
+      o !== null &&
+      (
+        to === null || // eslint-disable-line no-unmodified-loop-condition
+        o.val.id.lessThan(to) ||
+        o.val.id.equals(to)
+      )
+    ) {
+      f(o.val)
+      o = o.next()
+    }
+  }
+  find (id) {
+    let n = this.findNode(id)
+    if (n !== null) {
+      return n.val
+    } else {
+      return null
+    }
+  }
+  findNode (id) {
+    var o = this.root
+    if (o === null) {
+      return false
+    } else {
+      while (true) {
+        if (o === null) {
+          return false
+        }
+        if (id.lessThan(o.val.id)) {
+          o = o.left
+        } else if (o.val.id.lessThan(id)) {
+          o = o.right
+        } else {
+          return o
+        }
+      }
+    }
+  }
+  delete (id) {
+    if (id == null || id.constructor !== Array) {
+      throw new Error('id is expected to be an Array!')
+    }
+    var d = this.findNode(id)
+    if (d == null) {
+      // throw new Error('Element does not exist!')
+      return
+    }
+    this.length--
+    if (d.left !== null && d.right !== null) {
+      // switch d with the greates element in the left subtree.
+      // o should have at most one child.
+      var o = d.left
+      // find
+      while (o.right !== null) {
+        o = o.right
+      }
+      // switch
+      d.val = o.val
+      d = o
+    }
+    // d has at most one child
+    // let n be the node that replaces d
+    var isFakeChild
+    var child = d.left || d.right
+    if (child === null) {
+      isFakeChild = true
+      child = new N({id: 0})
+      child.blacken()
+      d.right = child
+    } else {
+      isFakeChild = false
+    }
+
+    if (d.parent === null) {
+      if (!isFakeChild) {
+        this.root = child
+        child.blacken()
+        child._parent = null
+      } else {
+        this.root = null
+      }
+      return
+    } else if (d.parent.left === d) {
+      d.parent.left = child
+    } else if (d.parent.right === d) {
+      d.parent.right = child
+    } else {
+      throw new Error('Impossible!')
+    }
+    if (d.isBlack()) {
+      if (child.isRed()) {
+        child.blacken()
+      } else {
+        this._fixDelete(child)
+      }
+    }
+    this.root.blacken()
+    if (isFakeChild) {
+      if (child.parent.left === child) {
+        child.parent.left = null
+      } else if (child.parent.right === child) {
+        child.parent.right = null
+      } else {
+        throw new Error('Impossible #3')
+      }
+    }
+  }
+  _fixDelete (n) {
+    function isBlack (node) {
+      return node !== null ? node.isBlack() : true
+    }
+    function isRed (node) {
+      return node !== null ? node.isRed() : false
+    }
+    if (n.parent === null) {
+      // this can only be called after the first iteration of fixDelete.
+      return
+    }
+    // d was already replaced by the child
+    // d is not the root
+    // d and child are black
+    var sibling = n.sibling
+    if (isRed(sibling)) {
+      // make sibling the grandfather
+      n.parent.redden()
+      sibling.blacken()
+      if (n === n.parent.left) {
+        n.parent.rotateLeft(this)
+      } else if (n === n.parent.right) {
+        n.parent.rotateRight(this)
+      } else {
+        throw new Error('Impossible #2')
+      }
+      sibling = n.sibling
+    }
+    // parent, sibling, and children of n are black
+    if (n.parent.isBlack() &&
+      sibling.isBlack() &&
+      isBlack(sibling.left) &&
+      isBlack(sibling.right)
+    ) {
+      sibling.redden()
+      this._fixDelete(n.parent)
+    } else if (n.parent.isRed() &&
+      sibling.isBlack() &&
+      isBlack(sibling.left) &&
+      isBlack(sibling.right)
+    ) {
+      sibling.redden()
+      n.parent.blacken()
+    } else {
+      if (n === n.parent.left &&
+        sibling.isBlack() &&
+        isRed(sibling.left) &&
+        isBlack(sibling.right)
+      ) {
+        sibling.redden()
+        sibling.left.blacken()
+        sibling.rotateRight(this)
+        sibling = n.sibling
+      } else if (n === n.parent.right &&
+        sibling.isBlack() &&
+        isRed(sibling.right) &&
+        isBlack(sibling.left)
+      ) {
+        sibling.redden()
+        sibling.right.blacken()
+        sibling.rotateLeft(this)
+        sibling = n.sibling
+      }
+      sibling.color = n.parent.color
+      n.parent.blacken()
+      if (n === n.parent.left) {
+        sibling.right.blacken()
+        n.parent.rotateLeft(this)
+      } else {
+        sibling.left.blacken()
+        n.parent.rotateRight(this)
+      }
+    }
+  }
+  put (v) {
+    var node = new N(v)
+    if (this.root !== null) {
+      var p = this.root // p abbrev. parent
+      while (true) {
+        if (node.val.id.lessThan(p.val.id)) {
+          if (p.left === null) {
+            p.left = node
+            break
+          } else {
+            p = p.left
+          }
+        } else if (p.val.id.lessThan(node.val.id)) {
+          if (p.right === null) {
+            p.right = node
+            break
+          } else {
+            p = p.right
+          }
+        } else {
+          p.val = node.val
+          return p
+        }
+      }
+      this._fixInsert(node)
+    } else {
+      this.root = node
+    }
+    this.length++
+    this.root.blacken()
+    return node
+  }
+  _fixInsert (n) {
+    if (n.parent === null) {
+      n.blacken()
+      return
+    } else if (n.parent.isBlack()) {
+      return
+    }
+    var uncle = n.getUncle()
+    if (uncle !== null && uncle.isRed()) {
+      // Note: parent: red, uncle: red
+      n.parent.blacken()
+      uncle.blacken()
+      n.grandparent.redden()
+      this._fixInsert(n.grandparent)
+    } else {
+      // Note: parent: red, uncle: black or null
+      // Now we transform the tree in such a way that
+      // either of these holds:
+      //   1) grandparent.left.isRed
+      //     and grandparent.left.left.isRed
+      //   2) grandparent.right.isRed
+      //     and grandparent.right.right.isRed
+      if (n === n.parent.right && n.parent === n.grandparent.left) {
+        n.parent.rotateLeft(this)
+        // Since we rotated and want to use the previous
+        // cases, we need to set n in such a way that
+        // n.parent.isRed again
+        n = n.left
+      } else if (n === n.parent.left && n.parent === n.grandparent.right) {
+        n.parent.rotateRight(this)
+        // see above
+        n = n.right
+      }
+      // Case 1) or 2) hold from here on.
+      // Now traverse grandparent, make parent a black node
+      // on the highest level which holds two red nodes.
+      n.parent.blacken()
+      n.grandparent.redden()
+      if (n === n.parent.left) {
+        // Case 1
+        n.grandparent.rotateRight(this)
+      } else {
+        // Case 2
+        n.grandparent.rotateLeft(this)
+      }
+    }
+  }
+  flush () {}
+}

src/Util/deleteItemRange.js

@@ -0,0 +1,9 @@
+import Delete from '../Struct/Delete'
+import ID from './ID'
+
+export default function deleteItemRange (y, user, clock, length) {
+  let del = new Delete()
+  del._target = new ID(user, clock)
+  del._length = length
+  del._integrate(y)
+}

src/Util/generateUserID.js

@@ -0,0 +1,16 @@
+/* global crypto */
+
+export default function generateUserID () {
+  if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) {
+    // browser
+    let arr = new Uint32Array(1)
+    crypto.getRandomValues(arr)
+    return arr[0]
+  } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) {
+    // node
+    let buf = crypto.randomBytes(4)
+    return new Uint32Array(buf.buffer)[0]
+  } else {
+    return Math.ceil(Math.random() * 0xFFFFFFFF)
+  }
+}

src/Util/relativePosition.js

@@ -0,0 +1,45 @@
+import ID from './ID'
+
+export function getRelativePosition (type, offset) {
+  if (offset === 0) {
+    return ['startof', type._id.user, type._id.clock]
+  } else {
+    let t = type.start
+    while (t !== null && t.length < offset) {
+      if (!t._deleted) {
+        offset -= t.length
+      }
+      t = t._right
+    }
+    return [t._id.user, t._id.clock + offset - 1]
+  }
+}
+
+export function fromRelativePosition (y, rpos) {
+  if (rpos[0] === 'startof') {
+    return {
+      type: y.os.get(new ID(rpos[1], rpos[2])),
+      offset: 0
+    }
+  } else {
+    let offset = 0
+    let struct = y.os.findNodeWithUpperBound(new ID(rpos[0], rpos[1]))
+    let parent = struct._parent
+    if (parent._deleted) {
+      return null
+    }
+    if (!struct.deleted) {
+      offset = rpos[1] - struct._id.clock
+    }
+    while (struct.left !== null) {
+      struct = struct.left
+      if (!struct.deleted) {
+        offset += struct._length
+      }
+    }
+    return {
+      type: parent,
+      offset: offset
+    }
+  }
+}

src/Util/structReferences.js

@@ -0,0 +1,30 @@
+import YArray from '../Type/YArray'
+import YMap from '../Type/YMap'
+import YText from '../Type/YText'
+import YXml from '../Type/YXml'
+
+import ItemJSON from '../Struct/ItemJSON'
+import ItemString from '../Struct/ItemString'
+
+const structs = new Map()
+const references = new Map()
+
+function addStruct (reference, structConstructor) {
+  structs.set(reference, structConstructor)
+  references.set(structConstructor, reference)
+}
+
+export function getStruct (reference) {
+  return structs.get(reference)
+}
+
+export function getReference (typeConstructor) {
+  return references.get(typeConstructor)
+}
+
+addStruct(0, YArray)
+addStruct(1, YMap)
+addStruct(2, YText)
+addStruct(3, YXml)
+addStruct(4, ItemJSON)
+addStruct(5, ItemString)

src/Util/writeJSONToType.js

@@ -0,0 +1,33 @@
+
+import YMap from '../Type/YMap'
+import YArray from '../Type/YArray'
+
+export function writeObjectToYMap (object, type) {
+  for (var key in object) {
+    var val = object[key]
+    if (Array.isArray(val)) {
+      type.set(key, YArray)
+      writeArrayToYArray(val, type.get(key))
+    } else if (typeof val === 'object') {
+      type.set(key, YMap)
+      writeObjectToYMap(val, type.get(key))
+    } else {
+      type.set(key, val)
+    }
+  }
+}
+
+export function writeArrayToYArray (array, type) {
+  for (var i = array.length - 1; i >= 0; i--) {
+    var val = array[i]
+    if (Array.isArray(val)) {
+      type.insert(0, [YArray])
+      writeArrayToYArray(val, type.get(0))
+    } else if (typeof val === 'object') {
+      type.insert(0, [YMap])
+      writeObjectToYMap(val, type.get(0))
+    } else {
+      type.insert(0, [val])
+    }
+  }
+}

src/Utils.js

@@ -1,935 +0,0 @@
-/* globals crypto */
-
-import { BinaryDecoder, BinaryEncoder } from './Encoding.js'
-
-/*
-  EventHandler is an helper class for constructing custom types.
-
-  Why: When constructing custom types, you sometimes want your types to work
-  synchronous: E.g.
-  ``` Synchronous
-    mytype.setSomething("yay")
-    mytype.getSomething() === "yay"
-  ```
-  versus
-  ``` Asynchronous
-    mytype.setSomething("yay")
-    mytype.getSomething() === undefined
-    mytype.waitForSomething().then(function(){
-      mytype.getSomething() === "yay"
-    })
-  ```
-
-  The structures usually work asynchronously (you have to wait for the
-  database request to finish). EventHandler helps you to make your type
-  synchronous.
-*/
-
-export default function Utils (Y) {
-  Y.utils = {
-    BinaryDecoder: BinaryDecoder,
-    BinaryEncoder: BinaryEncoder
-  }
-
-  Y.utils.bubbleEvent = function (type, event) {
-    type.eventHandler.callEventListeners(event)
-    event.path = []
-    while (type != null && type._deepEventHandler != null) {
-      type._deepEventHandler.callEventListeners(event)
-      var parent = null
-      if (type._parent != null) {
-        parent = type.os.getType(type._parent)
-      }
-      if (parent != null && parent._getPathToChild != null) {
-        event.path = [parent._getPathToChild(type._model)].concat(event.path)
-        type = parent
-      } else {
-        type = null
-      }
-    }
-  }
-
-  Y.utils.getRelativePosition = function (type, offset) {
-    if (type == null) {
-      return null
-    } else {
-      if (type._content.length <= offset) {
-        return ['endof', type._model[0], type._model[1]]
-      } else {
-        return type._content[offset].id
-      }
-    }
-  }
-
-  Y.utils.fromRelativePosition = function (y, id) {
-    var offset = 0
-    var op
-    if (id[0] === 'endof') {
-      id = y.db.os.find(id.slice(1)).end
-      op = y.db.os.findNodeWithUpperBound(id).val
-      if (!op.deleted) {
-        offset = op.content != null ? op.content.length : 1
-      }
-    } else {
-      op = y.db.os.findNodeWithUpperBound(id).val
-      if (!op.deleted) {
-        offset = id[1] - op.id[1]
-      }
-    }
-
-    var type = y.db.getType(op.parent)
-    if (type == null || y.db.os.find(op.parent).deleted) {
-      return null
-    }
-
-    while (op.left != null) {
-      op = y.db.os.findNodeWithUpperBound(op.left).val
-      if (!op.deleted) {
-        offset += op.content != null ? op.content.length : 1
-      }
-    }
-    return {
-      type: type,
-      offset: offset
-    }
-  }
-
-  class NamedEventHandler {
-    constructor () {
-      this._eventListener = {}
-    }
-    on (name, f) {
-      if (this._eventListener[name] == null) {
-        this._eventListener[name] = []
-      }
-      this._eventListener[name].push(f)
-    }
-    off (name, f) {
-      if (name == null || f == null) {
-        throw new Error('You must specify event name and function!')
-      }
-      let listener = this._eventListener[name] || []
-      this._eventListener[name] = listener.filter(e => e !== f)
-    }
-    emit (name, value) {
-      let listener = this._eventListener[name] || []
-      if (name === 'error' && listener.length === 0) {
-        console.error(value)
-      }
-      listener.forEach(l => l(value))
-    }
-    destroy () {
-      this._eventListener = null
-    }
-  }
-  Y.utils.NamedEventHandler = NamedEventHandler
-
-  class EventListenerHandler {
-    constructor () {
-      this.eventListeners = []
-    }
-    destroy () {
-      this.eventListeners = null
-    }
-     /*
-      Basic event listener boilerplate...
-    */
-    addEventListener (f) {
-      this.eventListeners.push(f)
-    }
-    removeEventListener (f) {
-      this.eventListeners = this.eventListeners.filter(function (g) {
-        return f !== g
-      })
-    }
-    removeAllEventListeners () {
-      this.eventListeners = []
-    }
-    callEventListeners (event) {
-      for (var i = 0; i < this.eventListeners.length; i++) {
-        try {
-          var _event = {}
-          for (var name in event) {
-            _event[name] = event[name]
-          }
-          this.eventListeners[i](_event)
-        } catch (e) {
-          /*
-            Your observer threw an error. This error was caught so that Yjs
-            can ensure data consistency! In order to debug this error you
-            have to check "Pause On Caught Exceptions" in developer tools.
-          */
-          console.error(e)
-        }
-      }
-    }
-  }
-  Y.utils.EventListenerHandler = EventListenerHandler
-
-  class EventHandler extends EventListenerHandler {
-    /* ::
-    waiting: Array<Insertion | Deletion>;
-    awaiting: number;
-    onevent: Function;
-    eventListeners: Array<Function>;
-    */
-    /*
-      onevent: is called when the structure changes.
-
-      Note: "awaiting opertations" is used to denote operations that were
-      prematurely called. Events for received operations can not be executed until
-      all prematurely called operations were executed ("waiting operations")
-    */
-    constructor (onevent /* : Function */) {
-      super()
-      this.waiting = []
-      this.awaiting = 0
-      this.onevent = onevent
-    }
-    destroy () {
-      super.destroy()
-      this.waiting = null
-      this.onevent = null
-    }
-    /*
-      Call this when a new operation arrives. It will be executed right away if
-      there are no waiting operations, that you prematurely executed
-    */
-    receivedOp (op) {
-      if (this.awaiting <= 0) {
-        this.onevent(op)
-      } else if (op.struct === 'Delete') {
-        var self = this
-        var checkDelete = function checkDelete (d) {
-          if (d.length == null) {
-            throw new Error('This shouldn\'t happen! d.length must be defined!')
-          }
-          // we check if o deletes something in self.waiting
-          // if so, we remove the deleted operation
-          for (var w = 0; w < self.waiting.length; w++) {
-            var i = self.waiting[w]
-            if (i.struct === 'Insert' && i.id[0] === d.target[0]) {
-              var iLength = i.hasOwnProperty('content') ? i.content.length : 1
-              var dStart = d.target[1]
-              var dEnd = d.target[1] + (d.length || 1)
-              var iStart = i.id[1]
-              var iEnd = i.id[1] + iLength
-              // Check if they don't overlap
-              if (iEnd <= dStart || dEnd <= iStart) {
-                // no overlapping
-                continue
-              }
-              // we check all overlapping cases. All cases:
-              /*
-                1)  iiiii
-                      ddddd
-                    --> modify i and d
-                2)  iiiiiii
-                      ddddd
-                    --> modify i, remove d
-                3)  iiiiiii
-                      ddd
-                    --> remove d, modify i, and create another i (for the right hand side)
-                4)  iiiii
-                    ddddddd
-                    --> remove i, modify d
-                5)  iiiiiii
-                    ddddddd
-                    --> remove both i and d (**)
-                6)  iiiiiii
-                    ddddd
-                    --> modify i, remove d
-                7)    iii
-                    ddddddd
-                    --> remove i, create and apply two d with checkDelete(d) (**)
-                8)    iiiii
-                    ddddddd
-                    --> remove i, modify d (**)
-                9)    iiiii
-                    ddddd
-                    --> modify i and d
-                (**) (also check if i contains content or type)
-              */
-              // TODO: I left some debugger statements, because I want to debug all cases once in production. REMEMBER END TODO
-              if (iStart < dStart) {
-                if (dStart < iEnd) {
-                  if (iEnd < dEnd) {
-                    // Case 1
-                    // remove the right part of i's content
-                    i.content.splice(dStart - iStart)
-                    // remove the start of d's deletion
-                    d.length = dEnd - iEnd
-                    d.target = [d.target[0], iEnd]
-                    continue
-                  } else if (iEnd === dEnd) {
-                    // Case 2
-                    i.content.splice(dStart - iStart)
-                    // remove d, we do that by simply ending this function
-                    return
-                  } else { // (dEnd < iEnd)
-                    // Case 3
-                    var newI = {
-                      id: [i.id[0], dEnd],
-                      content: i.content.slice(dEnd - iStart),
-                      struct: 'Insert'
-                    }
-                    self.waiting.push(newI)
-                    i.content.splice(dStart - iStart)
-                    return
-                  }
-                }
-              } else if (dStart === iStart) {
-                if (iEnd < dEnd) {
-                  // Case 4
-                  d.length = dEnd - iEnd
-                  d.target = [d.target[0], iEnd]
-                  i.content = []
-                  continue
-                } else if (iEnd === dEnd) {
-                  // Case 5
-                  self.waiting.splice(w, 1)
-                  return
-                } else { // (dEnd < iEnd)
-                  // Case 6
-                  i.content = i.content.slice(dEnd - iStart)
-                  i.id = [i.id[0], dEnd]
-                  return
-                }
-              } else { // (dStart < iStart)
-                if (iStart < dEnd) {
-                  // they overlap
-                  /*
-                  7)    iii
-                      ddddddd
-                      --> remove i, create and apply two d with checkDelete(d) (**)
-                  8)    iiiii
-                      ddddddd
-                      --> remove i, modify d (**)
-                  9)    iiiii
-                      ddddd
-                      --> modify i and d
-                  */
-                  if (iEnd < dEnd) {
-                    // Case 7
-                    // debugger // TODO: You did not test this case yet!!!! (add the debugger here)
-                    self.waiting.splice(w, 1)
-                    checkDelete({
-                      target: [d.target[0], dStart],
-                      length: iStart - dStart,
-                      struct: 'Delete'
-                    })
-                    checkDelete({
-                      target: [d.target[0], iEnd],
-                      length: iEnd - dEnd,
-                      struct: 'Delete'
-                    })
-                    return
-                  } else if (iEnd === dEnd) {
-                    // Case 8
-                    self.waiting.splice(w, 1)
-                    w--
-                    d.length -= iLength
-                    continue
-                  } else { // dEnd < iEnd
-                    // Case 9
-                    d.length = iStart - dStart
-                    i.content.splice(0, dEnd - iStart)
-                    i.id = [i.id[0], dEnd]
-                    continue
-                  }
-                }
-              }
-            }
-          }
-          // finished with remaining operations
-          self.waiting.push(d)
-        }
-        if (op.key == null) {
-          // deletes in list
-          checkDelete(op)
-        } else {
-          // deletes in map
-          this.waiting.push(op)
-        }
-      } else {
-        this.waiting.push(op)
-      }
-    }
-    /*
-      You created some operations, and you want the `onevent` function to be
-      called right away. Received operations will not be executed untill all
-      prematurely called operations are executed
-    */
-    awaitAndPrematurelyCall (ops) {
-      this.awaiting++
-      ops.map(Y.utils.copyOperation).forEach(this.onevent)
-    }
-    awaitOps (transaction, f, args) {
-      function notSoSmartSort (array) {
-        // this function sorts insertions in a executable order
-        var result = []
-        while (array.length > 0) {
-          for (var i = 0; i < array.length; i++) {
-            var independent = true
-            for (var j = 0; j < array.length; j++) {
-              if (Y.utils.matchesId(array[j], array[i].left)) {
-                // array[i] depends on array[j]
-                independent = false
-                break
-              }
-            }
-            if (independent) {
-              result.push(array.splice(i, 1)[0])
-              i--
-            }
-          }
-        }
-        return result
-      }
-      var before = this.waiting.length
-      // somehow create new operations
-      f.apply(transaction, args)
-      // remove all appended ops / awaited ops
-      this.waiting.splice(before)
-      if (this.awaiting > 0) this.awaiting--
-      // if there are no awaited ops anymore, we can update all waiting ops, and send execute them (if there are still no awaited ops)
-      if (this.awaiting === 0 && this.waiting.length > 0) {
-        // update all waiting ops
-        for (let i = 0; i < this.waiting.length; i++) {
-          var o = this.waiting[i]
-          if (o.struct === 'Insert') {
-            var _o = transaction.getInsertion(o.id)
-            if (_o.parentSub != null && _o.left != null) {
-              // if o is an insertion of a map struc (parentSub is defined), then it shouldn't be necessary to compute left
-              this.waiting.splice(i, 1)
-              i-- // update index
-            } else if (!Y.utils.compareIds(_o.id, o.id)) {
-              // o got extended
-              o.left = [o.id[0], o.id[1] - 1]
-            } else if (_o.left == null) {
-              o.left = null
-            } else {
-              // find next undeleted op
-              var left = transaction.getInsertion(_o.left)
-              while (left.deleted != null) {
-                if (left.left != null) {
-                  left = transaction.getInsertion(left.left)
-                } else {
-                  left = null
-                  break
-                }
-              }
-              o.left = left != null ? Y.utils.getLastId(left) : null
-            }
-          }
-        }
-        // the previous stuff was async, so we have to check again!
-        // We also pull changes from the bindings, if there exists such a method, this could increase awaiting too
-        if (this._pullChanges != null) {
-          this._pullChanges()
-        }
-        if (this.awaiting === 0) {
-          // sort by type, execute inserts first
-          var ins = []
-          var dels = []
-          this.waiting.forEach(function (o) {
-            if (o.struct === 'Delete') {
-              dels.push(o)
-            } else {
-              ins.push(o)
-            }
-          })
-          this.waiting = []
-          // put in executable order
-          ins = notSoSmartSort(ins)
-          // this.onevent can trigger the creation of another operation
-          // -> check if this.awaiting increased & stop computation if it does
-          for (var i = 0; i < ins.length; i++) {
-            if (this.awaiting === 0) {
-              this.onevent(ins[i])
-            } else {
-              this.waiting = this.waiting.concat(ins.slice(i))
-              break
-            }
-          }
-          for (i = 0; i < dels.length; i++) {
-            if (this.awaiting === 0) {
-              this.onevent(dels[i])
-            } else {
-              this.waiting = this.waiting.concat(dels.slice(i))
-              break
-            }
-          }
-        }
-      }
-    }
-    // TODO: Remove awaitedInserts and awaitedDeletes in favor of awaitedOps, as they are deprecated and do not always work
-    // Do this in one of the coming releases that are breaking anyway
-    /*
-      Call this when you successfully awaited the execution of n Insert operations
-    */
-    awaitedInserts (n) {
-      var ops = this.waiting.splice(this.waiting.length - n)
-      for (var oid = 0; oid < ops.length; oid++) {
-        var op = ops[oid]
-        if (op.struct === 'Insert') {
-          for (var i = this.waiting.length - 1; i >= 0; i--) {
-            let w = this.waiting[i]
-            // TODO: do I handle split operations correctly here? Super unlikely, but yeah..
-            // Also: can this case happen? Can op be inserted in the middle of a larger op that is in $waiting?
-            if (w.struct === 'Insert') {
-              if (Y.utils.matchesId(w, op.left)) {
-                // include the effect of op in w
-                w.right = op.id
-                // exclude the effect of w in op
-                op.left = w.left
-              } else if (Y.utils.compareIds(w.id, op.right)) {
-                // similar..
-                w.left = Y.utils.getLastId(op)
-                op.right = w.right
-              }
-            }
-          }
-        } else {
-          throw new Error('Expected Insert Operation!')
-        }
-      }
-      this._tryCallEvents(n)
-    }
-    /*
-      Call this when you successfully awaited the execution of n Delete operations
-    */
-    awaitedDeletes (n, newLeft) {
-      var ops = this.waiting.splice(this.waiting.length - n)
-      for (var j = 0; j < ops.length; j++) {
-        var del = ops[j]
-        if (del.struct === 'Delete') {
-          if (newLeft != null) {
-            for (var i = 0; i < this.waiting.length; i++) {
-              let w = this.waiting[i]
-              // We will just care about w.left
-              if (w.struct === 'Insert' && Y.utils.compareIds(del.target, w.left)) {
-                w.left = newLeft
-              }
-            }
-          }
-        } else {
-          throw new Error('Expected Delete Operation!')
-        }
-      }
-      this._tryCallEvents(n)
-    }
-    /* (private)
-      Try to execute the events for the waiting operations
-    */
-    _tryCallEvents () {
-      function notSoSmartSort (array) {
-        var result = []
-        while (array.length > 0) {
-          for (var i = 0; i < array.length; i++) {
-            var independent = true
-            for (var j = 0; j < array.length; j++) {
-              if (Y.utils.matchesId(array[j], array[i].left)) {
-                // array[i] depends on array[j]
-                independent = false
-                break
-              }
-            }
-            if (independent) {
-              result.push(array.splice(i, 1)[0])
-              i--
-            }
-          }
-        }
-        return result
-      }
-      if (this.awaiting > 0) this.awaiting--
-      if (this.awaiting === 0 && this.waiting.length > 0) {
-        var ins = []
-        var dels = []
-        this.waiting.forEach(function (o) {
-          if (o.struct === 'Delete') {
-            dels.push(o)
-          } else {
-            ins.push(o)
-          }
-        })
-        ins = notSoSmartSort(ins)
-        ins.forEach(this.onevent)
-        dels.forEach(this.onevent)
-        this.waiting = []
-      }
-    }
-  }
-  Y.utils.EventHandler = EventHandler
-
-  /*
-    Default class of custom types!
-  */
-  class CustomType {
-    getPath () {
-      var parent = null
-      if (this._parent != null) {
-        parent = this.os.getType(this._parent)
-      }
-      if (parent != null && parent._getPathToChild != null) {
-        var firstKey = parent._getPathToChild(this._model)
-        var parentKeys = parent.getPath()
-        parentKeys.push(firstKey)
-        return parentKeys
-      } else {
-        return []
-      }
-    }
-  }
-  Y.utils.CustomType = CustomType
-
-  /*
-    A wrapper for the definition of a custom type.
-    Every custom type must have three properties:
-
-    * struct
-      - Structname of this type
-    * initType
-      - Given a model, creates a custom type
-    * class
-      - the constructor of the custom type (e.g. in order to inherit from a type)
-  */
-  class CustomTypeDefinition { // eslint-disable-line
-    /* ::
-    struct: any;
-    initType: any;
-    class: Function;
-    name: String;
-    */
-    constructor (def) {
-      if (def.struct == null ||
-        def.initType == null ||
-        def.class == null ||
-        def.name == null ||
-        def.createType == null
-      ) {
-        throw new Error('Custom type was not initialized correctly!')
-      }
-      this.struct = def.struct
-      this.initType = def.initType
-      this.createType = def.createType
-      this.class = def.class
-      this.name = def.name
-      if (def.appendAdditionalInfo != null) {
-        this.appendAdditionalInfo = def.appendAdditionalInfo
-      }
-      this.parseArguments = (def.parseArguments || function () {
-        return [this]
-      }).bind(this)
-      this.parseArguments.typeDefinition = this
-    }
-  }
-  Y.utils.CustomTypeDefinition = CustomTypeDefinition
-
-  Y.utils.isTypeDefinition = function isTypeDefinition (v) {
-    if (v != null) {
-      if (v instanceof Y.utils.CustomTypeDefinition) return [v]
-      else if (v.constructor === Array && v[0] instanceof Y.utils.CustomTypeDefinition) return v
-      else if (v instanceof Function && v.typeDefinition instanceof Y.utils.CustomTypeDefinition) return [v.typeDefinition]
-    }
-    return false
-  }
-
-  /*
-    Make a flat copy of an object
-    (just copy properties)
-  */
-  function copyObject (o) {
-    var c = {}
-    for (var key in o) {
-      c[key] = o[key]
-    }
-    return c
-  }
-  Y.utils.copyObject = copyObject
-
-  /*
-    Copy an operation, so that it can be manipulated.
-    Note: You must not change subproperties (except o.content)!
-  */
-  function copyOperation (o) {
-    o = copyObject(o)
-    if (o.content != null) {
-      o.content = o.content.map(function (c) { return c })
-    }
-    return o
-  }
-
-  Y.utils.copyOperation = copyOperation
-
-  /*
-    Defines a smaller relation on Id's
-  */
-  function smaller (a, b) {
-    return a[0] < b[0] || (a[0] === b[0] && (a[1] < b[1] || typeof a[1] < typeof b[1]))
-  }
-  Y.utils.smaller = smaller
-
-  function inDeletionRange (del, ins) {
-    return del.target[0] === ins[0] && del.target[1] <= ins[1] && ins[1] < del.target[1] + (del.length || 1)
-  }
-  Y.utils.inDeletionRange = inDeletionRange
-
-  function compareIds (id1, id2) {
-    if (id1 == null || id2 == null) {
-      return id1 === id2
-    } else {
-      return id1[0] === id2[0] && id1[1] === id2[1]
-    }
-  }
-  Y.utils.compareIds = compareIds
-
-  function matchesId (op, id) {
-    if (id == null || op == null) {
-      return id === op
-    } else {
-      if (id[0] === op.id[0]) {
-        if (op.content == null) {
-          return id[1] === op.id[1]
-        } else {
-          return id[1] >= op.id[1] && id[1] < op.id[1] + op.content.length
-        }
-      }
-    }
-    return false
-  }
-  Y.utils.matchesId = matchesId
-
-  function getLastId (op) {
-    if (op.content == null || op.content.length === 1) {
-      return op.id
-    } else {
-      return [op.id[0], op.id[1] + op.content.length - 1]
-    }
-  }
-  Y.utils.getLastId = getLastId
-
-  function createEmptyOpsArray (n) {
-    var a = new Array(n)
-    for (var i = 0; i < a.length; i++) {
-      a[i] = {
-        id: [null, null]
-      }
-    }
-    return a
-  }
-
-  function createSmallLookupBuffer (Store) {
-    /*
-      This buffer implements a very small buffer that temporarily stores operations
-      after they are read / before they are written.
-      The buffer basically implements FIFO. Often requested lookups will be re-queued every time they are looked up / written.
-
-      It can speed up lookups on Operation Stores and State Stores. But it does not require notable use of memory or processing power.
-
-      Good for os and ss, bot not for ds (because it often uses methods that require a flush)
-
-      I tried to optimize this for performance, therefore no highlevel operations.
-    */
-    class SmallLookupBuffer extends Store {
-      constructor (arg1, arg2) {
-        // super(...arguments) -- do this when this is supported by stable nodejs
-        super(arg1, arg2)
-        this.writeBuffer = createEmptyOpsArray(5)
-        this.readBuffer = createEmptyOpsArray(10)
-      }
-      find (id, noSuperCall) {
-        var i, r
-        for (i = this.readBuffer.length - 1; i >= 0; i--) {
-          r = this.readBuffer[i]
-          // we don't have to use compareids, because id is always defined!
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            // found r
-            // move r to the end of readBuffer
-            for (; i < this.readBuffer.length - 1; i++) {
-              this.readBuffer[i] = this.readBuffer[i + 1]
-            }
-            this.readBuffer[this.readBuffer.length - 1] = r
-            return r
-          }
-        }
-        var o
-        for (i = this.writeBuffer.length - 1; i >= 0; i--) {
-          r = this.writeBuffer[i]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            o = r
-            break
-          }
-        }
-        if (i < 0 && noSuperCall === undefined) {
-          // did not reach break in last loop
-          // read id and put it to the end of readBuffer
-          o = super.find(id)
-        }
-        if (o != null) {
-          for (i = 0; i < this.readBuffer.length - 1; i++) {
-            this.readBuffer[i] = this.readBuffer[i + 1]
-          }
-          this.readBuffer[this.readBuffer.length - 1] = o
-        }
-        return o
-      }
-      put (o) {
-        var id = o.id
-        var i, r // helper variables
-        for (i = this.writeBuffer.length - 1; i >= 0; i--) {
-          r = this.writeBuffer[i]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            // is already in buffer
-            // forget r, and move o to the end of writeBuffer
-            for (; i < this.writeBuffer.length - 1; i++) {
-              this.writeBuffer[i] = this.writeBuffer[i + 1]
-            }
-            this.writeBuffer[this.writeBuffer.length - 1] = o
-            break
-          }
-        }
-        if (i < 0) {
-          // did not reach break in last loop
-          // write writeBuffer[0]
-          var write = this.writeBuffer[0]
-          if (write.id[0] !== null) {
-            super.put(write)
-          }
-          // put o to the end of writeBuffer
-          for (i = 0; i < this.writeBuffer.length - 1; i++) {
-            this.writeBuffer[i] = this.writeBuffer[i + 1]
-          }
-          this.writeBuffer[this.writeBuffer.length - 1] = o
-        }
-        // check readBuffer for every occurence of o.id, overwrite if found
-        // whether found or not, we'll append o to the readbuffer
-        for (i = 0; i < this.readBuffer.length - 1; i++) {
-          r = this.readBuffer[i + 1]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            this.readBuffer[i] = o
-          } else {
-            this.readBuffer[i] = r
-          }
-        }
-        this.readBuffer[this.readBuffer.length - 1] = o
-      }
-      delete (id) {
-        var i, r
-        for (i = 0; i < this.readBuffer.length; i++) {
-          r = this.readBuffer[i]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            this.readBuffer[i] = {
-              id: [null, null]
-            }
-          }
-        }
-        this.flush()
-        super.delete(id)
-      }
-      findWithLowerBound (id) {
-        var o = this.find(id, true)
-        if (o != null) {
-          return o
-        } else {
-          this.flush()
-          return super.findWithLowerBound.apply(this, arguments)
-        }
-      }
-      findWithUpperBound (id) {
-        var o = this.find(id, true)
-        if (o != null) {
-          return o
-        } else {
-          this.flush()
-          return super.findWithUpperBound.apply(this, arguments)
-        }
-      }
-      findNext () {
-        this.flush()
-        return super.findNext.apply(this, arguments)
-      }
-      findPrev () {
-        this.flush()
-        return super.findPrev.apply(this, arguments)
-      }
-      iterate () {
-        this.flush()
-        super.iterate.apply(this, arguments)
-      }
-      flush () {
-        for (var i = 0; i < this.writeBuffer.length; i++) {
-          var write = this.writeBuffer[i]
-          if (write.id[0] !== null) {
-            super.put(write)
-            this.writeBuffer[i] = {
-              id: [null, null]
-            }
-          }
-        }
-      }
-    }
-    return SmallLookupBuffer
-  }
-  Y.utils.createSmallLookupBuffer = createSmallLookupBuffer
-
-  function generateUserId () {
-    if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) {
-      // browser
-      let arr = new Uint32Array(1)
-      crypto.getRandomValues(arr)
-      return arr[0]
-    } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) {
-      // node
-      let buf = crypto.randomBytes(4)
-      return new Uint32Array(buf.buffer)[0]
-    } else {
-      return Math.ceil(Math.random() * 0xFFFFFFFF)
-    }
-  }
-  Y.utils.generateUserId = generateUserId
-
-  Y.utils.parseTypeDefinition = function parseTypeDefinition (type, typeArgs) {
-    var args = []
-    try {
-      args = JSON.parse('[' + typeArgs + ']')
-    } catch (e) {
-      throw new Error('Was not able to parse type definition!')
-    }
-    if (type.typeDefinition.parseArguments != null) {
-      args = type.typeDefinition.parseArguments(args[0])[1]
-    }
-    return args
-  }
-
-  Y.utils.writeObjectToYMap = function writeObjectToYMap (object, type) {
-    for (var key in object) {
-      var val = object[key]
-      if (Array.isArray(val)) {
-        type.set(key, Y.Array)
-        Y.utils.writeArrayToYArray(val, type.get(key))
-      } else if (typeof val === 'object') {
-        type.set(key, Y.Map)
-        Y.utils.writeObjectToYMap(val, type.get(key))
-      } else {
-        type.set(key, val)
-      }
-    }
-  }
-
-  Y.utils.writeArrayToYArray = function writeArrayToYArray (array, type) {
-    for (var i = array.length - 1; i >= 0; i--) {
-      var val = array[i]
-      if (Array.isArray(val)) {
-        type.insert(0, [Y.Array])
-        Y.utils.writeArrayToYArray(val, type.get(0))
-      } else if (typeof val === 'object') {
-        type.insert(0, [Y.Map])
-        Y.utils.writeObjectToYMap(val, type.get(0))
-      } else {
-        type.insert(0, [val])
-      }
-    }
-  }
-}

src/Y.js

@@ -0,0 +1,101 @@
+
+import debug from 'debug'
+
+import DeleteStore from './Store/DeleteStore'
+import OperationStore from './Store/OperationStore'
+import StateStore from './Store/StateStore'
+import generateUserID from './Function/generateUserID'
+import { RootID } from './Util/ID.js'
+
+import { formatYjsMessage, formatYjsMessageType } from './MessageHandler'
+
+import Connector from './Connector'
+import Persistence from './Persistence'
+import YArray from './Type/YArray'
+import YMap from './Type/YMap'
+import YText from './Type/YText'
+import YXml from './Type/YXml'
+
+export default class Y {
+  constructor (opts) {
+    this.userID = generateUserID()
+    this.ds = new DeleteStore(this)
+    this.os = new OperationStore(this)
+    this.ss = new StateStore(this)
+    this.connector = new Y[opts.connector.name](this, opts.connector)
+    if (opts.persistence != null) {
+      this.persistence = new Y[opts.persistence.name](this, opts.persistence)
+      this.persistence.retrieveContent()
+    } else {
+      this.persistence = null
+    }
+    this.connected = true
+    this._missingStructs = new Map()
+    this._readyToIntegrate = new Map()
+  }
+  get room () {
+    return this.connector.opts.room
+  }
+  get (name, TypeConstructor) {
+    let id = new RootID(name, TypeConstructor)
+    let type = this.os.get(id)
+    if (type === null) {
+      type = new TypeConstructor()
+      type._id = id
+      type._integrate(this)
+    }
+    return type
+  }
+  disconnect () {
+    if (this.connected) {
+      this.connected = false
+      return this.connector.disconnect()
+    } else {
+      return Promise.resolve()
+    }
+  }
+  reconnect () {
+    if (!this.connected) {
+      this.connected = true
+      return this.connector.reconnect()
+    } else {
+      return Promise.resolve()
+    }
+  }
+  destroy () {
+    this.share = null
+    if (this.connector.destroy != null) {
+      this.connector.destroy()
+    } else {
+      this.connector.disconnect()
+    }
+    this.os.iterate(null, null, function (struct) {
+      struct.destroy()
+    })
+    this.os = null
+    this.ds = null
+    this.ss = null
+  }
+}
+
+Y.extend = function extendYjs () {
+  for (var i = 0; i < arguments.length; i++) {
+    var f = arguments[i]
+    if (typeof f === 'function') {
+      f(Y)
+    } else {
+      throw new Error('Expected a function!')
+    }
+  }
+}
+
+Y.Connector = Connector
+Y.Persisence = Persistence
+Y.Array = YArray
+Y.Map = YMap
+Y.Text = YText
+Y.Xml = YXml
+
+Y.debug = debug
+debug.formatters.Y = formatYjsMessage
+debug.formatters.y = formatYjsMessageType

src/y-memory.js

@@ -1,67 +0,0 @@
-import extendRBTree from './RedBlackTree'
-
-export default function extend (Y) {
-  extendRBTree(Y)
-
-  class Transaction extends Y.Transaction {
-    constructor (store) {
-      super(store)
-      this.store = store
-      this.ss = store.ss
-      this.os = store.os
-      this.ds = store.ds
-    }
-  }
-  var Store = Y.utils.RBTree
-  var BufferedStore = Y.utils.createSmallLookupBuffer(Store)
-
-  class Database extends Y.AbstractDatabase {
-    constructor (y, opts) {
-      super(y, opts)
-      this.os = new BufferedStore()
-      this.ds = new Store()
-      this.ss = new BufferedStore()
-    }
-    logTable () {
-      var self = this
-      self.requestTransaction(function () {
-        console.log('User: ', this.store.y.connector.userId, "==============================") // eslint-disable-line
-        console.log("State Set (SS):", this.getStateSet()) // eslint-disable-line
-        console.log("Operation Store (OS):") // eslint-disable-line
-        this.os.logTable() // eslint-disable-line
-        console.log("Deletion Store (DS):") //eslint-disable-line
-        this.ds.logTable() // eslint-disable-line
-        if (this.store.gc1.length > 0 || this.store.gc2.length > 0) {
-          console.warn('GC1|2 not empty!', this.store.gc1, this.store.gc2)
-        }
-        if (JSON.stringify(this.store.listenersById) !== '{}') {
-          console.warn('listenersById not empty!')
-        }
-        if (JSON.stringify(this.store.listenersByIdExecuteNow) !== '[]') {
-          console.warn('listenersByIdExecuteNow not empty!')
-        }
-        if (this.store.transactionInProgress) {
-          console.warn('Transaction still in progress!')
-        }
-      }, true)
-    }
-    transact (makeGen) {
-      const t = new Transaction(this)
-      try {
-        while (makeGen != null) {
-          makeGen.call(t)
-          makeGen = this.getNextRequest()
-        }
-      } catch (e) {
-        this.y.emit('error', e)
-      }
-    }
-    destroy () {
-      super.destroy()
-      delete this.os
-      delete this.ss
-      delete this.ds
-    }
-  }
-  Y.memory = Database
-}

src/y.js

@@ -1,258 +0,0 @@
-import extendConnector from './Connector.js'
-import extendPersistence from './Persistence.js'
-import extendDatabase from './Database.js'
-import extendTransaction from './Transaction.js'
-import extendStruct from './Struct.js'
-import extendUtils from './Utils.js'
-import extendMemory from './y-memory.js'
-import debug from 'debug'
-import { formatYjsMessage, formatYjsMessageType } from './MessageHandler.js'
-
-extendConnector(Y)
-extendPersistence(Y)
-extendDatabase(Y)
-extendTransaction(Y)
-extendStruct(Y)
-extendUtils(Y)
-extendMemory(Y)
-
-Y.debug = debug
-debug.formatters.Y = formatYjsMessage
-debug.formatters.y = formatYjsMessageType
-
-var requiringModules = {}
-
-Y.requiringModules = requiringModules
-
-Y.extend = function (name, value) {
-  if (arguments.length === 2 && typeof name === 'string') {
-    if (value instanceof Y.utils.CustomTypeDefinition) {
-      Y[name] = value.parseArguments
-    } else {
-      Y[name] = value
-    }
-    if (requiringModules[name] != null) {
-      requiringModules[name].resolve()
-      delete requiringModules[name]
-    }
-  } else {
-    for (var i = 0; i < arguments.length; i++) {
-      var f = arguments[i]
-      if (typeof f === 'function') {
-        f(Y)
-      } else {
-        throw new Error('Expected function!')
-      }
-    }
-  }
-}
-
-Y.requestModules = requestModules
-function requestModules (modules) {
-  var sourceDir
-  if (Y.sourceDir === null) {
-    sourceDir = null
-  } else {
-    sourceDir = Y.sourceDir || '/bower_components'
-  }
-  // determine if this module was compiled for es5 or es6 (y.js vs. y.es6)
-  // if Insert.execute is a Function, then it isnt a generator..
-  // then load the es5(.js) files..
-  var extention = typeof regeneratorRuntime !== 'undefined' ? '.js' : '.es6'
-  var promises = []
-  for (var i = 0; i < modules.length; i++) {
-    var module = modules[i].split('(')[0]
-    var modulename = 'y-' + module.toLowerCase()
-    if (Y[module] == null) {
-      if (requiringModules[module] == null) {
-        // module does not exist
-        if (typeof window !== 'undefined' && window.Y !== 'undefined') {
-          if (sourceDir != null) {
-            var imported = document.createElement('script')
-            imported.src = sourceDir + '/' + modulename + '/' + modulename + extention
-            document.head.appendChild(imported)
-          }
-          let requireModule = {}
-          requiringModules[module] = requireModule
-          requireModule.promise = new Promise(function (resolve) {
-            requireModule.resolve = resolve
-          })
-          promises.push(requireModule.promise)
-        } else {
-          console.info('YJS: Please do not depend on automatic requiring of modules anymore! Extend modules as follows `require(\'y-modulename\')(Y)`')
-          require(modulename)(Y)
-        }
-      } else {
-        promises.push(requiringModules[modules[i]].promise)
-      }
-    }
-  }
-  return Promise.all(promises)
-}
-
-/* ::
-type MemoryOptions = {
-  name: 'memory'
-}
-type IndexedDBOptions = {
-  name: 'indexeddb',
-  namespace: string
-}
-type DbOptions = MemoryOptions | IndexedDBOptions
-
-type WebRTCOptions = {
-  name: 'webrtc',
-  room: string
-}
-type WebsocketsClientOptions = {
-  name: 'websockets-client',
-  room: string
-}
-type ConnectionOptions = WebRTCOptions | WebsocketsClientOptions
-
-type YOptions = {
-  connector: ConnectionOptions,
-  db: DbOptions,
-  types: Array<TypeName>,
-  sourceDir: string,
-  share: {[key: string]: TypeName}
-}
-*/
-
-export default function Y (opts/* :YOptions */) /* :Promise<YConfig> */ {
-  if (opts.hasOwnProperty('sourceDir')) {
-    Y.sourceDir = opts.sourceDir
-  }
-  opts.types = opts.types != null ? opts.types : []
-  var modules = [opts.db.name, opts.connector.name].concat(opts.types)
-  for (var name in opts.share) {
-    modules.push(opts.share[name])
-  }
-  return new Promise(function (resolve, reject) {
-    if (opts == null) reject(new Error('An options object is expected!'))
-    else if (opts.connector == null) reject(new Error('You must specify a connector! (missing connector property)'))
-    else if (opts.connector.name == null) reject(new Error('You must specify connector name! (missing connector.name property)'))
-    else if (opts.db == null) reject(new Error('You must specify a database! (missing db property)'))
-    else if (opts.connector.name == null) reject(new Error('You must specify db name! (missing db.name property)'))
-    else {
-      opts = Y.utils.copyObject(opts)
-      opts.connector = Y.utils.copyObject(opts.connector)
-      opts.db = Y.utils.copyObject(opts.db)
-      opts.share = Y.utils.copyObject(opts.share)
-      Y.requestModules(modules).then(function () {
-        var yconfig = new YConfig(opts)
-        let resolved = false
-        if (opts.timeout != null && opts.timeout >= 0) {
-          setTimeout(function () {
-            if (!resolved) {
-              reject(new Error('Yjs init timeout'))
-              yconfig.destroy()
-            }
-          }, opts.timeout)
-        }
-        if (yconfig.persistence != null) {
-          yconfig.persistence.retrieveContent()
-        }
-        yconfig.db.whenUserIdSet(function () {
-          yconfig.init(function () {
-            resolved = true
-            resolve(yconfig)
-          }, reject)
-        })
-      }).catch(reject)
-    }
-  })
-}
-
-class YConfig extends Y.utils.NamedEventHandler {
-  /* ::
-  db: Y.AbstractDatabase;
-  connector: Y.AbstractConnector;
-  share: {[key: string]: any};
-  options: Object;
-  */
-  constructor (opts, callback) {
-    super()
-    this.options = opts
-    this.db = new Y[opts.db.name](this, opts.db)
-    this.connector = new Y[opts.connector.name](this, opts.connector)
-    if (opts.persistence != null) {
-      this.persistence = new Y[opts.persistence.name](this, opts.persistence)
-    } else {
-      this.persistence = null
-    }
-    this.connected = true
-  }
-  init (callback) {
-    var opts = this.options
-    var share = {}
-    this.share = share
-    this.db.requestTransaction(function requestTransaction () {
-      // create shared object
-      for (var propertyname in opts.share) {
-        var typeConstructor = opts.share[propertyname].split('(')
-        let typeArgs = ''
-        if (typeConstructor.length === 2) {
-          typeArgs = typeConstructor[1].split(')')[0] || ''
-        }
-        var typeName = typeConstructor.splice(0, 1)
-        var type = Y[typeName]
-        var typedef = type.typeDefinition
-        var id = [0xFFFFFF, typedef.struct + '_' + typeName + '_' + propertyname + '_' + typeArgs]
-        let args = Y.utils.parseTypeDefinition(type, typeArgs)
-        share[propertyname] = this.store.initType.call(this, id, args)
-      }
-    })
-    this.db.whenTransactionsFinished()
-      .then(callback)
-  }
-  isConnected () {
-    return this.connector.isSynced
-  }
-  disconnect () {
-    if (this.connected) {
-      this.connected = false
-      return this.connector.disconnect()
-    } else {
-      return Promise.resolve()
-    }
-  }
-  reconnect () {
-    if (!this.connected) {
-      this.connected = true
-      return this.connector.reconnect()
-    } else {
-      return Promise.resolve()
-    }
-  }
-  destroy () {
-    var self = this
-    return this.close().then(function () {
-      if (self.db.deleteDB != null) {
-        return self.db.deleteDB()
-      } else {
-        return Promise.resolve()
-      }
-    }).then(() => {
-      // remove existing event listener
-      super.destroy()
-    })
-  }
-  close () {
-    var self = this
-    this.share = null
-    if (this.connector.destroy != null) {
-      this.connector.destroy()
-    } else {
-      this.connector.disconnect()
-    }
-    return this.db.whenTransactionsFinished().then(function () {
-      self.db.destroyTypes()
-      // make sure to wait for all transactions before destroying the db
-      self.db.requestTransaction(function () {
-        self.db.destroy()
-      })
-      return self.db.whenTransactionsFinished()
-    })
-  }
-}