v13.0.0-28 -- distribution files

.flowconfig

@@ -5,6 +5,8 @@
 
 [include]
 ./src/
+./tests-lib/
+./test/
 
 [libs]
 ./declarations/

examples/chat/index.html

@@ -13,10 +13,6 @@
       <input type="submit" value="Send">
     </form>
     <script src="../../y.js"></script>
-    <script src="../../../y-array/y-array.js"></script>
-    <script src="../../../y-map/dist/y-map.js"></script>
-    <script src="../../../y-text/dist/y-text.js"></script>
-    <script src="../../../y-memory/y-memory.js"></script>
     <script src="../../../y-websockets-client/dist/y-websockets-client.js"></script>
     <script src="./index.js"></script>
   </body>

examples/chat/index.js

@@ -1,73 +1,71 @@
-/* global Y, chat */
+/* global Y */
 
 // initialize a shared object. This function call returns a promise!
-Y({
-  db: {
-    name: 'memory'
-  },
+var y = new Y({
   connector: {
     name: 'websockets-client',
     room: 'chat-example'
-  },
-  sourceDir: '/bower_components',
-  share: {
-    chat: 'Array'
-  }
-}).then(function (y) {
-  window.yChat = y
-  // This functions inserts a message at the specified position in the DOM
-  function appendMessage (message, position) {
-    var p = document.createElement('p')
-    var uname = document.createElement('span')
-    uname.appendChild(document.createTextNode(message.username + ': '))
-    p.appendChild(uname)
-    p.appendChild(document.createTextNode(message.message))
-    document.querySelector('#chat').insertBefore(p, chat.children[position] || null)
-  }
-  // This function makes sure that only 7 messages exist in the chat history.
-  // The rest is deleted
-  function cleanupChat () {
-    if (y.share.chat.length > 7) {
-      y.share.chat.delete(0, y.chat.length - 7)
-    }
-  }
-  // Insert the initial content
-  y.share.chat.toArray().forEach(appendMessage)
-  cleanupChat()
-
-  // whenever content changes, make sure to reflect the changes in the DOM
-  y.share.chat.observe(function (event) {
-    if (event.type === 'insert') {
-      for (let i = 0; i < event.length; i++) {
-        appendMessage(event.values[i], event.index + i)
-      }
-    } else if (event.type === 'delete') {
-      for (let i = 0; i < event.length; i++) {
-        chat.children[event.index].remove()
-      }
-    }
-    // concurrent insertions may result in a history > 7, so cleanup here
-    cleanupChat()
-  })
-  document.querySelector('#chatform').onsubmit = function (event) {
-    // the form is submitted
-    var message = {
-      username: this.querySelector('[name=username]').value,
-      message: this.querySelector('[name=message]').value
-    }
-    if (message.username.length > 0 && message.message.length > 0) {
-      if (y.share.chat.length > 6) {
-        // If we are goint to insert the 8th element, make sure to delete first.
-        y.share.chat.delete(0)
-      }
-      // Here we insert a message in the shared chat type.
-      // This will call the observe function (see line 40)
-      // and reflect the change in the DOM
-      y.share.chat.push([message])
-      this.querySelector('[name=message]').value = ''
-    }
-    // Do not send this form!
-    event.preventDefault()
-    return false
   }
 })
+
+window.yChat = y
+
+let chatprotocol = y.define('chatprotocol', Y.Array)
+
+let chatcontainer = document.querySelector('#chat')
+
+// This functions inserts a message at the specified position in the DOM
+function appendMessage (message, position) {
+  var p = document.createElement('p')
+  var uname = document.createElement('span')
+  uname.appendChild(document.createTextNode(message.username + ': '))
+  p.appendChild(uname)
+  p.appendChild(document.createTextNode(message.message))
+  chatcontainer.insertBefore(p, chatcontainer.children[position] || null)
+}
+// This function makes sure that only 7 messages exist in the chat history.
+// The rest is deleted
+function cleanupChat () {
+  if (chatprotocol.length > 7) {
+    chatprotocol.delete(0, chatprotocol.length - 7)
+  }
+}
+// Insert the initial content
+chatprotocol.toArray().forEach(appendMessage)
+cleanupChat()
+
+// whenever content changes, make sure to reflect the changes in the DOM
+chatprotocol.observe(function (event) {
+  if (event.type === 'insert') {
+    for (let i = 0; i < event.length; i++) {
+      appendMessage(event.values[i], event.index + i)
+    }
+  } else if (event.type === 'delete') {
+    for (let i = 0; i < event.length; i++) {
+      chatcontainer.children[event.index].remove()
+    }
+  }
+  // concurrent insertions may result in a history > 7, so cleanup here
+  cleanupChat()
+})
+document.querySelector('#chatform').onsubmit = function (event) {
+  // the form is submitted
+  var message = {
+    username: this.querySelector('[name=username]').value,
+    message: this.querySelector('[name=message]').value
+  }
+  if (message.username.length > 0 && message.message.length > 0) {
+    if (chatprotocol.length > 6) {
+      // If we are goint to insert the 8th element, make sure to delete first.
+      chatprotocol.delete(0)
+    }
+    // Here we insert a message in the shared chat type.
+    // This will call the observe function (see line 40)
+    // and reflect the change in the DOM
+    chatprotocol.push([message])
+    this.querySelector('[name=message]').value = ''
+  }
+  // Do not send this form!
+  event.preventDefault()
+  return false
+}

examples/html-editor/index.js

@@ -1,21 +1,35 @@
 /* global Y */
 
 // initialize a shared object. This function call returns a promise!
-Y({
-  db: {
-    name: 'memory'
-  },
+let y = new Y({
   connector: {
     name: 'websockets-client',
-    // url: 'http://127.0.0.1:1234',
-    url: 'http://192.168.178.81:1234',
+    url: 'http://127.0.0.1:1234',
     room: 'html-editor-example6'
-  },
-  share: {
-    xml: 'XmlFragment()' // y.share.xml is of type Y.Xml with tagname "p"
+    // maxBufferLength: 100
   }
-}).then(function (y) {
-  window.yXml = y
-  // Bind children of XmlFragment to the document.body
-  window.yXml.share.xml.bindToDom(document.body)
 })
+window.yXml = y
+window.yXmlType = y.define('xml', Y.XmlFragment)
+window.onload = function () {
+  console.log('start!')
+  // Bind children of XmlFragment to the document.body
+  window.yXmlType.bindToDom(document.body)
+}
+window.undoManager = new Y.utils.UndoManager(window.yXmlType, {
+  captureTimeout: 0
+})
+
+document.onkeydown = function interceptUndoRedo (e) {
+  if (e.keyCode === 90 && e.metaKey) {
+    console.log('uidtaren')
+    if (!e.shiftKey) {
+      console.info('Undo!')
+      window.undoManager.undo()
+    } else {
+      console.info('Redo!')
+      window.undoManager.redo()
+    }
+    e.preventDefault()
+  }
+}

examples/indexeddb/index.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+  <head>
+  </head>
+  <body>
+    <div id="codeMirrorContainer"></div>
+    <script src="../bower_components/codemirror/lib/codemirror.js"></script>
+    <script src="../bower_components/codemirror/mode/javascript/javascript.js"></script>
+    <link rel="stylesheet" href="../bower_components/codemirror/lib/codemirror.css">
+    <style>
+      .CodeMirror {
+        position: fixed;
+        top: 0;
+        left: 0;
+        height: 100%;
+        width: 100%;
+      }
+    </style>
+    <script type="module" src="./index.js"></script>
+  </body>
+</html>

examples/indexeddb/index.js

@@ -0,0 +1,24 @@
+/* global Y, CodeMirror */
+
+// initialize a shared object. This function call returns a promise!
+Y({
+  db: {
+    name: 'memory'
+  },
+  connector: {
+    name: 'websockets-client',
+    room: 'codemirror-example'
+  },
+  sourceDir: '/bower_components',
+  share: {
+    codemirror: 'Text' // y.share.codemirror is of type Y.Text
+  }
+}).then(function (y) {
+  window.yCodeMirror = y
+
+  var editor = CodeMirror(document.querySelector('#codeMirrorContainer'), {
+    mode: 'javascript',
+    lineNumbers: true
+  })
+  y.share.codemirror.bindCodeMirror(editor)
+})

examples/textarea/index.html

@@ -4,8 +4,7 @@
   <textarea style="width:80%;" rows=40 id="textfield" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
   <script src="../../y.js"></script>
   <script src="../../../y-array/y-array.js"></script>
-  <script src="../../../y-text/dist/y-text.js"></script>
-  <script src="../../../y-memory/y-memory.js"></script>
+  <script src="../../../y-text/y-text.js"></script>
   <script src="../../../y-websockets-client/y-websockets-client.js"></script>
   <script src="./index.js"></script>
 </body>

examples/textarea/index.js

@@ -1,8 +1,5 @@
 /* global Y */
 
-// eslint-disable-next-line
-let search = new URLSearchParams(location.search)
-
 // initialize a shared object. This function call returns a promise!
 Y({
   db: {
@@ -10,10 +7,9 @@ Y({
   },
   connector: {
     name: 'websockets-client',
-    room: 'Textarea-example',
+    room: 'Textarea-example2',
     // url: '//localhost:1234',
     url: 'https://yjs-v13.herokuapp.com/'
-    // options: { transports: ['websocket'], upgrade: false }
   },
   share: {
     textarea: 'Text'
@@ -24,7 +20,4 @@ Y({
 
   // bind the textarea to a shared text element
   y.share.textarea.bind(document.getElementById('textfield'))
-  // thats it..
-}).catch(() => {
-  console.log('Something went wrong while creating the instance..')
 })

examples/yjs-dist.esm

@@ -1,12 +1,7 @@
 
-import Y from '../src/y.js'
-import yArray from '../../y-array/src/y-array.js'
-import yMap from '../../y-map/src/Map.js'
-import yText from '../../y-text/src/Text.js'
-import yXml from '../../y-xml/src/y-xml.js'
-import yMemory from '../../y-memory/src/y-memory.js'
+import Y from '../src/Y.js'
 import yWebsocketsClient from '../../y-websockets-client/src/y-websockets-client.js'
 
-Y.extend(yArray, yMap, yText, yXml, yMemory, yWebsocketsClient)
+Y.extend(yWebsocketsClient)
 
 export default Y

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yjs",
-  "version": "13.0.0-16",
+  "version": "13.0.0-28",
   "description": "A framework for real-time p2p shared editing on any data",
   "main": "./y.node.js",
   "browser": "./y.js",
@@ -65,6 +65,8 @@
   },
   "dependencies": {
     "debug": "^2.6.8",
-    "utf-8": "^1.0.0"
+    "fast-diff": "^1.1.2",
+    "utf-8": "^1.0.0",
+    "utf8": "^2.1.2"
   }
 }

rollup.browser.js

@@ -1,4 +1,3 @@
-import inject from 'rollup-plugin-inject'
 import babel from 'rollup-plugin-babel'
 import uglify from 'rollup-plugin-uglify'
 import nodeResolve from 'rollup-plugin-node-resolve'
@@ -6,7 +5,7 @@ import commonjs from 'rollup-plugin-commonjs'
 var pkg = require('./package.json')
 
 export default {
-  entry: 'src/y.js',
+  entry: 'src/Y.js',
   moduleName: 'Y',
   format: 'umd',
   plugins: [
@@ -16,13 +15,11 @@ export default {
       browser: true
     }),
     commonjs(),
-    babel({
-      runtimeHelpers: true
-    }),
-    inject({
-      regeneratorRuntime: 'regenerator-runtime'
-    }),
+    babel(),
     uglify({
+      mangle: {
+        except: ['YMap', 'Y', 'YArray', 'YText', 'YXmlFragment', 'YXmlElement', 'YXmlEvent', 'YXmlText', 'YEvent', 'YArrayEvent', 'YMapEvent', 'Type', 'Delete', 'ItemJSON', 'ItemString', 'Item']
+      },
       output: {
         comments: function (node, comment) {
           var text = comment.value

rollup.node.js

@@ -3,9 +3,9 @@ import commonjs from 'rollup-plugin-commonjs'
 var pkg = require('./package.json')
 
 export default {
-  entry: 'src/y.js',
+  entry: 'src/y-dist.cjs.js',
   moduleName: 'Y',
-  format: 'umd',
+  format: 'cjs',
   plugins: [
     nodeResolve({
       main: true,

rollup.test.js

@@ -3,7 +3,7 @@ import commonjs from 'rollup-plugin-commonjs'
 import multiEntry from 'rollup-plugin-multi-entry'
 
 export default {
-  entry: 'test/y-xml.tests.js',
+  entry: 'test/y-map.tests.js',
   moduleName: 'y-tests',
   format: 'umd',
   plugins: [

src/Binary/Decoder.js

@@ -0,0 +1,120 @@
+import utf8 from 'utf-8'
+import ID from '../Util/ID.js'
+import { default as RootID, RootFakeUserID } from '../Util/RootID.js'
+
+export default class BinaryDecoder {
+  constructor (buffer) {
+    if (buffer instanceof ArrayBuffer) {
+      this.uint8arr = new Uint8Array(buffer)
+    } else if (buffer instanceof Uint8Array || (typeof Buffer !== 'undefined' && buffer instanceof Buffer)) {
+      this.uint8arr = buffer
+    } else {
+      throw new Error('Expected an ArrayBuffer or Uint8Array!')
+    }
+    this.pos = 0
+  }
+  /**
+   * Clone this decoder instance
+   * Optionally set a new position parameter
+   */
+  clone (newPos = this.pos) {
+    let decoder = new BinaryDecoder(this.uint8arr)
+    decoder.pos = newPos
+    return decoder
+  }
+  /**
+   * Number of bytes
+   */
+  get length () {
+    return this.uint8arr.length
+  }
+  /**
+   * Skip one byte, jump to the next position
+   */
+  skip8 () {
+    this.pos++
+  }
+  /**
+   * Read one byte as unsigned integer
+   */
+  readUint8 () {
+    return this.uint8arr[this.pos++]
+  }
+  /**
+   * Read 4 bytes as unsigned integer
+   */
+  readUint32 () {
+    let uint =
+      this.uint8arr[this.pos] +
+      (this.uint8arr[this.pos + 1] << 8) +
+      (this.uint8arr[this.pos + 2] << 16) +
+      (this.uint8arr[this.pos + 3] << 24)
+    this.pos += 4
+    return uint
+  }
+  /**
+   * Look ahead without incrementing position
+   * to the next byte and read it as unsigned integer
+   */
+  peekUint8 () {
+    return this.uint8arr[this.pos]
+  }
+  /**
+   * Read unsigned integer (32bit) with variable length
+   * 1/8th of the storage is used as encoding overhead
+   *  - numbers < 2^7 is stored in one byte
+   *  - numbers < 2^14 is stored in two bytes
+   *  ..
+   */
+  readVarUint () {
+    let num = 0
+    let len = 0
+    while (true) {
+      let r = this.uint8arr[this.pos++]
+      num = num | ((r & 0b1111111) << len)
+      len += 7
+      if (r < 1 << 7) {
+        return num >>> 0 // return unsigned number!
+      }
+      if (len > 35) {
+        throw new Error('Integer out of range!')
+      }
+    }
+  }
+  /**
+   * Read string of variable length
+   * - varUint is used to store the length of the string
+   */
+  readVarString () {
+    let len = this.readVarUint()
+    let bytes = new Array(len)
+    for (let i = 0; i < len; i++) {
+      bytes[i] = this.uint8arr[this.pos++]
+    }
+    return utf8.getStringFromBytes(bytes)
+  }
+  /**
+   *  Look ahead and read varString without incrementing position
+   */
+  peekVarString () {
+    let pos = this.pos
+    let s = this.readVarString()
+    this.pos = pos
+    return s
+  }
+  /**
+   * Read ID
+   * - If first varUint read is 0xFFFFFF a RootID is returned
+   * - Otherwise an ID is returned
+   */
+  readID () {
+    let user = this.readVarUint()
+    if (user === RootFakeUserID) {
+      // read property name and type id
+      const rid = new RootID(this.readVarString(), null)
+      rid.type = this.readVarUint()
+      return rid
+    }
+    return new ID(user, this.readVarUint())
+  }
+}

src/Binary/Encoder.js

@@ -0,0 +1,83 @@
+import utf8 from 'utf-8'
+import { RootFakeUserID } from '../Util/RootID.js'
+
+const bits7 = 0b1111111
+const bits8 = 0b11111111
+
+export default class BinaryEncoder {
+  constructor () {
+    // TODO: implement chained Uint8Array buffers instead of Array buffer
+    this.data = []
+  }
+
+  get length () {
+    return this.data.length
+  }
+
+  get pos () {
+    return this.data.length
+  }
+
+  createBuffer () {
+    return Uint8Array.from(this.data).buffer
+  }
+
+  writeUint8 (num) {
+    this.data.push(num & bits8)
+  }
+
+  setUint8 (pos, num) {
+    this.data[pos] = num & bits8
+  }
+
+  writeUint16 (num) {
+    this.data.push(num & bits8, (num >>> 8) & bits8)
+  }
+
+  setUint16 (pos, num) {
+    this.data[pos] = num & bits8
+    this.data[pos + 1] = (num >>> 8) & bits8
+  }
+
+  writeUint32 (num) {
+    for (let i = 0; i < 4; i++) {
+      this.data.push(num & bits8)
+      num >>>= 8
+    }
+  }
+
+  setUint32 (pos, num) {
+    for (let i = 0; i < 4; i++) {
+      this.data[pos + i] = num & bits8
+      num >>>= 8
+    }
+  }
+
+  writeVarUint (num) {
+    while (num >= 0b10000000) {
+      this.data.push(0b10000000 | (bits7 & num))
+      num >>>= 7
+    }
+    this.data.push(bits7 & num)
+  }
+
+  writeVarString (str) {
+    let bytes = utf8.setBytesFromString(str)
+    let len = bytes.length
+    this.writeVarUint(len)
+    for (let i = 0; i < len; i++) {
+      this.data.push(bytes[i])
+    }
+  }
+
+  writeID (id) {
+    const user = id.user
+    this.writeVarUint(user)
+    if (user !== RootFakeUserID) {
+      this.writeVarUint(id.clock)
+    } else {
+      this.writeVarString(id.name)
+      this.writeVarUint(id.type)
+    }
+  }
+}

src/Connector.js

@@ -1,380 +1,294 @@
-import { BinaryEncoder, BinaryDecoder } from './Encoding.js'
-import { sendSyncStep1, computeMessageSyncStep1, computeMessageSyncStep2, computeMessageUpdate } from './MessageHandler.js'
+import BinaryEncoder from './Binary/Encoder.js'
+import BinaryDecoder from './Binary/Decoder.js'
 
-export default function extendConnector (Y/* :any */) {
-  class AbstractConnector {
-    /*
-      opts contains the following information:
-       role : String Role of this client ("master" or "slave")
-    */
-    constructor (y, opts) {
-      this.y = y
-      if (opts == null) {
-        opts = {}
-      }
-      this.opts = opts
-      // Prefer to receive untransformed operations. This does only work if
-      // this client receives operations from only one other client.
-      // In particular, this does not work with y-webrtc.
-      // It will work with y-websockets-client
-      this.preferUntransformed = opts.preferUntransformed || false
-      if (opts.role == null || opts.role === 'master') {
-        this.role = 'master'
-      } else if (opts.role === 'slave') {
-        this.role = 'slave'
-      } else {
-        throw new Error("Role must be either 'master' or 'slave'!")
-      }
-      this.log = Y.debug('y:connector')
-      this.logMessage = Y.debug('y:connector-message')
-      this.y.db.forwardAppliedOperations = opts.forwardAppliedOperations || false
-      this.role = opts.role
-      this.connections = new Map()
-      this.isSynced = false
-      this.userEventListeners = []
-      this.whenSyncedListeners = []
-      this.currentSyncTarget = null
-      this.debug = opts.debug === true
-      this.broadcastOpBuffer = []
-      this.protocolVersion = 11
-      this.authInfo = opts.auth || null
-      this.checkAuth = opts.checkAuth || function () { return Promise.resolve('write') } // default is everyone has write access
-      if (opts.generateUserId !== false) {
-        this.setUserId(Y.utils.generateUserId())
-      }
-    }
+import { sendSyncStep1, readSyncStep1 } from './MessageHandler/syncStep1.js'
+import { readSyncStep2 } from './MessageHandler/syncStep2.js'
+import { integrateRemoteStructs } from './MessageHandler/integrateRemoteStructs.js'
 
-    reconnect () {
-      this.log('reconnecting..')
-      return this.y.db.startGarbageCollector()
-    }
+import debug from 'debug'
 
-    disconnect () {
-      this.log('discronnecting..')
-      this.connections = new Map()
-      this.isSynced = false
-      this.currentSyncTarget = null
-      this.whenSyncedListeners = []
-      this.y.db.stopGarbageCollector()
-      return this.y.db.whenTransactionsFinished()
+export default class AbstractConnector {
+  constructor (y, opts) {
+    this.y = y
+    this.opts = opts
+    if (opts.role == null || opts.role === 'master') {
+      this.role = 'master'
+    } else if (opts.role === 'slave') {
+      this.role = 'slave'
+    } else {
+      throw new Error("Role must be either 'master' or 'slave'!")
     }
+    this.log = debug('y:connector')
+    this.logMessage = debug('y:connector-message')
+    this._forwardAppliedStructs = opts.forwardAppliedOperations || false // TODO: rename
+    this.role = opts.role
+    this.connections = new Map()
+    this.isSynced = false
+    this.userEventListeners = []
+    this.whenSyncedListeners = []
+    this.currentSyncTarget = null
+    this.debug = opts.debug === true
+    this.broadcastBuffer = new BinaryEncoder()
+    this.broadcastBufferSize = 0
+    this.protocolVersion = 11
+    this.authInfo = opts.auth || null
+    this.checkAuth = opts.checkAuth || function () { return Promise.resolve('write') } // default is everyone has write access
+    if (opts.maxBufferLength == null) {
+      this.maxBufferLength = -1
+    } else {
+      this.maxBufferLength = opts.maxBufferLength
+    }
+  }
 
-    repair () {
-      this.log('Repairing the state of Yjs. This can happen if messages get lost, and Yjs detects that something is wrong. If this happens often, please report an issue here: https://github.com/y-js/yjs/issues')
-      this.isSynced = false
-      this.connections.forEach((user, userId) => {
-        user.isSynced = false
-        this._syncWithUser(userId)
-      })
-    }
+  reconnect () {
+    this.log('reconnecting..')
+  }
 
-    setUserId (userId) {
-      if (this.userId == null) {
-        if (!Number.isInteger(userId)) {
-          let err = new Error('UserId must be an integer!')
-          this.y.emit('error', err)
-          throw err
-        }
-        this.log('Set userId to "%s"', userId)
-        this.userId = userId
-        return this.y.db.setUserId(userId)
-      } else {
-        return null
-      }
-    }
+  disconnect () {
+    this.log('discronnecting..')
+    this.connections = new Map()
+    this.isSynced = false
+    this.currentSyncTarget = null
+    this.whenSyncedListeners = []
+    return Promise.resolve()
+  }
 
-    onUserEvent (f) {
-      this.userEventListeners.push(f)
-    }
+  onUserEvent (f) {
+    this.userEventListeners.push(f)
+  }
 
-    removeUserEventListener (f) {
-      this.userEventListeners = this.userEventListeners.filter(g => f !== g)
-    }
+  removeUserEventListener (f) {
+    this.userEventListeners = this.userEventListeners.filter(g => f !== g)
+  }
 
-    userLeft (user) {
-      if (this.connections.has(user)) {
-        this.log('%s: User left %s', this.userId, user)
-        this.connections.delete(user)
-        // check if isSynced event can be sent now
-        this._setSyncedWith(null)
-        for (var f of this.userEventListeners) {
-          f({
-            action: 'userLeft',
-            user: user
-          })
-        }
-      }
-    }
-    userJoined (user, role, auth) {
-      if (role == null) {
-        throw new Error('You must specify the role of the joined user!')
-      }
-      if (this.connections.has(user)) {
-        throw new Error('This user already joined!')
-      }
-      this.log('%s: User joined %s', this.userId, user)
-      this.connections.set(user, {
-        uid: user,
-        isSynced: false,
-        role: role,
-        processAfterAuth: [],
-        auth: auth || null,
-        receivedSyncStep2: false
-      })
-      let defer = {}
-      defer.promise = new Promise(function (resolve) { defer.resolve = resolve })
-      this.connections.get(user).syncStep2 = defer
+  userLeft (user) {
+    if (this.connections.has(user)) {
+      this.log('%s: User left %s', this.y.userID, user)
+      this.connections.delete(user)
+      // check if isSynced event can be sent now
+      this._setSyncedWith(null)
       for (var f of this.userEventListeners) {
         f({
-          action: 'userJoined',
-          user: user,
-          role: role
+          action: 'userLeft',
+          user: user
         })
       }
-      this._syncWithUser(user)
-    }
-    // Execute a function _when_ we are connected.
-    // If not connected, wait until connected
-    whenSynced (f) {
-      if (this.isSynced) {
-        f()
-      } else {
-        this.whenSyncedListeners.push(f)
-      }
-    }
-    _syncWithUser (userid) {
-      if (this.role === 'slave') {
-        return // "The current sync has not finished or this is controlled by a master!"
-      }
-      sendSyncStep1(this, userid)
-    }
-    _fireIsSyncedListeners () {
-      this.y.db.whenTransactionsFinished().then(() => {
-        if (!this.isSynced) {
-          this.isSynced = true
-          // It is safer to remove this!
-          // TODO: remove: yield * this.garbageCollectAfterSync()
-          // call whensynced listeners
-          for (var f of this.whenSyncedListeners) {
-            f()
-          }
-          this.whenSyncedListeners = []
-        }
-      })
-    }
-    send (uid, buffer) {
-      if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
-        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages')
-      }
-      this.log('%s: Send \'%y\' to %s', this.userId, buffer, uid)
-      this.logMessage('Message: %Y', buffer)
-    }
-    broadcast (buffer) {
-      if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
-        throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - please don\'t use this method to send custom messages')
-      }
-      this.log('%s: Broadcast \'%y\'', this.userId, buffer)
-      this.logMessage('Message: %Y', buffer)
-    }
-    /*
-      Buffer operations, and broadcast them when ready.
-    */
-    broadcastOps (ops) {
-      ops = ops.map(function (op) {
-        return Y.Struct[op.struct].encode(op)
-      })
-      var self = this
-      function broadcastOperations () {
-        if (self.broadcastOpBuffer.length > 0) {
-          let encoder = new BinaryEncoder()
-          encoder.writeVarString(self.opts.room)
-          encoder.writeVarString('update')
-          let ops = self.broadcastOpBuffer
-          self.broadcastOpBuffer = []
-          let length = ops.length
-          encoder.writeUint32(length)
-          for (var i = 0; i < length; i++) {
-            let op = ops[i]
-            Y.Struct[op.struct].binaryEncode(encoder, op)
-          }
-          self.broadcast(encoder.createBuffer())
-        }
-      }
-      if (this.broadcastOpBuffer.length === 0) {
-        this.broadcastOpBuffer = ops
-        this.y.db.whenTransactionsFinished().then(broadcastOperations)
-      } else {
-        this.broadcastOpBuffer = this.broadcastOpBuffer.concat(ops)
-      }
-    }
-    /*
-      You received a raw message, and you know that it is intended for Yjs. Then call this function.
-    */
-    receiveMessage (sender, buffer, skipAuth) {
-      skipAuth = skipAuth || false
-      if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
-        return Promise.reject(new Error('Expected Message to be an ArrayBuffer or Uint8Array!'))
-      }
-      if (sender === this.userId) {
-        return Promise.resolve()
-      }
-      let decoder = new BinaryDecoder(buffer)
-      let encoder = new BinaryEncoder()
-      let roomname = decoder.readVarString() // read room name
-      encoder.writeVarString(roomname)
-      let messageType = decoder.readVarString()
-      let senderConn = this.connections.get(sender)
-
-      this.log('%s: Receive \'%s\' from %s', this.userId, messageType, sender)
-      this.logMessage('Message: %Y', buffer)
-
-      if (senderConn == null && !skipAuth) {
-        throw new Error('Received message from unknown peer!')
-      }
-
-      if (messageType === 'sync step 1' || messageType === 'sync step 2') {
-        let auth = decoder.readVarUint()
-        if (senderConn.auth == null) {
-          senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender])
-          // check auth
-          return this.checkAuth(auth, this.y, sender).then(authPermissions => {
-            if (senderConn.auth == null) {
-              senderConn.auth = authPermissions
-              this.y.emit('userAuthenticated', {
-                user: senderConn.uid,
-                auth: authPermissions
-              })
-            }
-            let messages = senderConn.processAfterAuth
-            senderConn.processAfterAuth = []
-
-            return messages.reduce((p, m) =>
-              p.then(() => this.computeMessage(m[0], m[1], m[2], m[3], m[4]))
-            , Promise.resolve())
-          })
-        }
-      }
-      if (skipAuth || senderConn.auth != null) {
-        return this.computeMessage(messageType, senderConn, decoder, encoder, sender, skipAuth)
-      } else {
-        senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender, false])
-      }
-    }
-
-    computeMessage (messageType, senderConn, decoder, encoder, sender, skipAuth) {
-      if (messageType === 'sync step 1' && (senderConn.auth === 'write' || senderConn.auth === 'read')) {
-        // cannot wait for sync step 1 to finish, because we may wait for sync step 2 in sync step 1 (->lock)
-        computeMessageSyncStep1(decoder, encoder, this, senderConn, sender)
-        return this.y.db.whenTransactionsFinished()
-      } else if (messageType === 'sync step 2' && senderConn.auth === 'write') {
-        return computeMessageSyncStep2(decoder, encoder, this, senderConn, sender)
-      } else if (messageType === 'update' && (skipAuth || senderConn.auth === 'write')) {
-        return computeMessageUpdate(decoder, encoder, this, senderConn, sender)
-      } else {
-        return Promise.reject(new Error('Unable to receive message'))
-      }
-    }
-
-    _setSyncedWith (user) {
-      if (user != null) {
-        this.connections.get(user).isSynced = true
-      }
-      let conns = Array.from(this.connections.values())
-      if (conns.length > 0 && conns.every(u => u.isSynced)) {
-        this._fireIsSyncedListeners()
-      }
-    }
-
-    /*
-      Currently, the HB encodes operations as JSON. For the moment I want to keep it
-      that way. Maybe we support encoding in the HB as XML in the future, but for now I don't want
-      too much overhead. Y is very likely to get changed a lot in the future
-
-      Because we don't want to encode JSON as string (with character escaping, wich makes it pretty much unreadable)
-      we encode the JSON as XML.
-
-      When the HB support encoding as XML, the format should look pretty much like this.
-
-      does not support primitive values as array elements
-      expects an ltx (less than xml) object
-    */
-    parseMessageFromXml (m/* :any */) {
-      function parseArray (node) {
-        for (var n of node.children) {
-          if (n.getAttribute('isArray') === 'true') {
-            return parseArray(n)
-          } else {
-            return parseObject(n)
-          }
-        }
-      }
-      function parseObject (node/* :any */) {
-        var json = {}
-        for (var attrName in node.attrs) {
-          var value = node.attrs[attrName]
-          var int = parseInt(value, 10)
-          if (isNaN(int) || ('' + int) !== value) {
-            json[attrName] = value
-          } else {
-            json[attrName] = int
-          }
-        }
-        for (var n/* :any */ in node.children) {
-          var name = n.name
-          if (n.getAttribute('isArray') === 'true') {
-            json[name] = parseArray(n)
-          } else {
-            json[name] = parseObject(n)
-          }
-        }
-        return json
-      }
-      parseObject(m)
-    }
-    /*
-      encode message in xml
-      we use string because Strophe only accepts an "xml-string"..
-      So {a:4,b:{c:5}} will look like
-      <y a="4">
-        <b c="5"></b>
-      </y>
-      m - ltx element
-      json - Object
-    */
-    encodeMessageToXml (msg, obj) {
-      // attributes is optional
-      function encodeObject (m, json) {
-        for (var name in json) {
-          var value = json[name]
-          if (name == null) {
-            // nop
-          } else if (value.constructor === Object) {
-            encodeObject(m.c(name), value)
-          } else if (value.constructor === Array) {
-            encodeArray(m.c(name), value)
-          } else {
-            m.setAttribute(name, value)
-          }
-        }
-      }
-      function encodeArray (m, array) {
-        m.setAttribute('isArray', 'true')
-        for (var e of array) {
-          if (e.constructor === Object) {
-            encodeObject(m.c('array-element'), e)
-          } else {
-            encodeArray(m.c('array-element'), e)
-          }
-        }
-      }
-      if (obj.constructor === Object) {
-        encodeObject(msg.c('y', { xmlns: 'http://y.ninja/connector-stanza' }), obj)
-      } else if (obj.constructor === Array) {
-        encodeArray(msg.c('y', { xmlns: 'http://y.ninja/connector-stanza' }), obj)
-      } else {
-        throw new Error("I can't encode this json!")
-      }
     }
   }
-  Y.AbstractConnector = AbstractConnector
+
+  userJoined (user, role, auth) {
+    if (role == null) {
+      throw new Error('You must specify the role of the joined user!')
+    }
+    if (this.connections.has(user)) {
+      throw new Error('This user already joined!')
+    }
+    this.log('%s: User joined %s', this.y.userID, user)
+    this.connections.set(user, {
+      uid: user,
+      isSynced: false,
+      role: role,
+      processAfterAuth: [],
+      processAfterSync: [],
+      auth: auth || null,
+      receivedSyncStep2: false
+    })
+    let defer = {}
+    defer.promise = new Promise(function (resolve) { defer.resolve = resolve })
+    this.connections.get(user).syncStep2 = defer
+    for (var f of this.userEventListeners) {
+      f({
+        action: 'userJoined',
+        user: user,
+        role: role
+      })
+    }
+    this._syncWithUser(user)
+  }
+
+  // Execute a function _when_ we are connected.
+  // If not connected, wait until connected
+  whenSynced (f) {
+    if (this.isSynced) {
+      f()
+    } else {
+      this.whenSyncedListeners.push(f)
+    }
+  }
+
+  _syncWithUser (userID) {
+    if (this.role === 'slave') {
+      return // "The current sync has not finished or this is controlled by a master!"
+    }
+    sendSyncStep1(this, userID)
+  }
+
+  _fireIsSyncedListeners () {
+    if (!this.isSynced) {
+      this.isSynced = true
+      // It is safer to remove this!
+      // call whensynced listeners
+      for (var f of this.whenSyncedListeners) {
+        f()
+      }
+      this.whenSyncedListeners = []
+      this.y.emit('synced')
+    }
+  }
+
+  send (uid, buffer) {
+    const y = this.y
+    if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
+      throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - don\'t use this method to send custom messages')
+    }
+    this.log('User%s to User%s: Send \'%y\'', y.userID, uid, buffer)
+    this.logMessage('User%s to User%s: Send %Y', y.userID, uid, [y, buffer])
+  }
+
+  broadcast (buffer) {
+    const y = this.y
+    if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
+      throw new Error('Expected Message to be an ArrayBuffer or Uint8Array - don\'t use this method to send custom messages')
+    }
+    this.log('User%s: Broadcast \'%y\'', y.userID, buffer)
+    this.logMessage('User%s: Broadcast: %Y', y.userID, [y, buffer])
+  }
+
+  /*
+    Buffer operations, and broadcast them when ready.
+  */
+  broadcastStruct (struct) {
+    const firstContent = this.broadcastBuffer.length === 0
+    if (firstContent) {
+      this.broadcastBuffer.writeVarString(this.y.room)
+      this.broadcastBuffer.writeVarString('update')
+      this.broadcastBufferSize = 0
+      this.broadcastBufferSizePos = this.broadcastBuffer.pos
+      this.broadcastBuffer.writeUint32(0)
+    }
+    this.broadcastBufferSize++
+    struct._toBinary(this.broadcastBuffer)
+    if (this.maxBufferLength > 0 && this.broadcastBuffer.length > this.maxBufferLength) {
+      // it is necessary to send the buffer now
+      // cache the buffer and check if server is responsive
+      const buffer = this.broadcastBuffer
+      buffer.setUint32(this.broadcastBufferSizePos, this.broadcastBufferSize)
+      this.broadcastBuffer = new BinaryEncoder()
+      this.whenRemoteResponsive().then(() => {
+        this.broadcast(buffer.createBuffer())
+      })
+    } else if (firstContent) {
+      // send the buffer when all transactions are finished
+      // (or buffer exceeds maxBufferLength)
+      setTimeout(() => {
+        if (this.broadcastBuffer.length > 0) {
+          const buffer = this.broadcastBuffer
+          buffer.setUint32(this.broadcastBufferSizePos, this.broadcastBufferSize)
+          this.broadcast(buffer.createBuffer())
+          this.broadcastBuffer = new BinaryEncoder()
+        }
+      }, 0)
+    }
+  }
+
+  /*
+   * Somehow check the responsiveness of the remote clients/server
+   * Default behavior:
+   *   Wait 100ms before broadcasting the next batch of operations
+   *
+   * Only used when maxBufferLength is set
+   *
+   */
+  whenRemoteResponsive () {
+    return new Promise(function (resolve) {
+      setTimeout(resolve, 100)
+    })
+  }
+
+  /*
+    You received a raw message, and you know that it is intended for Yjs. Then call this function.
+  */
+  receiveMessage (sender, buffer, skipAuth) {
+    const y = this.y
+    const userID = y.userID
+    skipAuth = skipAuth || false
+    if (!(buffer instanceof ArrayBuffer || buffer instanceof Uint8Array)) {
+      return Promise.reject(new Error('Expected Message to be an ArrayBuffer or Uint8Array!'))
+    }
+    if (sender === userID) {
+      return Promise.resolve()
+    }
+    let decoder = new BinaryDecoder(buffer)
+    let encoder = new BinaryEncoder()
+    let roomname = decoder.readVarString() // read room name
+    encoder.writeVarString(roomname)
+    let messageType = decoder.readVarString()
+    let senderConn = this.connections.get(sender)
+    this.log('User%s from User%s: Receive \'%s\'', userID, sender, messageType)
+    this.logMessage('User%s from User%s: Receive %Y', userID, sender, [y, buffer])
+    if (senderConn == null && !skipAuth) {
+      throw new Error('Received message from unknown peer!')
+    }
+    if (messageType === 'sync step 1' || messageType === 'sync step 2') {
+      let auth = decoder.readVarUint()
+      if (senderConn.auth == null) {
+        senderConn.processAfterAuth.push([messageType, senderConn, decoder, encoder, sender])
+        // check auth
+        return this.checkAuth(auth, y, sender).then(authPermissions => {
+          if (senderConn.auth == null) {
+            senderConn.auth = authPermissions
+            y.emit('userAuthenticated', {
+              user: senderConn.uid,
+              auth: authPermissions
+            })
+          }
+          let messages = senderConn.processAfterAuth
+          senderConn.processAfterAuth = []
+
+          messages.forEach(m =>
+            this.computeMessage(m[0], m[1], m[2], m[3], m[4])
+          )
+        })
+      }
+    }
+    if ((skipAuth || senderConn.auth != null) && (messageType !== 'update' || senderConn.isSynced)) {
+      this.computeMessage(messageType, senderConn, decoder, encoder, sender, skipAuth)
+    } else {
+      senderConn.processAfterSync.push([messageType, senderConn, decoder, encoder, sender, false])
+    }
+  }
+
+  computeMessage (messageType, senderConn, decoder, encoder, sender, skipAuth) {
+    if (messageType === 'sync step 1' && (senderConn.auth === 'write' || senderConn.auth === 'read')) {
+      // cannot wait for sync step 1 to finish, because we may wait for sync step 2 in sync step 1 (->lock)
+      readSyncStep1(decoder, encoder, this.y, senderConn, sender)
+    } else {
+      const y = this.y
+      y.transact(function () {
+        if (messageType === 'sync step 2' && senderConn.auth === 'write') {
+          readSyncStep2(decoder, encoder, y, senderConn, sender)
+        } else if (messageType === 'update' && (skipAuth || senderConn.auth === 'write')) {
+          integrateRemoteStructs(decoder, encoder, y, senderConn, sender)
+        } else {
+          throw new Error('Unable to receive message')
+        }
+      }, true)
+    }
+  }
+
+  _setSyncedWith (user) {
+    if (user != null) {
+      const userConn = this.connections.get(user)
+      userConn.isSynced = true
+      const messages = userConn.processAfterSync
+      userConn.processAfterSync = []
+      messages.forEach(m => {
+        this.computeMessage(m[0], m[1], m[2], m[3], m[4])
+      })
+    }
+    const conns = Array.from(this.connections.values())
+    if (conns.length > 0 && conns.every(u => u.isSynced)) {
+      this._fireIsSyncedListeners()
+    }
+  }
 }

src/Database.js

@@ -1,607 +0,0 @@
-/* @flow */
-'use strict'
-
-export default function extendDatabase (Y /* :any */) {
-  /*
-    Partial definition of an OperationStore.
-    TODO: name it Database, operation store only holds operations.
-
-    A database definition must alse define the following methods:
-    * logTable() (optional)
-      - show relevant information information in a table
-    * requestTransaction(makeGen)
-      - request a transaction
-    * destroy()
-      - destroy the database
-  */
-  class AbstractDatabase {
-    /* ::
-    y: YConfig;
-    forwardAppliedOperations: boolean;
-    listenersById: Object;
-    listenersByIdExecuteNow: Array<Object>;
-    listenersByIdRequestPending: boolean;
-    initializedTypes: Object;
-    whenUserIdSetListener: ?Function;
-    waitingTransactions: Array<Transaction>;
-    transactionInProgress: boolean;
-    executeOrder: Array<Object>;
-    gc1: Array<Struct>;
-    gc2: Array<Struct>;
-    gcTimeout: number;
-    gcInterval: any;
-    garbageCollect: Function;
-    executeOrder: Array<any>; // for debugging only
-    userId: UserId;
-    opClock: number;
-    transactionsFinished: ?{promise: Promise, resolve: any};
-    transact: (x: ?Generator) => any;
-    */
-    constructor (y, opts) {
-      this.y = y
-      opts.gc = opts.gc === true
-      this.dbOpts = opts
-      var os = this
-      this.userId = null
-      var resolve_
-      this.userIdPromise = new Promise(function (resolve) {
-        resolve_ = resolve
-      })
-      this.userIdPromise.resolve = resolve_
-      // whether to broadcast all applied operations (insert & delete hook)
-      this.forwardAppliedOperations = false
-      // E.g. this.listenersById[id] : Array<Listener>
-      this.listenersById = {}
-      // Execute the next time a transaction is requested
-      this.listenersByIdExecuteNow = []
-      // A transaction is requested
-      this.listenersByIdRequestPending = false
-      /* To make things more clear, the following naming conventions:
-         * ls : we put this.listenersById on ls
-         * l : Array<Listener>
-         * id : Id (can't use as property name)
-         * sid : String (converted from id via JSON.stringify
-                         so we can use it as a property name)
-
-        Always remember to first overwrite
-        a property before you iterate over it!
-      */
-      // TODO: Use ES7 Weak Maps. This way types that are no longer user,
-      // wont be kept in memory.
-      this.initializedTypes = {}
-      this.waitingTransactions = []
-      this.transactionInProgress = false
-      this.transactionIsFlushed = false
-      if (typeof YConcurrencyTestingMode !== 'undefined') {
-        this.executeOrder = []
-      }
-      this.gc1 = [] // first stage
-      this.gc2 = [] // second stage -> after that, remove the op
-
-      function garbageCollect () {
-        return os.whenTransactionsFinished().then(function () {
-          if (os.gcTimeout > 0 && (os.gc1.length > 0 || os.gc2.length > 0)) {
-            if (!os.y.connector.isSynced) {
-              console.warn('gc should be empty when not synced!')
-            }
-            return new Promise((resolve) => {
-              os.requestTransaction(function * () {
-                if (os.y.connector != null && os.y.connector.isSynced) {
-                  for (var i = 0; i < os.gc2.length; i++) {
-                    var oid = os.gc2[i]
-                    yield * this.garbageCollectOperation(oid)
-                  }
-                  os.gc2 = os.gc1
-                  os.gc1 = []
-                }
-                // TODO: Use setInterval here instead (when garbageCollect is called several times there will be several timeouts..)
-                if (os.gcTimeout > 0) {
-                  os.gcInterval = setTimeout(garbageCollect, os.gcTimeout)
-                }
-                resolve()
-              })
-            })
-          } else {
-            // TODO: see above
-            if (os.gcTimeout > 0) {
-              os.gcInterval = setTimeout(garbageCollect, os.gcTimeout)
-            }
-            return Promise.resolve()
-          }
-        })
-      }
-      this.garbageCollect = garbageCollect
-      this.startGarbageCollector()
-
-      this.repairCheckInterval = !opts.repairCheckInterval ? 6000 : opts.repairCheckInterval
-      this.opsReceivedTimestamp = new Date()
-      this.startRepairCheck()
-    }
-    startGarbageCollector () {
-      this.gc = this.dbOpts.gc
-      if (this.gc) {
-        this.gcTimeout = !this.dbOpts.gcTimeout ? 30000 : this.dbOpts.gcTimeout
-      } else {
-        this.gcTimeout = -1
-      }
-      if (this.gcTimeout > 0) {
-        this.garbageCollect()
-      }
-    }
-    startRepairCheck () {
-      var os = this
-      if (this.repairCheckInterval > 0) {
-        this.repairCheckIntervalHandler = setInterval(function repairOnMissingOperations () {
-          /*
-            Case 1. No ops have been received in a while (new Date() - os.opsReceivedTimestamp > os.repairCheckInterval)
-              - 1.1 os.listenersById is empty. Then the state was correct the whole time. -> Nothing to do (nor to update)
-              - 1.2 os.listenersById is not empty.
-                      * Then the state was incorrect for at least {os.repairCheckInterval} seconds.
-                      * -> Remove everything in os.listenersById and sync again (connector.repair())
-            Case 2. An op has been received in the last {os.repairCheckInterval } seconds.
-                    It is not yet necessary to check for faulty behavior. Everything can still resolve itself. Wait for more messages.
-                    If nothing was received for a while and os.listenersById is still not emty, we are in case 1.2
-                    -> Do nothing
-
-            Baseline here is: we really only have to catch case 1.2..
-          */
-          if (
-            new Date() - os.opsReceivedTimestamp > os.repairCheckInterval &&
-            Object.keys(os.listenersById).length > 0 // os.listenersById is not empty
-          ) {
-            // haven't received operations for over {os.repairCheckInterval} seconds, resend state vector
-            os.listenersById = {}
-            os.opsReceivedTimestamp = new Date() // update so you don't send repair several times in a row
-            os.y.connector.repair()
-          }
-        }, this.repairCheckInterval)
-      }
-    }
-    stopRepairCheck () {
-      clearInterval(this.repairCheckIntervalHandler)
-    }
-    queueGarbageCollector (id) {
-      if (this.y.connector.isSynced && this.gc) {
-        this.gc1.push(id)
-      }
-    }
-    emptyGarbageCollector () {
-      return new Promise(resolve => {
-        var check = () => {
-          if (this.gc1.length > 0 || this.gc2.length > 0) {
-            this.garbageCollect().then(check)
-          } else {
-            resolve()
-          }
-        }
-        setTimeout(check, 0)
-      })
-    }
-    addToDebug () {
-      if (typeof YConcurrencyTestingMode !== 'undefined') {
-        var command /* :string */ = Array.prototype.map.call(arguments, function (s) {
-          if (typeof s === 'string') {
-            return s
-          } else {
-            return JSON.stringify(s)
-          }
-        }).join('').replace(/"/g, "'").replace(/,/g, ', ').replace(/:/g, ': ')
-        this.executeOrder.push(command)
-      }
-    }
-    getDebugData () {
-      console.log(this.executeOrder.join('\n'))
-    }
-    stopGarbageCollector () {
-      var self = this
-      this.gc = false
-      this.gcTimeout = -1
-      return new Promise(function (resolve) {
-        self.requestTransaction(function * () {
-          var ungc /* :Array<Struct> */ = self.gc1.concat(self.gc2)
-          self.gc1 = []
-          self.gc2 = []
-          for (var i = 0; i < ungc.length; i++) {
-            var op = yield * this.getOperation(ungc[i])
-            if (op != null) {
-              delete op.gc
-              yield * this.setOperation(op)
-            }
-          }
-          resolve()
-        })
-      })
-    }
-    /*
-      Try to add to GC.
-
-      TODO: rename this function
-
-      Rulez:
-      * Only gc if this user is online & gc turned on
-      * The most left element in a list must not be gc'd.
-        => There is at least one element in the list
-
-      returns true iff op was added to GC
-    */
-    * addToGarbageCollector (op, left) {
-      if (
-        op.gc == null &&
-        op.deleted === true &&
-        this.store.gc &&
-        this.store.y.connector.isSynced
-      ) {
-        var gc = false
-        if (left != null && left.deleted === true) {
-          gc = true
-        } else if (op.content != null && op.content.length > 1) {
-          op = yield * this.getInsertionCleanStart([op.id[0], op.id[1] + 1])
-          gc = true
-        }
-        if (gc) {
-          op.gc = true
-          yield * this.setOperation(op)
-          this.store.queueGarbageCollector(op.id)
-          return true
-        }
-      }
-      return false
-    }
-    removeFromGarbageCollector (op) {
-      function filter (o) {
-        return !Y.utils.compareIds(o, op.id)
-      }
-      this.gc1 = this.gc1.filter(filter)
-      this.gc2 = this.gc2.filter(filter)
-      delete op.gc
-    }
-    destroyTypes () {
-      for (var key in this.initializedTypes) {
-        var type = this.initializedTypes[key]
-        if (type._destroy != null) {
-          type._destroy()
-        } else {
-          console.error('The type you included does not provide destroy functionality, it will remain in memory (updating your packages will help).')
-        }
-      }
-    }
-    * destroy () {
-      clearTimeout(this.gcInterval)
-      this.gcInterval = null
-      this.stopRepairCheck()
-    }
-    setUserId (userId) {
-      if (!this.userIdPromise.inProgress) {
-        this.userIdPromise.inProgress = true
-        var self = this
-        self.requestTransaction(function * () {
-          self.userId = userId
-          var state = yield * this.getState(userId)
-          self.opClock = state.clock
-          self.userIdPromise.resolve(userId)
-        })
-      }
-      return this.userIdPromise
-    }
-    whenUserIdSet (f) {
-      this.userIdPromise.then(f)
-    }
-    getNextOpId (numberOfIds) {
-      if (numberOfIds == null) {
-        throw new Error('getNextOpId expects the number of created ids to create!')
-      } else if (this.userId == null) {
-        throw new Error('OperationStore not yet initialized!')
-      } else {
-        var id = [this.userId, this.opClock]
-        this.opClock += numberOfIds
-        return id
-      }
-    }
-    /*
-      Apply a list of operations.
-
-      * we save a timestamp, because we received new operations that could resolve ops in this.listenersById (see this.startRepairCheck)
-      * get a transaction
-      * check whether all Struct.*.requiredOps are in the OS
-      * check if it is an expected op (otherwise wait for it)
-      * check if was deleted, apply a delete operation after op was applied
-    */
-    applyOperations (decoder) {
-      this.opsReceivedTimestamp = new Date()
-      let length = decoder.readUint32()
-
-      for (var i = 0; i < length; i++) {
-        let o = Y.Struct.binaryDecodeOperation(decoder)
-        if (o.id == null || o.id[0] !== this.y.connector.userId) {
-          var required = Y.Struct[o.struct].requiredOps(o)
-          if (o.requires != null) {
-            required = required.concat(o.requires)
-          }
-          this.whenOperationsExist(required, o)
-        }
-      }
-    }
-    /*
-      op is executed as soon as every operation requested is available.
-      Note that Transaction can (and should) buffer requests.
-    */
-    whenOperationsExist (ids, op) {
-      if (ids.length > 0) {
-        let listener = {
-          op: op,
-          missing: ids.length
-        }
-
-        for (let i = 0; i < ids.length; i++) {
-          let id = ids[i]
-          let sid = JSON.stringify(id)
-          let l = this.listenersById[sid]
-          if (l == null) {
-            l = []
-            this.listenersById[sid] = l
-          }
-          l.push(listener)
-        }
-      } else {
-        this.listenersByIdExecuteNow.push({
-          op: op
-        })
-      }
-
-      if (this.listenersByIdRequestPending) {
-        return
-      }
-
-      this.listenersByIdRequestPending = true
-      var store = this
-
-      this.requestTransaction(function * () {
-        var exeNow = store.listenersByIdExecuteNow
-        store.listenersByIdExecuteNow = []
-
-        var ls = store.listenersById
-        store.listenersById = {}
-
-        store.listenersByIdRequestPending = false
-
-        for (let key = 0; key < exeNow.length; key++) {
-          let o = exeNow[key].op
-          yield * store.tryExecute.call(this, o)
-        }
-
-        for (var sid in ls) {
-          var l = ls[sid]
-          var id = JSON.parse(sid)
-          var op
-          if (typeof id[1] === 'string') {
-            op = yield * this.getOperation(id)
-          } else {
-            op = yield * this.getInsertion(id)
-          }
-          if (op == null) {
-            store.listenersById[sid] = l
-          } else {
-            for (let i = 0; i < l.length; i++) {
-              let listener = l[i]
-              let o = listener.op
-              if (--listener.missing === 0) {
-                yield * store.tryExecute.call(this, o)
-              }
-            }
-          }
-        }
-      })
-    }
-    /*
-      Actually execute an operation, when all expected operations are available.
-    */
-    /* :: // TODO: this belongs somehow to transaction
-    store: Object;
-    getOperation: any;
-    isGarbageCollected: any;
-    addOperation: any;
-    whenOperationsExist: any;
-    */
-    * tryExecute (op) {
-      this.store.addToDebug('yield * this.store.tryExecute.call(this, ', JSON.stringify(op), ')')
-      if (op.struct === 'Delete') {
-        yield * Y.Struct.Delete.execute.call(this, op)
-        // this is now called in Transaction.deleteOperation!
-        // yield * this.store.operationAdded(this, op)
-      } else {
-        // check if this op was defined
-        var defined = yield * this.getInsertion(op.id)
-        while (defined != null && defined.content != null) {
-          // check if this op has a longer content in the case it is defined
-          if (defined.id[1] + defined.content.length < op.id[1] + op.content.length) {
-            var overlapSize = defined.content.length - (op.id[1] - defined.id[1])
-            op.content.splice(0, overlapSize)
-            op.id = [op.id[0], op.id[1] + overlapSize]
-            op.left = Y.utils.getLastId(defined)
-            op.origin = op.left
-            defined = yield * this.getOperation(op.id) // getOperation suffices here
-          } else {
-            break
-          }
-        }
-        if (defined == null) {
-          var opid = op.id
-          var isGarbageCollected = yield * this.isGarbageCollected(opid)
-          if (!isGarbageCollected) {
-            // TODO: reduce number of get / put calls for op ..
-            yield * Y.Struct[op.struct].execute.call(this, op)
-            yield * this.addOperation(op)
-            yield * this.store.operationAdded(this, op)
-            // operationAdded can change op..
-            op = yield * this.getOperation(opid)
-            // if insertion, try to combine with left
-            yield * this.tryCombineWithLeft(op)
-          }
-        }
-      }
-    }
-    /*
-     * Called by a transaction when an operation is added.
-     * This function is especially important for y-indexeddb, where several instances may share a single database.
-     * Every time an operation is created by one instance, it is send to all other instances and operationAdded is called
-     *
-     * If it's not a Delete operation:
-     *   * Checks if another operation is executable (listenersById)
-     *   * Update state, if possible
-     *
-     * Always:
-     *   * Call type
-     */
-    * operationAdded (transaction, op) {
-      if (op.struct === 'Delete') {
-        var type = this.initializedTypes[JSON.stringify(op.targetParent)]
-        if (type != null) {
-          yield * type._changed(transaction, op)
-        }
-      } else {
-        // increase SS
-        yield * transaction.updateState(op.id[0])
-        var opLen = op.content != null ? op.content.length : 1
-        for (let i = 0; i < opLen; i++) {
-          // notify whenOperation listeners (by id)
-          var sid = JSON.stringify([op.id[0], op.id[1] + i])
-          var l = this.listenersById[sid]
-          delete this.listenersById[sid]
-          if (l != null) {
-            for (var key in l) {
-              var listener = l[key]
-              if (--listener.missing === 0) {
-                this.whenOperationsExist([], listener.op)
-              }
-            }
-          }
-        }
-        var t = this.initializedTypes[JSON.stringify(op.parent)]
-
-        // if parent is deleted, mark as gc'd and return
-        if (op.parent != null) {
-          var parentIsDeleted = yield * transaction.isDeleted(op.parent)
-          if (parentIsDeleted) {
-            yield * transaction.deleteList(op.id)
-            return
-          }
-        }
-
-        // notify parent, if it was instanciated as a custom type
-        if (t != null) {
-          let o = Y.utils.copyOperation(op)
-          yield * t._changed(transaction, o)
-        }
-        if (!op.deleted) {
-          // Delete if DS says this is actually deleted
-          var len = op.content != null ? op.content.length : 1
-          var startId = op.id // You must not use op.id in the following loop, because op will change when deleted
-            // TODO: !! console.log('TODO: change this before commiting')
-          for (let i = 0; i < len; i++) {
-            var id = [startId[0], startId[1] + i]
-            var opIsDeleted = yield * transaction.isDeleted(id)
-            if (opIsDeleted) {
-              var delop = {
-                struct: 'Delete',
-                target: id
-              }
-              yield * this.tryExecute.call(transaction, delop)
-            }
-          }
-        }
-      }
-    }
-    whenTransactionsFinished () {
-      if (this.transactionInProgress) {
-        if (this.transactionsFinished == null) {
-          var resolve_
-          var promise = new Promise(function (resolve) {
-            resolve_ = resolve
-          })
-          this.transactionsFinished = {
-            resolve: resolve_,
-            promise: promise
-          }
-        }
-        return this.transactionsFinished.promise
-      } else {
-        return Promise.resolve()
-      }
-    }
-    // Check if there is another transaction request.
-    // * the last transaction is always a flush :)
-    getNextRequest () {
-      if (this.waitingTransactions.length === 0) {
-        if (this.transactionIsFlushed) {
-          this.transactionInProgress = false
-          this.transactionIsFlushed = false
-          if (this.transactionsFinished != null) {
-            this.transactionsFinished.resolve()
-            this.transactionsFinished = null
-          }
-          return null
-        } else {
-          this.transactionIsFlushed = true
-          return function * () {
-            yield * this.flush()
-          }
-        }
-      } else {
-        this.transactionIsFlushed = false
-        return this.waitingTransactions.shift()
-      }
-    }
-    requestTransaction (makeGen/* :any */, callImmediately) {
-      this.waitingTransactions.push(makeGen)
-      if (!this.transactionInProgress) {
-        this.transactionInProgress = true
-        setTimeout(() => {
-          this.transact(this.getNextRequest())
-        }, 0)
-      }
-    }
-    /*
-      Get a created/initialized type.
-    */
-    getType (id) {
-      return this.initializedTypes[JSON.stringify(id)]
-    }
-    /*
-      Init type. This is called when a remote operation is retrieved, and transformed to a type
-      TODO: delete type from store.initializedTypes[id] when corresponding id was deleted!
-    */
-    * initType (id, args) {
-      var sid = JSON.stringify(id)
-      var t = this.store.initializedTypes[sid]
-      if (t == null) {
-        var op/* :MapStruct | ListStruct */ = yield * this.getOperation(id)
-        if (op != null) {
-          t = yield * Y[op.type].typeDefinition.initType.call(this, this.store, op, args)
-          this.store.initializedTypes[sid] = t
-        }
-      }
-      return t
-    }
-    /*
-     Create type. This is called when the local user creates a type (which is a synchronous action)
-    */
-    createType (typedefinition, id) {
-      var structname = typedefinition[0].struct
-      id = id || this.getNextOpId(1)
-      var op = Y.Struct[structname].create(id, typedefinition[1])
-      op.type = typedefinition[0].name
-
-      this.requestTransaction(function * () {
-        if (op.id[0] === 0xFFFFFF) {
-          yield * this.setOperation(op)
-        } else {
-          yield * this.applyCreatedOperations([op])
-        }
-      })
-      var t = Y[op.type].typeDefinition.createType(this, op, typedefinition[1])
-      this.initializedTypes[JSON.stringify(op.id)] = t
-      return t
-    }
-  }
-  Y.AbstractDatabase = AbstractDatabase
-}

src/Database.spec.js

@@ -1,354 +0,0 @@
-/* global async, databases, describe, beforeEach, afterEach */
-/* eslint-env browser,jasmine,console */
-'use strict'
-
-var Y = require('./SpecHelper.js')
-
-for (let database of databases) {
-  describe(`Database (${database})`, function () {
-    var store
-    describe('DeleteStore', function () {
-      describe('Basic', function () {
-        beforeEach(function () {
-          store = new Y[database](null, {
-            gcTimeout: -1,
-            namespace: 'testing'
-          })
-        })
-        afterEach(function (done) {
-          store.requestTransaction(function * () {
-            yield * this.store.destroy()
-            done()
-          })
-        })
-        it('Deleted operation is deleted', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['u1', 10], 1)
-            expect(yield * this.isDeleted(['u1', 10])).toBeTruthy()
-            expect(yield * this.getDeleteSet()).toEqual({'u1': [[10, 1, false]]})
-            done()
-          })
-        }))
-        it('Deleted operation extends other deleted operation', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['u1', 10], 1)
-            yield * this.markDeleted(['u1', 11], 1)
-            expect(yield * this.isDeleted(['u1', 10])).toBeTruthy()
-            expect(yield * this.isDeleted(['u1', 11])).toBeTruthy()
-            expect(yield * this.getDeleteSet()).toEqual({'u1': [[10, 2, false]]})
-            done()
-          })
-        }))
-        it('Deleted operation extends other deleted operation', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['0', 3], 1)
-            yield * this.markDeleted(['0', 4], 1)
-            yield * this.markDeleted(['0', 2], 1)
-            expect(yield * this.getDeleteSet()).toEqual({'0': [[2, 3, false]]})
-            done()
-          })
-        }))
-        it('Debug #1', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['166', 0], 1)
-            yield * this.markDeleted(['166', 2], 1)
-            yield * this.markDeleted(['166', 0], 1)
-            yield * this.markDeleted(['166', 2], 1)
-            yield * this.markGarbageCollected(['166', 2], 1)
-            yield * this.markDeleted(['166', 1], 1)
-            yield * this.markDeleted(['166', 3], 1)
-            yield * this.markGarbageCollected(['166', 3], 1)
-            yield * this.markDeleted(['166', 0], 1)
-            expect(yield * this.getDeleteSet()).toEqual({'166': [[0, 2, false], [2, 2, true]]})
-            done()
-          })
-        }))
-        it('Debug #2', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['293', 0], 1)
-            yield * this.markDeleted(['291', 2], 1)
-            yield * this.markDeleted(['291', 2], 1)
-            yield * this.markGarbageCollected(['293', 0], 1)
-            yield * this.markDeleted(['293', 1], 1)
-            yield * this.markGarbageCollected(['291', 2], 1)
-            expect(yield * this.getDeleteSet()).toEqual({'291': [[2, 1, true]], '293': [[0, 1, true], [1, 1, false]]})
-            done()
-          })
-        }))
-        it('Debug #3', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['581', 0], 1)
-            yield * this.markDeleted(['581', 1], 1)
-            yield * this.markDeleted(['580', 0], 1)
-            yield * this.markDeleted(['580', 0], 1)
-            yield * this.markGarbageCollected(['581', 0], 1)
-            yield * this.markDeleted(['581', 2], 1)
-            yield * this.markDeleted(['580', 1], 1)
-            yield * this.markDeleted(['580', 2], 1)
-            yield * this.markDeleted(['580', 1], 1)
-            yield * this.markDeleted(['580', 2], 1)
-            yield * this.markGarbageCollected(['581', 2], 1)
-            yield * this.markGarbageCollected(['581', 1], 1)
-            yield * this.markGarbageCollected(['580', 1], 1)
-            expect(yield * this.getDeleteSet()).toEqual({'580': [[0, 1, false], [1, 1, true], [2, 1, false]], '581': [[0, 3, true]]})
-            done()
-          })
-        }))
-        it('Debug #4', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['544', 0], 1)
-            yield * this.markDeleted(['543', 2], 1)
-            yield * this.markDeleted(['544', 0], 1)
-            yield * this.markDeleted(['543', 2], 1)
-            yield * this.markGarbageCollected(['544', 0], 1)
-            yield * this.markDeleted(['545', 1], 1)
-            yield * this.markDeleted(['543', 4], 1)
-            yield * this.markDeleted(['543', 3], 1)
-            yield * this.markDeleted(['544', 1], 1)
-            yield * this.markDeleted(['544', 2], 1)
-            yield * this.markDeleted(['544', 1], 1)
-            yield * this.markDeleted(['544', 2], 1)
-            yield * this.markGarbageCollected(['543', 2], 1)
-            yield * this.markGarbageCollected(['543', 4], 1)
-            yield * this.markGarbageCollected(['544', 2], 1)
-            yield * this.markGarbageCollected(['543', 3], 1)
-            expect(yield * this.getDeleteSet()).toEqual({'543': [[2, 3, true]], '544': [[0, 1, true], [1, 1, false], [2, 1, true]], '545': [[1, 1, false]]})
-            done()
-          })
-        }))
-        it('Debug #5', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.applyDeleteSet({'16': [[1, 2, false]], '17': [[0, 1, true], [1, 3, false]]})
-            expect(yield * this.getDeleteSet()).toEqual({'16': [[1, 2, false]], '17': [[0, 1, true], [1, 3, false]]})
-            yield * this.applyDeleteSet({'16': [[1, 2, false]], '17': [[0, 4, true]]})
-            expect(yield * this.getDeleteSet()).toEqual({'16': [[1, 2, false]], '17': [[0, 4, true]]})
-            done()
-          })
-        }))
-        it('Debug #6', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.applyDeleteSet({'40': [[0, 3, false]]})
-            expect(yield * this.getDeleteSet()).toEqual({'40': [[0, 3, false]]})
-            yield * this.applyDeleteSet({'39': [[2, 2, false]], '40': [[0, 1, true], [1, 2, false]], '41': [[2, 1, false]]})
-            expect(yield * this.getDeleteSet()).toEqual({'39': [[2, 2, false]], '40': [[0, 1, true], [1, 2, false]], '41': [[2, 1, false]]})
-            done()
-          })
-        }))
-        it('Debug #7', async(function * (done) {
-          store.requestTransaction(function * () {
-            yield * this.markDeleted(['9', 2], 1)
-            yield * this.markDeleted(['11', 2], 1)
-            yield * this.markDeleted(['11', 4], 1)
-            yield * this.markDeleted(['11', 1], 1)
-            yield * this.markDeleted(['9', 4], 1)
-            yield * this.markDeleted(['10', 0], 1)
-            yield * this.markGarbageCollected(['11', 2], 1)
-            yield * this.markDeleted(['11', 2], 1)
-            yield * this.markGarbageCollected(['11', 3], 1)
-            yield * this.markDeleted(['11', 3], 1)
-            yield * this.markDeleted(['11', 3], 1)
-            yield * this.markDeleted(['9', 4], 1)
-            yield * this.markDeleted(['10', 0], 1)
-            yield * this.markGarbageCollected(['11', 1], 1)
-            yield * this.markDeleted(['11', 1], 1)
-            expect(yield * this.getDeleteSet()).toEqual({'9': [[2, 1, false], [4, 1, false]], '10': [[0, 1, false]], '11': [[1, 3, true], [4, 1, false]]})
-            done()
-          })
-        }))
-      })
-    })
-    describe('OperationStore', function () {
-      describe('Basic Tests', function () {
-        beforeEach(function () {
-          store = new Y[database](null, {
-            gcTimeout: -1,
-            namespace: 'testing'
-          })
-        })
-        afterEach(function (done) {
-          store.requestTransaction(function * () {
-            yield * this.store.destroy()
-            done()
-          })
-        })
-        it('debug #1', function (done) {
-          store.requestTransaction(function * () {
-            yield * this.os.put({id: [2]})
-            yield * this.os.put({id: [0]})
-            yield * this.os.delete([2])
-            yield * this.os.put({id: [1]})
-            expect(yield * this.os.find([0])).toBeTruthy()
-            expect(yield * this.os.find([1])).toBeTruthy()
-            expect(yield * this.os.find([2])).toBeFalsy()
-            done()
-          })
-        })
-        it('can add&retrieve 5 elements', function (done) {
-          store.requestTransaction(function * () {
-            yield * this.os.put({val: 'four', id: [4]})
-            yield * this.os.put({val: 'one', id: [1]})
-            yield * this.os.put({val: 'three', id: [3]})
-            yield * this.os.put({val: 'two', id: [2]})
-            yield * this.os.put({val: 'five', id: [5]})
-            expect((yield * this.os.find([1])).val).toEqual('one')
-            expect((yield * this.os.find([2])).val).toEqual('two')
-            expect((yield * this.os.find([3])).val).toEqual('three')
-            expect((yield * this.os.find([4])).val).toEqual('four')
-            expect((yield * this.os.find([5])).val).toEqual('five')
-            done()
-          })
-        })
-        it('5 elements do not exist anymore after deleting them', function (done) {
-          store.requestTransaction(function * () {
-            yield * this.os.put({val: 'four', id: [4]})
-            yield * this.os.put({val: 'one', id: [1]})
-            yield * this.os.put({val: 'three', id: [3]})
-            yield * this.os.put({val: 'two', id: [2]})
-            yield * this.os.put({val: 'five', id: [5]})
-            yield * this.os.delete([4])
-            expect(yield * this.os.find([4])).not.toBeTruthy()
-            yield * this.os.delete([3])
-            expect(yield * this.os.find([3])).not.toBeTruthy()
-            yield * this.os.delete([2])
-            expect(yield * this.os.find([2])).not.toBeTruthy()
-            yield * this.os.delete([1])
-            expect(yield * this.os.find([1])).not.toBeTruthy()
-            yield * this.os.delete([5])
-            expect(yield * this.os.find([5])).not.toBeTruthy()
-            done()
-          })
-        })
-      })
-      var numberOfOSTests = 1000
-      describe(`Random Tests - after adding&deleting (0.8/0.2) ${numberOfOSTests} times`, function () {
-        var elements = []
-        beforeAll(function (done) {
-          store = new Y[database](null, {
-            gcTimeout: -1,
-            namespace: 'testing'
-          })
-          store.requestTransaction(function * () {
-            for (var i = 0; i < numberOfOSTests; i++) {
-              var r = Math.random()
-              if (r < 0.8) {
-                var obj = [Math.floor(Math.random() * numberOfOSTests * 10000)]
-                if (!(yield * this.os.find(obj))) {
-                  elements.push(obj)
-                  yield * this.os.put({id: obj})
-                }
-              } else if (elements.length > 0) {
-                var elemid = Math.floor(Math.random() * elements.length)
-                var elem = elements[elemid]
-                elements = elements.filter(function (e) {
-                  return !Y.utils.compareIds(e, elem)
-                })
-                yield * this.os.delete(elem)
-              }
-            }
-            done()
-          })
-        })
-        afterAll(function (done) {
-          store.requestTransaction(function * () {
-            yield * this.store.destroy()
-            done()
-          })
-        })
-        it('can find every object', function (done) {
-          store.requestTransaction(function * () {
-            for (var id of elements) {
-              expect((yield * this.os.find(id)).id).toEqual(id)
-            }
-            done()
-          })
-        })
-
-        it('can find every object with lower bound search', function (done) {
-          store.requestTransaction(function * () {
-            for (var id of elements) {
-              var e = yield * this.os.findWithLowerBound(id)
-              expect(e.id).toEqual(id)
-            }
-            done()
-          })
-        })
-
-        it('iterating over a tree with lower bound yields the right amount of results', function (done) {
-          var lowerBound = elements[Math.floor(Math.random() * elements.length)]
-          var expectedResults = elements.filter(function (e, pos) {
-            return (Y.utils.smaller(lowerBound, e) || Y.utils.compareIds(e, lowerBound)) && elements.indexOf(e) === pos
-          }).length
-
-          var actualResults = 0
-          store.requestTransaction(function * () {
-            yield * this.os.iterate(this, lowerBound, null, function * (val) {
-              expect(val).toBeDefined()
-              actualResults++
-            })
-            expect(expectedResults).toEqual(actualResults)
-            done()
-          })
-        })
-
-        it('iterating over a tree without bounds yield the right amount of results', function (done) {
-          var lowerBound = null
-          var expectedResults = elements.filter(function (e, pos) {
-            return elements.indexOf(e) === pos
-          }).length
-          var actualResults = 0
-          store.requestTransaction(function * () {
-            yield * this.os.iterate(this, lowerBound, null, function * (val) {
-              expect(val).toBeDefined()
-              actualResults++
-            })
-            expect(expectedResults).toEqual(actualResults)
-            done()
-          })
-        })
-
-        it('iterating over a tree with upper bound yields the right amount of results', function (done) {
-          var upperBound = elements[Math.floor(Math.random() * elements.length)]
-          var expectedResults = elements.filter(function (e, pos) {
-            return (Y.utils.smaller(e, upperBound) || Y.utils.compareIds(e, upperBound)) && elements.indexOf(e) === pos
-          }).length
-
-          var actualResults = 0
-          store.requestTransaction(function * () {
-            yield * this.os.iterate(this, null, upperBound, function * (val) {
-              expect(val).toBeDefined()
-              actualResults++
-            })
-            expect(expectedResults).toEqual(actualResults)
-            done()
-          })
-        })
-
-        it('iterating over a tree with upper and lower bounds yield the right amount of results', function (done) {
-          var b1 = elements[Math.floor(Math.random() * elements.length)]
-          var b2 = elements[Math.floor(Math.random() * elements.length)]
-          var upperBound, lowerBound
-          if (Y.utils.smaller(b1, b2)) {
-            lowerBound = b1
-            upperBound = b2
-          } else {
-            lowerBound = b2
-            upperBound = b1
-          }
-          var expectedResults = elements.filter(function (e, pos) {
-            return (Y.utils.smaller(lowerBound, e) || Y.utils.compareIds(e, lowerBound)) &&
-              (Y.utils.smaller(e, upperBound) || Y.utils.compareIds(e, upperBound)) && elements.indexOf(e) === pos
-          }).length
-          var actualResults = 0
-          store.requestTransaction(function * () {
-            yield * this.os.iterate(this, lowerBound, upperBound, function * (val) {
-              expect(val).toBeDefined()
-              actualResults++
-            })
-            expect(expectedResults).toEqual(actualResults)
-            done()
-          })
-        })
-      })
-    })
-  })
-}

src/Encoding.js

@@ -1,152 +0,0 @@
-import utf8 from 'utf-8'
-
-const bits7 = 0b1111111
-const bits8 = 0b11111111
-
-export class BinaryEncoder {
-  constructor () {
-    this.data = []
-  }
-
-  get pos () {
-    return this.data.length
-  }
-
-  createBuffer () {
-    return Uint8Array.from(this.data).buffer
-  }
-
-  writeUint8 (num) {
-    this.data.push(num & bits8)
-  }
-
-  setUint8 (pos, num) {
-    this.data[pos] = num & bits8
-  }
-
-  writeUint16 (num) {
-    this.data.push(num & bits8, (num >>> 8) & bits8)
-  }
-
-  setUint16 (pos, num) {
-    this.data[pos] = num & bits8
-    this.data[pos + 1] = (num >>> 8) & bits8
-  }
-
-  writeUint32 (num) {
-    for (let i = 0; i < 4; i++) {
-      this.data.push(num & bits8)
-      num >>>= 8
-    }
-  }
-
-  setUint32 (pos, num) {
-    for (let i = 0; i < 4; i++) {
-      this.data[pos + i] = num & bits8
-      num >>>= 8
-    }
-  }
-
-  writeVarUint (num) {
-    while (num >= 0b10000000) {
-      this.data.push(0b10000000 | (bits7 & num))
-      num >>>= 7
-    }
-    this.data.push(bits7 & num)
-  }
-
-  writeVarString (str) {
-    let bytes = utf8.setBytesFromString(str)
-    let len = bytes.length
-    this.writeVarUint(len)
-    for (let i = 0; i < len; i++) {
-      this.data.push(bytes[i])
-    }
-  }
-
-  writeOpID (id) {
-    let user = id[0]
-    this.writeVarUint(user)
-    if (user !== 0xFFFFFF) {
-      this.writeVarUint(id[1])
-    } else {
-      this.writeVarString(id[1])
-    }
-  }
-}
-
-export class BinaryDecoder {
-  constructor (buffer) {
-    if (buffer instanceof ArrayBuffer) {
-      this.uint8arr = new Uint8Array(buffer)
-    } else if (buffer instanceof Uint8Array || (typeof Buffer !== 'undefined' && buffer instanceof Buffer)) {
-      this.uint8arr = buffer
-    } else {
-      throw new Error('Expected an ArrayBuffer or Uint8Array!')
-    }
-    this.pos = 0
-  }
-
-  skip8 () {
-    this.pos++
-  }
-
-  readUint8 () {
-    return this.uint8arr[this.pos++]
-  }
-
-  readUint32 () {
-    let uint =
-      this.uint8arr[this.pos] +
-      (this.uint8arr[this.pos + 1] << 8) +
-      (this.uint8arr[this.pos + 2] << 16) +
-      (this.uint8arr[this.pos + 3] << 24)
-    this.pos += 4
-    return uint
-  }
-
-  peekUint8 () {
-    return this.uint8arr[this.pos]
-  }
-
-  readVarUint () {
-    let num = 0
-    let len = 0
-    while (true) {
-      let r = this.uint8arr[this.pos++]
-      num = num | ((r & bits7) << len)
-      len += 7
-      if (r < 1 << 7) {
-        return num >>> 0 // return unsigned number!
-      }
-      if (len > 35) {
-        throw new Error('Integer out of range!')
-      }
-    }
-  }
-
-  readVarString () {
-    let len = this.readVarUint()
-    let bytes = new Array(len)
-    for (let i = 0; i < len; i++) {
-      bytes[i] = this.uint8arr[this.pos++]
-    }
-    return utf8.getStringFromBytes(bytes)
-  }
-
-  peekVarString () {
-    let pos = this.pos
-    let s = this.readVarString()
-    this.pos = pos
-    return s
-  }
-
-  readOpID () {
-    let user = this.readVarUint()
-    if (user !== 0xFFFFFF) {
-      return [user, this.readVarUint()]
-    } else {
-      return [user, this.readVarString()]
-    }
-  }
-}

src/MessageHandler.js

@@ -1,193 +0,0 @@
-
-import Y from './y.js'
-import { BinaryDecoder, BinaryEncoder } from './Encoding.js'
-
-export function formatYjsMessage (buffer) {
-  let decoder = new BinaryDecoder(buffer)
-  decoder.readVarString() // read roomname
-  let type = decoder.readVarString()
-  let strBuilder = []
-  strBuilder.push('\n === ' + type + ' ===\n')
-  if (type === 'update') {
-    logMessageUpdate(decoder, strBuilder)
-  } else if (type === 'sync step 1') {
-    logMessageSyncStep1(decoder, strBuilder)
-  } else if (type === 'sync step 2') {
-    logMessageSyncStep2(decoder, strBuilder)
-  } else {
-    strBuilder.push('-- Unknown message type - probably an encoding issue!!!')
-  }
-  return strBuilder.join('')
-}
-
-export function formatYjsMessageType (buffer) {
-  let decoder = new BinaryDecoder(buffer)
-  decoder.readVarString() // roomname
-  return decoder.readVarString()
-}
-
-export function logMessageUpdate (decoder, strBuilder) {
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    strBuilder.push(JSON.stringify(Y.Struct.binaryDecodeOperation(decoder)) + '\n')
-  }
-}
-
-export function computeMessageUpdate (decoder, encoder, conn) {
-  if (conn.y.db.forwardAppliedOperations || conn.y.persistence != null) {
-    let messagePosition = decoder.pos
-    let len = decoder.readUint32()
-    let delops = []
-    for (let i = 0; i < len; i++) {
-      let op = Y.Struct.binaryDecodeOperation(decoder)
-      if (op.struct === 'Delete') {
-        delops.push(op)
-      }
-    }
-    if (delops.length > 0) {
-      if (conn.y.db.forwardAppliedOperations) {
-        conn.broadcastOps(delops)
-      }
-      if (conn.y.persistence) {
-        conn.y.persistence.saveOperations(delops)
-      }
-    }
-    decoder.pos = messagePosition
-  }
-  conn.y.db.applyOperations(decoder)
-}
-
-export function sendSyncStep1 (conn, syncUser) {
-  conn.y.db.requestTransaction(function * () {
-    let encoder = new BinaryEncoder()
-    encoder.writeVarString(conn.opts.room || '')
-    encoder.writeVarString('sync step 1')
-    encoder.writeVarString(conn.authInfo || '')
-    encoder.writeVarUint(conn.protocolVersion)
-    let preferUntransformed = conn.preferUntransformed && this.os.length === 0 // TODO: length may not be defined
-    encoder.writeUint8(preferUntransformed ? 1 : 0)
-    yield * this.writeStateSet(encoder)
-    conn.send(syncUser, encoder.createBuffer())
-  })
-}
-
-export function logMessageSyncStep1 (decoder, strBuilder) {
-  let auth = decoder.readVarString()
-  let protocolVersion = decoder.readVarUint()
-  let preferUntransformed = decoder.readUint8() === 1
-  strBuilder.push(`
-  - auth: "${auth}"
-  - protocolVersion: ${protocolVersion}
-  - preferUntransformed: ${preferUntransformed}
-`)
-  logSS(decoder, strBuilder)
-}
-
-export function computeMessageSyncStep1 (decoder, encoder, conn, senderConn, sender) {
-  let protocolVersion = decoder.readVarUint()
-  let preferUntransformed = decoder.readUint8() === 1
-
-  // check protocol version
-  if (protocolVersion !== conn.protocolVersion) {
-    console.warn(
-      `You tried to sync with a yjs instance that has a different protocol version
-      (You: ${protocolVersion}, Client: ${protocolVersion}).
-      The sync was stopped. You need to upgrade your dependencies (especially Yjs & the Connector)!
-      `)
-    conn.y.destroy()
-  }
-
-  return conn.y.db.whenTransactionsFinished().then(() => {
-    // send sync step 2
-    conn.y.db.requestTransaction(function * () {
-      encoder.writeVarString('sync step 2')
-      encoder.writeVarString(conn.authInfo || '')
-
-      if (preferUntransformed) {
-        encoder.writeUint8(1)
-        yield * this.writeOperationsUntransformed(encoder)
-      } else {
-        encoder.writeUint8(0)
-        yield * this.writeOperations(encoder, decoder)
-      }
-
-      yield * this.writeDeleteSet(encoder)
-      conn.send(senderConn.uid, encoder.createBuffer())
-      senderConn.receivedSyncStep2 = true
-    })
-    return conn.y.db.whenTransactionsFinished().then(() => {
-      if (conn.role === 'slave') {
-        sendSyncStep1(conn, sender)
-      }
-    })
-  })
-}
-
-export function logSS (decoder, strBuilder) {
-  strBuilder.push('  == SS: \n')
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    let user = decoder.readVarUint()
-    let clock = decoder.readVarUint()
-    strBuilder.push(`     ${user}: ${clock}\n`)
-  }
-}
-
-export function logOS (decoder, strBuilder) {
-  strBuilder.push('  == OS: \n')
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    let op = Y.Struct.binaryDecodeOperation(decoder)
-    strBuilder.push(JSON.stringify(op) + '\n')
-  }
-}
-
-export function logDS (decoder, strBuilder) {
-  strBuilder.push('  == DS: \n')
-  let len = decoder.readUint32()
-  for (let i = 0; i < len; i++) {
-    let user = decoder.readVarUint()
-    strBuilder.push(`    User: ${user}: `)
-    let len2 = decoder.readVarUint()
-    for (let j = 0; j < len2; j++) {
-      let from = decoder.readVarUint()
-      let to = decoder.readVarUint()
-      let gc = decoder.readUint8() === 1
-      strBuilder.push(`[${from}, ${to}, ${gc}]`)
-    }
-  }
-}
-
-export function logMessageSyncStep2 (decoder, strBuilder) {
-  strBuilder.push('     - auth: ' + decoder.readVarString() + '\n')
-  let osTransformed = decoder.readUint8() === 1
-  strBuilder.push('     - osUntransformed: ' + osTransformed + '\n')
-  logOS(decoder, strBuilder)
-  if (osTransformed) {
-    logSS(decoder, strBuilder)
-  }
-  logDS(decoder, strBuilder)
-}
-
-export function computeMessageSyncStep2 (decoder, encoder, conn, senderConn, sender) {
-  var db = conn.y.db
-  let defer = senderConn.syncStep2
-
-  // apply operations first
-  db.requestTransaction(function * () {
-    let osUntransformed = decoder.readUint8()
-    if (osUntransformed === 1) {
-      yield * this.applyOperationsUntransformed(decoder)
-    } else {
-      this.store.applyOperations(decoder)
-    }
-  })
-  // then apply ds
-  db.requestTransaction(function * () {
-    yield * this.applyDeleteSet(decoder)
-  })
-  return db.whenTransactionsFinished().then(() => {
-    conn._setSyncedWith(sender)
-    defer.resolve()
-  })
-}

src/MessageHandler/deleteSet.js

@@ -0,0 +1,130 @@
+import { deleteItemRange } from '../Struct/Delete.js'
+import ID from '../Util/ID.js'
+
+export function stringifyDeleteSet (y, decoder, strBuilder) {
+  let dsLength = decoder.readUint32()
+  for (let i = 0; i < dsLength; i++) {
+    let user = decoder.readVarUint()
+    strBuilder.push(' -' + user + ':')
+    let dvLength = decoder.readVarUint()
+    for (let j = 0; j < dvLength; j++) {
+      let from = decoder.readVarUint()
+      let len = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      strBuilder.push(`clock: ${from}, length: ${len}, gc: ${gc}`)
+    }
+  }
+  return strBuilder
+}
+
+export function writeDeleteSet (y, encoder) {
+  let currentUser = null
+  let currentLength
+  let lastLenPos
+
+  let numberOfUsers = 0
+  let laterDSLenPus = encoder.pos
+  encoder.writeUint32(0)
+
+  y.ds.iterate(null, null, function (n) {
+    var user = n._id.user
+    var clock = n._id.clock
+    var len = n.len
+    var gc = n.gc
+    if (currentUser !== user) {
+      numberOfUsers++
+      // a new user was found
+      if (currentUser !== null) { // happens on first iteration
+        encoder.setUint32(lastLenPos, currentLength)
+      }
+      currentUser = user
+      encoder.writeVarUint(user)
+      // pseudo-fill pos
+      lastLenPos = encoder.pos
+      encoder.writeUint32(0)
+      currentLength = 0
+    }
+    encoder.writeVarUint(clock)
+    encoder.writeVarUint(len)
+    encoder.writeUint8(gc ? 1 : 0)
+    currentLength++
+  })
+  if (currentUser !== null) { // happens on first iteration
+    encoder.setUint32(lastLenPos, currentLength)
+  }
+  encoder.setUint32(laterDSLenPus, numberOfUsers)
+}
+
+export function readDeleteSet (y, decoder) {
+  let dsLength = decoder.readUint32()
+  for (let i = 0; i < dsLength; i++) {
+    let user = decoder.readVarUint()
+    let dv = []
+    let dvLength = decoder.readUint32()
+    for (let j = 0; j < dvLength; j++) {
+      let from = decoder.readVarUint()
+      let len = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      dv.push([from, len, gc])
+    }
+    if (dvLength > 0) {
+      let pos = 0
+      let d = dv[pos]
+      let deletions = []
+      y.ds.iterate(new ID(user, 0), new ID(user, Number.MAX_VALUE), function (n) {
+        // cases:
+        // 1. d deletes something to the right of n
+        //  => go to next n (break)
+        // 2. d deletes something to the left of n
+        //  => create deletions
+        //  => reset d accordingly
+        //  *)=> if d doesn't delete anything anymore, go to next d (continue)
+        // 3. not 2) and d deletes something that also n deletes
+        //  => reset d so that it doesn't contain n's deletion
+        //  *)=> if d does not delete anything anymore, go to next d (continue)
+        while (d != null) {
+          var diff = 0 // describe the diff of length in 1) and 2)
+          if (n._id.clock + n.len <= d[0]) {
+            // 1)
+            break
+          } else if (d[0] < n._id.clock) {
+            // 2)
+            // delete maximum the len of d
+            // else delete as much as possible
+            diff = Math.min(n._id.clock - d[0], d[1])
+            // deleteItemRange(y, user, d[0], diff)
+            deletions.push([user, d[0], diff])
+          } else {
+            // 3)
+            diff = n._id.clock + n.len - d[0] // never null (see 1)
+            if (d[2] && !n.gc) {
+              // d marks as gc'd but n does not
+              // then delete either way
+              // deleteItemRange(y, user, d[0], Math.min(diff, d[1]))
+              deletions.push([user, d[0], Math.min(diff, d[1])])
+            }
+          }
+          if (d[1] <= diff) {
+            // d doesn't delete anything anymore
+            d = dv[++pos]
+          } else {
+            d[0] = d[0] + diff // reset pos
+            d[1] = d[1] - diff // reset length
+          }
+        }
+      })
+      // TODO: It would be more performant to apply the deletes in the above loop
+      // Adapt the Tree implementation to support delete while iterating
+      for (let i = deletions.length - 1; i >= 0; i--) {
+        const del = deletions[i]
+        deleteItemRange(y, del[0], del[1], del[2])
+      }
+      // for the rest.. just apply it
+      for (; pos < dv.length; pos++) {
+        d = dv[pos]
+        deleteItemRange(y, user, d[0], d[1])
+        // deletions.push([user, d[0], d[1], d[2]])
+      }
+    }
+  }
+}

src/MessageHandler/integrateRemoteStructs.js

@@ -0,0 +1,100 @@
+import { getStruct } from '../Util/structReferences.js'
+import BinaryDecoder from '../Binary/Decoder.js'
+import { logID } from './messageToString.js'
+
+class MissingEntry {
+  constructor (decoder, missing, struct) {
+    this.decoder = decoder
+    this.missing = missing.length
+    this.struct = struct
+  }
+}
+
+/**
+ * Integrate remote struct
+ * When a remote struct is integrated, other structs might be ready to ready to
+ * integrate.
+ */
+function _integrateRemoteStructHelper (y, struct) {
+  const id = struct._id
+  if (id === undefined) {
+    struct._integrate(y)
+  } else {
+    if (y.ss.getState(id.user) > id.clock) {
+      return
+    }
+    struct._integrate(y)
+    let msu = y._missingStructs.get(id.user)
+    if (msu != null) {
+      let clock = id.clock
+      const finalClock = clock + struct._length
+      for (;clock < finalClock; clock++) {
+        const missingStructs = msu.get(clock)
+        if (missingStructs !== undefined) {
+          missingStructs.forEach(missingDef => {
+            missingDef.missing--
+            if (missingDef.missing === 0) {
+              const decoder = missingDef.decoder
+              let oldPos = decoder.pos
+              let missing = missingDef.struct._fromBinary(y, decoder)
+              decoder.pos = oldPos
+              if (missing.length === 0) {
+                y._readyToIntegrate.push(missingDef.struct)
+              }
+            }
+          })
+          msu.delete(clock)
+        }
+      }
+    }
+  }
+}
+
+export function stringifyStructs (y, decoder, strBuilder) {
+  const len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let reference = decoder.readVarUint()
+    let Constr = getStruct(reference)
+    let struct = new Constr()
+    let missing = struct._fromBinary(y, decoder)
+    let logMessage = '  ' + struct._logString()
+    if (missing.length > 0) {
+      logMessage += ' .. missing: ' + missing.map(logID).join(', ')
+    }
+    strBuilder.push(logMessage)
+  }
+}
+
+export function integrateRemoteStructs (decoder, encoder, y) {
+  const len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let reference = decoder.readVarUint()
+    let Constr = getStruct(reference)
+    let struct = new Constr()
+    let decoderPos = decoder.pos
+    let missing = struct._fromBinary(y, decoder)
+    if (missing.length === 0) {
+      while (struct != null) {
+        _integrateRemoteStructHelper(y, struct)
+        struct = y._readyToIntegrate.shift()
+      }
+    } else {
+      let _decoder = new BinaryDecoder(decoder.uint8arr)
+      _decoder.pos = decoderPos
+      let missingEntry = new MissingEntry(_decoder, missing, struct)
+      let missingStructs = y._missingStructs
+      for (let i = missing.length - 1; i >= 0; i--) {
+        let m = missing[i]
+        if (!missingStructs.has(m.user)) {
+          missingStructs.set(m.user, new Map())
+        }
+        let msu = missingStructs.get(m.user)
+        if (!msu.has(m.clock)) {
+          msu.set(m.clock, [])
+        }
+        let mArray = msu = msu.get(m.clock)
+        mArray.push(missingEntry)
+      }
+    }
+  }
+}

src/MessageHandler/messageToString.js

@@ -0,0 +1,48 @@
+import BinaryDecoder from '../Binary/Decoder.js'
+import { stringifyStructs } from './integrateRemoteStructs.js'
+import { stringifySyncStep1 } from './syncStep1.js'
+import { stringifySyncStep2 } from './syncStep2.js'
+import ID from '../Util/ID.js'
+import RootID from '../Util/RootID.js'
+import Y from '../Y.js'
+
+export function messageToString ([y, buffer]) {
+  let decoder = new BinaryDecoder(buffer)
+  decoder.readVarString() // read roomname
+  let type = decoder.readVarString()
+  let strBuilder = []
+  strBuilder.push('\n === ' + type + ' ===')
+  if (type === 'update') {
+    stringifyStructs(y, decoder, strBuilder)
+  } else if (type === 'sync step 1') {
+    stringifySyncStep1(y, decoder, strBuilder)
+  } else if (type === 'sync step 2') {
+    stringifySyncStep2(y, decoder, strBuilder)
+  } else {
+    strBuilder.push('-- Unknown message type - probably an encoding issue!!!')
+  }
+  return strBuilder.join('\n')
+}
+
+export function messageToRoomname (buffer) {
+  let decoder = new BinaryDecoder(buffer)
+  decoder.readVarString() // roomname
+  return decoder.readVarString() // messageType
+}
+
+export function logID (id) {
+  if (id !== null && id._id != null) {
+    id = id._id
+  }
+  if (id === null) {
+    return '()'
+  } else if (id instanceof ID) {
+    return `(${id.user},${id.clock})`
+  } else if (id instanceof RootID) {
+    return `(${id.name},${id.type})`
+  } else if (id.constructor === Y) {
+    return `y`
+  } else {
+    throw new Error('This is not a valid ID!')
+  }
+}

src/MessageHandler/stateSet.js

@@ -0,0 +1,23 @@
+
+export function readStateSet (decoder) {
+  let ss = new Map()
+  let ssLength = decoder.readUint32()
+  for (let i = 0; i < ssLength; i++) {
+    let user = decoder.readVarUint()
+    let clock = decoder.readVarUint()
+    ss.set(user, clock)
+  }
+  return ss
+}
+
+export function writeStateSet (y, encoder) {
+  let lenPosition = encoder.pos
+  let len = 0
+  encoder.writeUint32(0)
+  for (let [user, clock] of y.ss.state) {
+    encoder.writeVarUint(user)
+    encoder.writeVarUint(clock)
+    len++
+  }
+  encoder.setUint32(lenPosition, len)
+}

src/MessageHandler/syncStep1.js

@@ -0,0 +1,70 @@
+import BinaryEncoder from '../Binary/Encoder.js'
+import { readStateSet, writeStateSet } from './stateSet.js'
+import { writeDeleteSet } from './deleteSet.js'
+import ID from '../Util/ID.js'
+import { RootFakeUserID } from '../Util/RootID.js'
+
+export function stringifySyncStep1 (y, decoder, strBuilder) {
+  let auth = decoder.readVarString()
+  let protocolVersion = decoder.readVarUint()
+  strBuilder.push(`  - auth: "${auth}"`)
+  strBuilder.push(`  - protocolVersion: ${protocolVersion}`)
+  // write SS
+  let ssBuilder = []
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let user = decoder.readVarUint()
+    let clock = decoder.readVarUint()
+    ssBuilder.push(`(${user}:${clock})`)
+  }
+  strBuilder.push('  == SS: ' + ssBuilder.join(','))
+}
+
+export function sendSyncStep1 (connector, syncUser) {
+  let encoder = new BinaryEncoder()
+  encoder.writeVarString(connector.y.room)
+  encoder.writeVarString('sync step 1')
+  encoder.writeVarString(connector.authInfo || '')
+  encoder.writeVarUint(connector.protocolVersion)
+  writeStateSet(connector.y, encoder)
+  connector.send(syncUser, encoder.createBuffer())
+}
+
+export default function writeStructs (encoder, decoder, y, ss) {
+  const lenPos = encoder.pos
+  encoder.writeUint32(0)
+  let len = 0
+  for (let user of y.ss.state.keys()) {
+    let clock = ss.get(user) || 0
+    if (user !== RootFakeUserID) {
+      y.os.iterate(new ID(user, clock), new ID(user, Number.MAX_VALUE), function (struct) {
+        struct._toBinary(encoder)
+        len++
+      })
+    }
+  }
+  encoder.setUint32(lenPos, len)
+}
+
+export function readSyncStep1 (decoder, encoder, y, senderConn, sender) {
+  let protocolVersion = decoder.readVarUint()
+  // check protocol version
+  if (protocolVersion !== y.connector.protocolVersion) {
+    console.warn(
+      `You tried to sync with a Yjs instance that has a different protocol version
+      (You: ${protocolVersion}, Client: ${protocolVersion}).
+      `)
+    y.destroy()
+  }
+  // write sync step 2
+  encoder.writeVarString('sync step 2')
+  encoder.writeVarString(y.connector.authInfo || '')
+  const ss = readStateSet(decoder)
+  writeStructs(encoder, decoder, y, ss)
+  writeDeleteSet(y, encoder)
+  y.connector.send(senderConn.uid, encoder.createBuffer())
+  senderConn.receivedSyncStep2 = true
+  if (y.connector.role === 'slave') {
+    sendSyncStep1(y.connector, sender)
+  }
+}

src/MessageHandler/syncStep2.js

@@ -0,0 +1,28 @@
+import { stringifyStructs, integrateRemoteStructs } from './integrateRemoteStructs.js'
+import { readDeleteSet } from './deleteSet.js'
+
+export function stringifySyncStep2 (y, decoder, strBuilder) {
+  strBuilder.push('     - auth: ' + decoder.readVarString())
+  strBuilder.push('  == OS:')
+  stringifyStructs(y, decoder, strBuilder)
+  // write DS to string
+  strBuilder.push('  == DS:')
+  let len = decoder.readUint32()
+  for (let i = 0; i < len; i++) {
+    let user = decoder.readVarUint()
+    strBuilder.push(`    User: ${user}: `)
+    let len2 = decoder.readUint32()
+    for (let j = 0; j < len2; j++) {
+      let from = decoder.readVarUint()
+      let to = decoder.readVarUint()
+      let gc = decoder.readUint8() === 1
+      strBuilder.push(`[${from}, ${to}, ${gc}]`)
+    }
+  }
+}
+
+export function readSyncStep2 (decoder, encoder, y, senderConn, sender) {
+  integrateRemoteStructs(decoder, encoder, y)
+  readDeleteSet(y, decoder)
+  y.connector._setSyncedWith(sender)
+}

src/Notes.md

@@ -1,12 +0,0 @@
-
-# Notes
-
-### Terminology
-
-* DB: DataBase that holds all the information of the shared object. It is devided into the OS, DS, and SS. This can be a persistent database or an in-memory database. Depending on the type of database, it could make sense to store OS, DS, and SS in different tables, or maybe different databases.
-* OS: OperationStore holds all the operations. An operation is a js object with a fixed number of name fields.
-* DS: DeleteStore holds the information about which operations are deleted and which operations were garbage collected (no longer available in the OS).
-* SS: StateSet holds the current state of the OS. SS.getState(username) refers to the amount of operations that were received by that respective user.
-* Op: Operation defines an action on a shared type. But it is also the format in which we store the model of a type. This is why it is also called a Struct/Structure.
-* Type and Structure: We crearly distinguish between type and structure. Short explanation: A type (e.g. Strings, Numbers) have a number of functions that you can apply on them. (+) is well defined on both of them. They are *modeled* by a structure - the functions really change the structure of a type. Types can be implemented differently but still provide the same functionality. In Yjs, almost all types are realized as a doubly linked list (on which Yjs can provide eventual convergence)
-*

src/Persistence.js

@@ -1,4 +1,4 @@
-import { BinaryEncoder } from './Encoding.js'
+// import BinaryEncoder from './Binary/Encoder.js'
 
 export default function extendPersistence (Y) {
   class AbstractPersistence {
@@ -8,13 +8,16 @@ export default function extendPersistence (Y) {
       this.saveOperationsBuffer = []
       this.log = Y.debug('y:persistence')
     }
+
     saveToMessageQueue (binary) {
       this.log('Room %s: Save message to message queue', this.y.options.connector.room)
     }
+
     saveOperations (ops) {
       ops = ops.map(function (op) {
         return Y.Struct[op.struct].encode(op)
       })
+      /*
       const saveOperations = () => {
         if (this.saveOperationsBuffer.length > 0) {
           let encoder = new BinaryEncoder()
@@ -31,13 +34,14 @@ export default function extendPersistence (Y) {
           this.saveToMessageQueue(encoder.createBuffer())
         }
       }
+      */
       if (this.saveOperationsBuffer.length === 0) {
         this.saveOperationsBuffer = ops
-        this.y.db.whenTransactionsFinished().then(saveOperations)
       } else {
         this.saveOperationsBuffer = this.saveOperationsBuffer.concat(ops)
       }
     }
   }
+
   Y.AbstractPersistence = AbstractPersistence
 }

src/Store/DeleteStore.js

@@ -0,0 +1,125 @@
+import Tree from '../Util/Tree.js'
+import ID from '../Util/ID.js'
+
+class DSNode {
+  constructor (id, len, gc) {
+    this._id = id
+    this.len = len
+    this.gc = gc
+  }
+  clone () {
+    return new DSNode(this._id, this.len, this.gc)
+  }
+}
+
+export default class DeleteStore extends Tree {
+  logTable () {
+    const deletes = []
+    this.iterate(null, null, function (n) {
+      deletes.push({
+        user: n._id.user,
+        clock: n._id.clock,
+        len: n.len,
+        gc: n.gc
+      })
+    })
+    console.table(deletes)
+  }
+  isDeleted (id) {
+    var n = this.findWithUpperBound(id)
+    return n !== null && n._id.user === id.user && id.clock < n._id.clock + n.len
+  }
+  /*
+   * Mark an operation as deleted. returns the deleted node
+   */
+  markDeleted (id, length) {
+    if (length == null) {
+      throw new Error('length must be defined')
+    }
+    var n = this.findWithUpperBound(id)
+    if (n != null && n._id.user === id.user) {
+      if (n._id.clock <= id.clock && id.clock <= n._id.clock + n.len) {
+        // id is in n's range
+        var diff = id.clock + length - (n._id.clock + n.len) // overlapping right
+        if (diff > 0) {
+          // id+length overlaps n
+          if (!n.gc) {
+            n.len += diff
+          } else {
+            diff = n._id.clock + n.len - id.clock // overlapping left (id till n.end)
+            if (diff < length) {
+              // a partial deletion
+              let nId = id.clone()
+              nId.clock += diff
+              n = new DSNode(nId, length - diff, false)
+              this.put(n)
+            } else {
+              // already gc'd
+              throw new Error(
+                'DS reached an inconsistent state. Please report this issue!'
+              )
+            }
+          }
+        } else {
+          // no overlapping, already deleted
+          return n
+        }
+      } else {
+        // cannot extend left (there is no left!)
+        n = new DSNode(id, length, false)
+        this.put(n) // TODO: you double-put !!
+      }
+    } else {
+      // cannot extend left
+      n = new DSNode(id, length, false)
+      this.put(n)
+    }
+    // can extend right?
+    var next = this.findNext(n._id)
+    if (
+      next != null &&
+      n._id.user === next._id.user &&
+      n._id.clock + n.len >= next._id.clock
+    ) {
+      diff = n._id.clock + n.len - next._id.clock // from next.start to n.end
+      while (diff >= 0) {
+        // n overlaps with next
+        if (next.gc) {
+          // gc is stronger, so reduce length of n
+          n.len -= diff
+          if (diff >= next.len) {
+            // delete the missing range after next
+            diff = diff - next.len // missing range after next
+            if (diff > 0) {
+              this.put(n) // unneccessary? TODO!
+              this.markDeleted(new ID(next._id.user, next._id.clock + next.len), diff)
+            }
+          }
+          break
+        } else {
+          // we can extend n with next
+          if (diff > next.len) {
+            // n is even longer than next
+            // get next.next, and try to extend it
+            var _next = this.findNext(next._id)
+            this.delete(next._id)
+            if (_next == null || n._id.user !== _next._id.user) {
+              break
+            } else {
+              next = _next
+              diff = n._id.clock + n.len - next._id.clock // from next.start to n.end
+              // continue!
+            }
+          } else {
+            // n just partially overlaps with next. extend n, delete next, and break this loop
+            n.len += next.len - diff
+            this.delete(next._id)
+            break
+          }
+        }
+      }
+    }
+    this.put(n)
+    return n
+  }
+}

src/Store/OperationStore.js

@@ -0,0 +1,85 @@
+import Tree from '../Util/Tree.js'
+import RootID from '../Util/RootID.js'
+import { getStruct } from '../Util/structReferences.js'
+import { logID } from '../MessageHandler/messageToString.js'
+
+export default class OperationStore extends Tree {
+  constructor (y) {
+    super()
+    this.y = y
+  }
+  logTable () {
+    const items = []
+    this.iterate(null, null, function (item) {
+      items.push({
+        id: logID(item),
+        origin: logID(item._origin === null ? null : item._origin._lastId),
+        left: logID(item._left === null ? null : item._left._lastId),
+        right: logID(item._right),
+        right_origin: logID(item._right_origin),
+        parent: logID(item._parent),
+        parentSub: item._parentSub,
+        deleted: item._deleted,
+        content: JSON.stringify(item._content)
+      })
+    })
+    console.table(items)
+  }
+  get (id) {
+    let struct = this.find(id)
+    if (struct === null && id instanceof RootID) {
+      const Constr = getStruct(id.type)
+      const y = this.y
+      struct = new Constr()
+      struct._id = id
+      struct._parent = y
+      y.transact(() => {
+        struct._integrate(y)
+      })
+      this.put(struct)
+    }
+    return struct
+  }
+  // Use getItem for structs with _length > 1
+  getItem (id) {
+    var item = this.findWithUpperBound(id)
+    if (item === null) {
+      return null
+    }
+    const itemID = item._id
+    if (id.user === itemID.user && id.clock < itemID.clock + item._length) {
+      return item
+    } else {
+      return null
+    }
+  }
+  // Return an insertion such that id is the first element of content
+  // This function manipulates an item, if necessary
+  getItemCleanStart (id) {
+    var ins = this.getItem(id)
+    if (ins === null || ins._length === 1) {
+      return ins
+    }
+    const insID = ins._id
+    if (insID.clock === id.clock) {
+      return ins
+    } else {
+      return ins._splitAt(this.y, id.clock - insID.clock)
+    }
+  }
+  // Return an insertion such that id is the last element of content
+  // This function manipulates an operation, if necessary
+  getItemCleanEnd (id) {
+    var ins = this.getItem(id)
+    if (ins === null || ins._length === 1) {
+      return ins
+    }
+    const insID = ins._id
+    if (insID.clock + ins._length - 1 === id.clock) {
+      return ins
+    } else {
+      ins._splitAt(this.y, id.clock - insID.clock + 1)
+      return ins
+    }
+  }
+}

src/Store/StateStore.js

@@ -0,0 +1,47 @@
+import ID from '../Util/ID.js'
+
+export default class StateStore {
+  constructor (y) {
+    this.y = y
+    this.state = new Map()
+  }
+  logTable () {
+    const entries = []
+    for (let [user, state] of this.state) {
+      entries.push({
+        user, state
+      })
+    }
+    console.table(entries)
+  }
+  getNextID (len) {
+    const user = this.y.userID
+    const state = this.getState(user)
+    this.setState(user, state + len)
+    return new ID(user, state)
+  }
+  updateRemoteState (struct) {
+    let user = struct._id.user
+    let userState = this.state.get(user)
+    while (struct !== null && struct._id.clock === userState) {
+      userState += struct._length
+      struct = this.y.os.get(new ID(user, userState))
+    }
+    this.state.set(user, userState)
+  }
+  getState (user) {
+    let state = this.state.get(user)
+    if (state == null) {
+      return 0
+    }
+    return state
+  }
+  setState (user, state) {
+    // TODO: modify missingi structs here
+    const beforeState = this.y._transaction.beforeState
+    if (!beforeState.has(user)) {
+      beforeState.set(user, this.getState(user))
+    }
+    this.state.set(user, state)
+  }
+}

src/Struct.js

@@ -1,619 +0,0 @@
-const CDELETE = 0
-const CINSERT = 1
-const CLIST = 2
-const CMAP = 3
-const CXML = 4
-
-/*
- An operation also defines the structure of a type. This is why operation and
- structure are used interchangeably here.
-
- It must be of the type Object. I hope to achieve some performance
- improvements when working on databases that support the json format.
-
- An operation must have the following properties:
-
- * encode
-     - Encode the structure in a readable format (preferably string- todo)
- * decode (todo)
-     - decode structure to json
- * execute
-     - Execute the semantics of an operation.
- * requiredOps
-     - Operations that are required to execute this operation.
-*/
-export default function extendStruct (Y) {
-  let Struct = {}
-  Y.Struct = Struct
-  Struct.binaryDecodeOperation = function (decoder) {
-    let code = decoder.peekUint8()
-    if (code === CDELETE) {
-      return Struct.Delete.binaryDecode(decoder)
-    } else if (code === CINSERT) {
-      return Struct.Insert.binaryDecode(decoder)
-    } else if (code === CLIST) {
-      return Struct.List.binaryDecode(decoder)
-    } else if (code === CMAP) {
-      return Struct.Map.binaryDecode(decoder)
-    } else if (code === CXML) {
-      return Struct.Xml.binaryDecode(decoder)
-    } else {
-      throw new Error('Unable to decode operation!')
-    }
-  }
-
-  /* This is the only operation that is actually not a structure, because
-  it is not stored in the OS. This is why it _does not_ have an id
-
-  op = {
-    target: Id
-  }
-  */
-  Struct.Delete = {
-    encode: function (op) {
-      return {
-        target: op.target,
-        length: op.length || 0,
-        struct: 'Delete'
-      }
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CDELETE)
-      encoder.writeOpID(op.target)
-      encoder.writeVarUint(op.length || 0)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      return {
-        target: decoder.readOpID(),
-        length: decoder.readVarUint(),
-        struct: 'Delete'
-      }
-    },
-    requiredOps: function (op) {
-      return [] // [op.target]
-    },
-    execute: function * (op) {
-      return yield * this.deleteOperation(op.target, op.length || 1)
-    }
-  }
-
-  /* {
-      content: [any],
-      opContent: Id,
-      id: Id,
-      left: Id,
-      origin: Id,
-      right: Id,
-      parent: Id,
-      parentSub: string (optional), // child of Map type
-    }
-  */
-  Struct.Insert = {
-    encode: function (op/* :Insertion */) /* :Insertion */ {
-      // TODO: you could not send the "left" property, then you also have to
-      // "op.left = null" in $execute or $decode
-      var e/* :any */ = {
-        id: op.id,
-        left: op.left,
-        right: op.right,
-        origin: op.origin,
-        parent: op.parent,
-        struct: op.struct
-      }
-      if (op.parentSub != null) {
-        e.parentSub = op.parentSub
-      }
-      if (op.hasOwnProperty('opContent')) {
-        e.opContent = op.opContent
-      } else {
-        e.content = op.content.slice()
-      }
-
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CINSERT)
-      // compute info property
-      let contentIsText = op.content != null && op.content.every(c => typeof c === 'string' && c.length === 1)
-      let originIsLeft = Y.utils.compareIds(op.left, op.origin)
-      let info =
-        (op.parentSub != null ? 1 : 0) |
-        (op.opContent != null ? 2 : 0) |
-        (contentIsText ? 4 : 0) |
-        (originIsLeft ? 8 : 0) |
-        (op.left != null ? 16 : 0) |
-        (op.right != null ? 32 : 0) |
-        (op.origin != null ? 64 : 0)
-      encoder.writeUint8(info)
-      encoder.writeOpID(op.id)
-      encoder.writeOpID(op.parent)
-      if (info & 16) {
-        encoder.writeOpID(op.left)
-      }
-      if (info & 32) {
-        encoder.writeOpID(op.right)
-      }
-      if (!originIsLeft && info & 64) {
-        encoder.writeOpID(op.origin)
-      }
-      if (info & 1) {
-        // write parentSub
-        encoder.writeVarString(op.parentSub)
-      }
-      if (info & 2) {
-        // write opContent
-        encoder.writeOpID(op.opContent)
-      } else if (info & 4) {
-        // write text
-        encoder.writeVarString(op.content.join(''))
-      } else {
-        // convert to JSON and write
-        encoder.writeVarString(JSON.stringify(op.content))
-      }
-    },
-    binaryDecode: function (decoder) {
-      let op = {
-        struct: 'Insert'
-      }
-      decoder.skip8()
-      // get info property
-      let info = decoder.readUint8()
-
-      op.id = decoder.readOpID()
-      op.parent = decoder.readOpID()
-      if (info & 16) {
-        op.left = decoder.readOpID()
-      } else {
-        op.left = null
-      }
-      if (info & 32) {
-        op.right = decoder.readOpID()
-      } else {
-        op.right = null
-      }
-      if (info & 8) {
-        // origin is left
-        op.origin = op.left
-      } else if (info & 64) {
-        op.origin = decoder.readOpID()
-      } else {
-        op.origin = null
-      }
-      if (info & 1) {
-        // has parentSub
-        op.parentSub = decoder.readVarString()
-      }
-      if (info & 2) {
-        // has opContent
-        op.opContent = decoder.readOpID()
-      } else if (info & 4) {
-        // has pure text content
-        op.content = decoder.readVarString().split('')
-      } else {
-        // has mixed content
-        let s = decoder.readVarString()
-        op.content = JSON.parse(s)
-      }
-      return op
-    },
-    requiredOps: function (op) {
-      var ids = []
-      if (op.left != null) {
-        ids.push(op.left)
-      }
-      if (op.right != null) {
-        ids.push(op.right)
-      }
-      if (op.origin != null && !Y.utils.compareIds(op.left, op.origin)) {
-        ids.push(op.origin)
-      }
-      // if (op.right == null && op.left == null) {
-      ids.push(op.parent)
-
-      if (op.opContent != null) {
-        ids.push(op.opContent)
-      }
-      return ids
-    },
-    getDistanceToOrigin: function * (op) {
-      if (op.left == null) {
-        return 0
-      } else {
-        var d = 0
-        var o = yield * this.getInsertion(op.left)
-        while (!Y.utils.matchesId(o, op.origin)) {
-          d++
-          if (o.left == null) {
-            break
-          } else {
-            o = yield * this.getInsertion(o.left)
-          }
-        }
-        return d
-      }
-    },
-    /*
-    # $this has to find a unique position between origin and the next known character
-    # case 1: $origin equals $o.origin: the $creator parameter decides if left or right
-    #         let $OL= [o1,o2,o3,o4], whereby $this is to be inserted between o1 and o4
-    #         o2,o3 and o4 origin is 1 (the position of o2)
-    #         there is the case that $this.creator < o2.creator, but o3.creator < $this.creator
-    #         then o2 knows o3. Since on another client $OL could be [o1,o3,o4] the problem is complex
-    #         therefore $this would be always to the right of o3
-    # case 2: $origin < $o.origin
-    #         if current $this insert_position > $o origin: $this ins
-    #         else $insert_position will not change
-    #         (maybe we encounter case 1 later, then this will be to the right of $o)
-    # case 3: $origin > $o.origin
-    #         $this insert_position is to the left of $o (forever!)
-    */
-    execute: function * (op) {
-      var i // loop counter
-
-      // during this function some ops may get split into two pieces (e.g. with getInsertionCleanEnd)
-      // We try to merge them later, if possible
-      var tryToRemergeLater = []
-
-      if (op.origin != null) { // TODO: !== instead of !=
-        // we save in origin that op originates in it
-        // we need that later when we eventually garbage collect origin (see transaction)
-        var origin = yield * this.getInsertionCleanEnd(op.origin)
-        if (origin.originOf == null) {
-          origin.originOf = []
-        }
-        origin.originOf.push(op.id)
-        yield * this.setOperation(origin)
-        if (origin.right != null) {
-          tryToRemergeLater.push(origin.right)
-        }
-      }
-      var distanceToOrigin = i = yield * Struct.Insert.getDistanceToOrigin.call(this, op) // most cases: 0 (starts from 0)
-
-      // now we begin to insert op in the list of insertions..
-      var o
-      var parent
-      var start
-
-      // find o. o is the first conflicting operation
-      if (op.left != null) {
-        o = yield * this.getInsertionCleanEnd(op.left)
-        if (!Y.utils.compareIds(op.left, op.origin) && o.right != null) {
-          // only if not added previously
-          tryToRemergeLater.push(o.right)
-        }
-        o = (o.right == null) ? null : yield * this.getOperation(o.right)
-      } else { // left == null
-        parent = yield * this.getOperation(op.parent)
-        let startId = op.parentSub ? parent.map[op.parentSub] : parent.start
-        start = startId == null ? null : yield * this.getOperation(startId)
-        o = start
-      }
-
-      // make sure to split op.right if necessary (also add to tryCombineWithLeft)
-      if (op.right != null) {
-        tryToRemergeLater.push(op.right)
-        yield * this.getInsertionCleanStart(op.right)
-      }
-
-      // handle conflicts
-      while (true) {
-        if (o != null && !Y.utils.compareIds(o.id, op.right)) {
-          var oOriginDistance = yield * Struct.Insert.getDistanceToOrigin.call(this, o)
-          if (oOriginDistance === i) {
-            // case 1
-            if (o.id[0] < op.id[0]) {
-              op.left = Y.utils.getLastId(o)
-              distanceToOrigin = i + 1 // just ignore o.content.length, doesn't make a difference
-            }
-          } else if (oOriginDistance < i) {
-            // case 2
-            if (i - distanceToOrigin <= oOriginDistance) {
-              op.left = Y.utils.getLastId(o)
-              distanceToOrigin = i + 1 // just ignore o.content.length, doesn't make a difference
-            }
-          } else {
-            break
-          }
-          i++
-          if (o.right != null) {
-            o = yield * this.getInsertion(o.right)
-          } else {
-            o = null
-          }
-        } else {
-          break
-        }
-      }
-
-      // reconnect..
-      var left = null
-      var right = null
-      if (parent == null) {
-        parent = yield * this.getOperation(op.parent)
-      }
-
-      // reconnect left and set right of op
-      if (op.left != null) {
-        left = yield * this.getInsertion(op.left)
-        // link left
-        op.right = left.right
-        left.right = op.id
-
-        yield * this.setOperation(left)
-      } else {
-        // set op.right from parent, if necessary
-        op.right = op.parentSub ? parent.map[op.parentSub] || null : parent.start
-      }
-      // reconnect right
-      if (op.right != null) {
-        // TODO: wanna connect right too?
-        right = yield * this.getOperation(op.right)
-        right.left = Y.utils.getLastId(op)
-
-        // if right exists, and it is supposed to be gc'd. Remove it from the gc
-        if (right.gc != null) {
-          if (right.content != null && right.content.length > 1) {
-            right = yield * this.getInsertionCleanEnd(right.id)
-          }
-          this.store.removeFromGarbageCollector(right)
-        }
-        yield * this.setOperation(right)
-      }
-
-      // update parents .map/start/end properties
-      if (op.parentSub != null) {
-        if (left == null) {
-          parent.map[op.parentSub] = op.id
-          yield * this.setOperation(parent)
-        }
-        // is a child of a map struct.
-        // Then also make sure that only the most left element is not deleted
-        // We do not call the type in this case (this is what the third parameter is for)
-        if (op.right != null) {
-          yield * this.deleteOperation(op.right, 1, true)
-        }
-        if (op.left != null) {
-          yield * this.deleteOperation(op.id, 1, true)
-        }
-      } else {
-        if (right == null || left == null) {
-          if (right == null) {
-            parent.end = Y.utils.getLastId(op)
-          }
-          if (left == null) {
-            parent.start = op.id
-          }
-          yield * this.setOperation(parent)
-        }
-      }
-
-      // try to merge original op.left and op.origin
-      for (i = 0; i < tryToRemergeLater.length; i++) {
-        var m = yield * this.getOperation(tryToRemergeLater[i])
-        yield * this.tryCombineWithLeft(m)
-      }
-    }
-  }
-
-  /*
-  {
-    start: null,
-    end: null,
-    struct: "List",
-    type: "",
-    id: this.os.getNextOpId(1)
-  }
-  */
-  Struct.List = {
-    create: function (id) {
-      return {
-        start: null,
-        end: null,
-        struct: 'List',
-        id: id
-      }
-    },
-    encode: function (op) {
-      var e = {
-        struct: 'List',
-        id: op.id,
-        type: op.type
-      }
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CLIST)
-      encoder.writeOpID(op.id)
-      encoder.writeVarString(op.type)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      let op = {
-        id: decoder.readOpID(),
-        type: decoder.readVarString(),
-        struct: 'List',
-        start: null,
-        end: null
-      }
-      return op
-    },
-    requiredOps: function () {
-      /*
-      var ids = []
-      if (op.start != null) {
-        ids.push(op.start)
-      }
-      if (op.end != null){
-        ids.push(op.end)
-      }
-      return ids
-      */
-      return []
-    },
-    execute: function * (op) {
-      op.start = null
-      op.end = null
-    },
-    ref: function * (op, pos) {
-      if (op.start == null) {
-        return null
-      }
-      var res = null
-      var o = yield * this.getOperation(op.start)
-
-      while (true) {
-        if (!o.deleted) {
-          res = o
-          pos--
-        }
-        if (pos >= 0 && o.right != null) {
-          o = yield * this.getOperation(o.right)
-        } else {
-          break
-        }
-      }
-      return res
-    },
-    map: function * (o, f) {
-      o = o.start
-      var res = []
-      while (o != null) { // TODO: change to != (at least some convention)
-        var operation = yield * this.getOperation(o)
-        if (!operation.deleted) {
-          res.push(f(operation))
-        }
-        o = operation.right
-      }
-      return res
-    }
-  }
-
-  /*
-    {
-      map: {},
-      struct: "Map",
-      type: "",
-      id: this.os.getNextOpId(1)
-    }
-  */
-  Struct.Map = {
-    create: function (id) {
-      return {
-        id: id,
-        map: {},
-        struct: 'Map'
-      }
-    },
-    encode: function (op) {
-      var e = {
-        struct: 'Map',
-        type: op.type,
-        id: op.id,
-        map: {} // overwrite map!!
-      }
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CMAP)
-      encoder.writeOpID(op.id)
-      encoder.writeVarString(op.type)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      let op = {
-        id: decoder.readOpID(),
-        type: decoder.readVarString(),
-        struct: 'Map',
-        map: {}
-      }
-      return op
-    },
-    requiredOps: function () {
-      return []
-    },
-    execute: function * (op) {
-      op.start = null
-      op.end = null
-    },
-    /*
-      Get a property by name
-    */
-    get: function * (op, name) {
-      var oid = op.map[name]
-      if (oid != null) {
-        var res = yield * this.getOperation(oid)
-        if (res == null || res.deleted) {
-          return void 0
-        } else if (res.opContent == null) {
-          return res.content[0]
-        } else {
-          return yield * this.getType(res.opContent)
-        }
-      }
-    }
-  }
-
-  /*
-    {
-      map: {},
-      start: null,
-      end: null,
-      struct: "Xml",
-      type: "",
-      id: this.os.getNextOpId(1)
-    }
-  */
-  Struct.Xml = {
-    create: function (id, args) {
-      let nodeName = args != null ? args.nodeName : null
-      return {
-        id: id,
-        map: {},
-        start: null,
-        end: null,
-        struct: 'Xml',
-        nodeName
-      }
-    },
-    encode: function (op) {
-      var e = {
-        struct: 'Xml',
-        type: op.type,
-        id: op.id,
-        map: {},
-        nodeName: op.nodeName
-      }
-      return e
-    },
-    binaryEncode: function (encoder, op) {
-      encoder.writeUint8(CXML)
-      encoder.writeOpID(op.id)
-      encoder.writeVarString(op.type)
-      encoder.writeVarString(op.nodeName)
-    },
-    binaryDecode: function (decoder) {
-      decoder.skip8()
-      let op = {
-        id: decoder.readOpID(),
-        type: decoder.readVarString(),
-        struct: 'Xml',
-        map: {},
-        start: null,
-        end: null,
-        nodeName: decoder.readVarString()
-      }
-      return op
-    },
-    requiredOps: function () {
-      return []
-    },
-    execute: function * () {},
-    ref: Struct.List.ref,
-    map: Struct.List.map,
-    /*
-      Get a property by name
-    */
-    get: Struct.Map.get
-  }
-}

src/Struct/Delete.js

@@ -0,0 +1,84 @@
+import { getReference } from '../Util/structReferences.js'
+import ID from '../Util/ID.js'
+import { logID } from '../MessageHandler/messageToString.js'
+
+/**
+ * Delete all items in an ID-range
+ * TODO: implement getItemCleanStartNode for better performance (only one lookup)
+ */
+export function deleteItemRange (y, user, clock, range) {
+  const createDelete = y.connector._forwardAppliedStructs
+  let item = y.os.getItemCleanStart(new ID(user, clock))
+  if (item !== null) {
+    if (!item._deleted) {
+      item._splitAt(y, range)
+      item._delete(y, createDelete)
+    }
+    let itemLen = item._length
+    range -= itemLen
+    clock += itemLen
+    if (range > 0) {
+      let node = y.os.findNode(new ID(user, clock))
+      while (node !== null && range > 0 && node.val._id.equals(new ID(user, clock))) {
+        const nodeVal = node.val
+        if (!nodeVal._deleted) {
+          nodeVal._splitAt(y, range)
+          nodeVal._delete(y, createDelete)
+        }
+        const nodeLen = nodeVal._length
+        range -= nodeLen
+        clock += nodeLen
+        node = node.next()
+      }
+    }
+  }
+}
+
+/**
+ * Delete is not a real struct. It will not be saved in OS
+ */
+export default class Delete {
+  constructor () {
+    this._target = null
+    this._length = null
+  }
+  _fromBinary (y, decoder) {
+    // TODO: set target, and add it to missing if not found
+    // There is an edge case in p2p networks!
+    const targetID = decoder.readID()
+    this._targetID = targetID
+    this._length = decoder.readVarUint()
+    if (y.os.getItem(targetID) === null) {
+      return [targetID]
+    } else {
+      return []
+    }
+  }
+  _toBinary (encoder) {
+    encoder.writeUint8(getReference(this.constructor))
+    encoder.writeID(this._targetID)
+    encoder.writeVarUint(this._length)
+  }
+  /**
+   * - If created remotely (a remote user deleted something),
+   *   this Delete is applied to all structs in id-range.
+   * - If created lokally (e.g. when y-array deletes a range of elements),
+   *   this struct is broadcasted only (it is already executed)
+   */
+  _integrate (y, locallyCreated = false) {
+    if (!locallyCreated) {
+      // from remote
+      const id = this._targetID
+      deleteItemRange(y, id.user, id.clock, this._length)
+    } else {
+      // from local
+      y.connector.broadcastStruct(this)
+    }
+    if (y.persistence !== null) {
+      y.persistence.saveOperations(this)
+    }
+  }
+  _logString () {
+    return `Delete - target: ${logID(this._targetID)}, len: ${this._length}`
+  }
+}

src/Struct/Item.js

@@ -0,0 +1,327 @@
+import { getReference } from '../Util/structReferences.js'
+import ID from '../Util/ID.js'
+import { RootFakeUserID } from '../Util/RootID.js'
+import Delete from './Delete.js'
+import { transactionTypeChanged } from '../Transaction.js'
+
+/**
+ * Helper utility to split an Item (see _splitAt)
+ * - copy all properties from a to b
+ * - connect a to b
+ * - assigns the correct _id
+ * - save b to os
+ */
+export function splitHelper (y, a, b, diff) {
+  const aID = a._id
+  b._id = new ID(aID.user, aID.clock + diff)
+  b._origin = a
+  b._left = a
+  b._right = a._right
+  if (b._right !== null) {
+    b._right._left = b
+  }
+  b._right_origin = a._right_origin
+  // do not set a._right_origin, as this will lead to problems when syncing
+  a._right = b
+  b._parent = a._parent
+  b._parentSub = a._parentSub
+  b._deleted = a._deleted
+  // now search all relevant items to the right and update origin
+  // if origin is not it foundOrigins, we don't have to search any longer
+  let foundOrigins = new Set()
+  foundOrigins.add(a)
+  let o = b._right
+  while (o !== null && foundOrigins.has(o._origin)) {
+    if (o._origin === a) {
+      o._origin = b
+    }
+    foundOrigins.add(o)
+    o = o._right
+  }
+  y.os.put(b)
+}
+
+export default class Item {
+  constructor () {
+    this._id = null
+    this._origin = null
+    this._left = null
+    this._right = null
+    this._right_origin = null
+    this._parent = null
+    this._parentSub = null
+    this._deleted = false
+  }
+  /**
+   * Copy the effect of struct
+   */
+  _copy () {
+    let struct = new this.constructor()
+    struct._origin = this._left
+    struct._left = this._left
+    struct._right = this
+    struct._right_origin = this
+    struct._parent = this._parent
+    struct._parentSub = this._parentSub
+    return struct
+  }
+  get _lastId () {
+    return new ID(this._id.user, this._id.clock + this._length - 1)
+  }
+  get _length () {
+    return 1
+  }
+  /**
+   * Splits this struct so that another struct can be inserted in-between.
+   * This must be overwritten if _length > 1
+   * Returns right part after split
+   * - diff === 0 => this
+   * - diff === length => this._right
+   * - otherwise => split _content and return right part of split
+   * (see ItemJSON/ItemString for implementation)
+   */
+  _splitAt (y, diff) {
+    if (diff === 0) {
+      return this
+    }
+    return this._right
+  }
+  _delete (y, createDelete = true) {
+    this._deleted = true
+    y.ds.markDeleted(this._id, this._length)
+    if (createDelete) {
+      let del = new Delete()
+      del._targetID = this._id
+      del._length = this._length
+      del._integrate(y, true)
+    }
+    transactionTypeChanged(y, this._parent, this._parentSub)
+    y._transaction.deletedStructs.add(this)
+  }
+  /**
+   * This is called right before this struct receives any children.
+   * It can be overwritten to apply pending changes before applying remote changes
+   */
+  _beforeChange () {
+    // nop
+  }
+  /*
+   * - Integrate the struct so that other types/structs can see it
+   * - Add this struct to y.os
+   * - Check if this is struct deleted
+   */
+  _integrate (y) {
+    const parent = this._parent
+    const selfID = this._id
+    const userState = selfID === null ? 0 : y.ss.getState(selfID.user)
+    if (selfID === null) {
+      this._id = y.ss.getNextID(this._length)
+    } else if (selfID.user === RootFakeUserID) {
+      // nop
+    } else if (selfID.clock < userState) {
+      // already applied..
+      return []
+    } else if (selfID.clock === userState) {
+      y.ss.setState(selfID.user, userState + this._length)
+    } else {
+      // missing content from user
+      throw new Error('Can not apply yet!')
+    }
+    if (!parent._deleted && !y._transaction.changedTypes.has(parent) && !y._transaction.newTypes.has(parent)) {
+      // this is the first time parent is updated
+      // or this types is new
+      this._parent._beforeChange()
+    }
+    /*
+    # $this has to find a unique position between origin and the next known character
+    # case 1: $origin equals $o.origin: the $creator parameter decides if left or right
+    #         let $OL= [o1,o2,o3,o4], whereby $this is to be inserted between o1 and o4
+    #         o2,o3 and o4 origin is 1 (the position of o2)
+    #         there is the case that $this.creator < o2.creator, but o3.creator < $this.creator
+    #         then o2 knows o3. Since on another client $OL could be [o1,o3,o4] the problem is complex
+    #         therefore $this would be always to the right of o3
+    # case 2: $origin < $o.origin
+    #         if current $this insert_position > $o origin: $this ins
+    #         else $insert_position will not change
+    #         (maybe we encounter case 1 later, then this will be to the right of $o)
+    # case 3: $origin > $o.origin
+    #         $this insert_position is to the left of $o (forever!)
+    */
+    // handle conflicts
+    let o
+    // set o to the first conflicting item
+    if (this._left !== null) {
+      o = this._left._right
+    } else if (this._parentSub !== null) {
+      o = this._parent._map.get(this._parentSub) || null
+    } else {
+      o = this._parent._start
+    }
+    let conflictingItems = new Set()
+    let itemsBeforeOrigin = new Set()
+    // Let c in conflictingItems, b in itemsBeforeOrigin
+    // ***{origin}bbbb{this}{c,b}{c,b}{o}***
+    // Note that conflictingItems is a subset of itemsBeforeOrigin
+    while (o !== null && o !== this._right) {
+      itemsBeforeOrigin.add(o)
+      conflictingItems.add(o)
+      if (this._origin === o._origin) {
+        // case 1
+        if (o._id.user < this._id.user) {
+          this._left = o
+          conflictingItems.clear()
+        }
+      } else if (itemsBeforeOrigin.has(o._origin)) {
+        // case 2
+        if (!conflictingItems.has(o._origin)) {
+          this._left = o
+          conflictingItems.clear()
+        }
+      } else {
+        break
+      }
+      // TODO: try to use right_origin instead.
+      // Then you could basically omit conflictingItems!
+      // Note: you probably can't use right_origin in every case.. only when setting _left
+      o = o._right
+    }
+    // reconnect left/right + update parent map/start if necessary
+    const parentSub = this._parentSub
+    if (this._left === null) {
+      let right
+      if (parentSub !== null) {
+        const pmap = parent._map
+        right = pmap.get(parentSub) || null
+        pmap.set(parentSub, this)
+      } else {
+        right = parent._start
+        parent._start = this
+      }
+      this._right = right
+      if (right !== null) {
+        right._left = this
+      }
+    } else {
+      const left = this._left
+      const right = left._right
+      this._right = right
+      left._right = this
+      if (right !== null) {
+        right._left = this
+      }
+    }
+    if (parent._deleted) {
+      this._delete(y, false)
+    }
+    y.os.put(this)
+    transactionTypeChanged(y, parent, parentSub)
+    if (this._id.user !== RootFakeUserID) {
+      if (y.connector._forwardAppliedStructs || this._id.user === y.userID) {
+        y.connector.broadcastStruct(this)
+      }
+      if (y.persistence !== null) {
+        y.persistence.saveOperations(this)
+      }
+    }
+  }
+  _toBinary (encoder) {
+    encoder.writeUint8(getReference(this.constructor))
+    let info = 0
+    if (this._origin !== null) {
+      info += 0b1 // origin is defined
+    }
+    // TODO: remove
+    /* no longer send _left
+    if (this._left !== this._origin) {
+      info += 0b10 // do not copy origin to left
+    }
+    */
+    if (this._right_origin !== null) {
+      info += 0b100
+    }
+    if (this._parentSub !== null) {
+      info += 0b1000
+    }
+    encoder.writeUint8(info)
+    encoder.writeID(this._id)
+    if (info & 0b1) {
+      encoder.writeID(this._origin._lastId)
+    }
+    // TODO: remove
+    /* see above
+    if (info & 0b10) {
+      encoder.writeID(this._left._lastId)
+    }
+    */
+    if (info & 0b100) {
+      encoder.writeID(this._right_origin._id)
+    }
+    if ((info & 0b101) === 0) {
+      // neither origin nor right is defined
+      encoder.writeID(this._parent._id)
+    }
+    if (info & 0b1000) {
+      encoder.writeVarString(JSON.stringify(this._parentSub))
+    }
+  }
+  _fromBinary (y, decoder) {
+    let missing = []
+    const info = decoder.readUint8()
+    const id = decoder.readID()
+    this._id = id
+    // read origin
+    if (info & 0b1) {
+      // origin != null
+      const originID = decoder.readID()
+      // we have to query for left again because it might have been split/merged..
+      const origin = y.os.getItemCleanEnd(originID)
+      if (origin === null) {
+        missing.push(originID)
+      } else {
+        this._origin = origin
+        this._left = this._origin
+      }
+    }
+    // read right
+    if (info & 0b100) {
+      // right != null
+      const rightID = decoder.readID()
+      // we have to query for right again because it might have been split/merged..
+      const right = y.os.getItemCleanStart(rightID)
+      if (right === null) {
+        missing.push(rightID)
+      } else {
+        this._right = right
+        this._right_origin = right
+      }
+    }
+    // read parent
+    if ((info & 0b101) === 0) {
+      // neither origin nor right is defined
+      const parentID = decoder.readID()
+      // parent does not change, so we don't have to search for it again
+      if (this._parent === null) {
+        const parent = y.os.get(parentID)
+        if (parent === null) {
+          missing.push(parentID)
+        } else {
+          this._parent = parent
+        }
+      }
+    } else if (this._parent === null) {
+      if (this._origin !== null) {
+        this._parent = this._origin._parent
+      } else if (this._right_origin !== null) {
+        this._parent = this._right_origin._parent
+      }
+    }
+    if (info & 0b1000) {
+      // TODO: maybe put this in read parent condition (you can also read parentsub from left/right)
+      this._parentSub = JSON.parse(decoder.readVarString())
+    }
+    if (y.ss.getState(id.user) < id.clock) {
+      missing.push(new ID(id.user, id.clock - 1))
+    }
+    return missing
+  }
+}

src/Struct/ItemJSON.js

@@ -0,0 +1,64 @@
+import { splitHelper, default as Item } from './Item.js'
+import { logID } from '../MessageHandler/messageToString.js'
+
+export default class ItemJSON extends Item {
+  constructor () {
+    super()
+    this._content = null
+  }
+  _copy () {
+    let struct = super._copy()
+    struct._content = this._content
+    return struct
+  }
+  get _length () {
+    return this._content.length
+  }
+  _fromBinary (y, decoder) {
+    let missing = super._fromBinary(y, decoder)
+    let len = decoder.readVarUint()
+    this._content = new Array(len)
+    for (let i = 0; i < len; i++) {
+      const ctnt = decoder.readVarString()
+      let parsed
+      if (ctnt === 'undefined') {
+        parsed = undefined
+      } else {
+        parsed = JSON.parse(ctnt)
+      }
+      this._content[i] = parsed
+    }
+    return missing
+  }
+  _toBinary (encoder) {
+    super._toBinary(encoder)
+    let len = this._content.length
+    encoder.writeVarUint(len)
+    for (let i = 0; i < len; i++) {
+      let encoded
+      let content = this._content[i]
+      if (content === undefined) {
+        encoded = 'undefined'
+      } else {
+        encoded = JSON.stringify(content)
+      }
+      encoder.writeVarString(encoded)
+    }
+  }
+  _logString () {
+    const left = this._left !== null ? this._left._lastId : null
+    const origin = this._origin !== null ? this._origin._lastId : null
+    return `ItemJSON(id:${logID(this._id)},content:${JSON.stringify(this._content)},left:${logID(left)},origin:${logID(origin)},right:${logID(this._right)},parent:${logID(this._parent)},parentSub:${this._parentSub})`
+  }
+  _splitAt (y, diff) {
+    if (diff === 0) {
+      return this
+    } else if (diff >= this._length) {
+      return this._right
+    }
+    let item = new ItemJSON()
+    item._content = this._content.splice(diff)
+    splitHelper(y, this, item, diff)
+    return item
+  }
+}

src/Struct/ItemString.js

@@ -0,0 +1,43 @@
+import { splitHelper, default as Item } from './Item.js'
+import { logID } from '../MessageHandler/messageToString.js'
+
+export default class ItemString extends Item {
+  constructor () {
+    super()
+    this._content = null
+  }
+  _copy () {
+    let struct = super._copy()
+    struct._content = this._content
+    return struct
+  }
+  get _length () {
+    return this._content.length
+  }
+  _fromBinary (y, decoder) {
+    let missing = super._fromBinary(y, decoder)
+    this._content = decoder.readVarString()
+    return missing
+  }
+  _toBinary (encoder) {
+    super._toBinary(encoder)
+    encoder.writeVarString(this._content)
+  }
+  _logString () {
+    const left = this._left !== null ? this._left._lastId : null
+    const origin = this._origin !== null ? this._origin._lastId : null
+    return `ItemJSON(id:${logID(this._id)},content:${JSON.stringify(this._content)},left:${logID(left)},origin:${logID(origin)},right:${logID(this._right)},parent:${logID(this._parent)},parentSub:${this._parentSub})`
+  }
+  _splitAt (y, diff) {
+    if (diff === 0) {
+      return this
+    } else if (diff >= this._length) {
+      return this._right
+    }
+    let item = new ItemString()
+    item._content = this._content.slice(diff)
+    this._content = this._content.slice(0, diff)
+    splitHelper(y, this, item, diff)
+    return item
+  }
+}

src/Struct/Type.js

@@ -0,0 +1,172 @@
+import Item from './Item.js'
+import EventHandler from '../Util/EventHandler.js'
+import ID from '../Util/ID.js'
+
+// restructure children as if they were inserted one after another
+function integrateChildren (y, start) {
+  let right
+  do {
+    right = start._right
+    start._right = null
+    start._right_origin = null
+    start._origin = start._left
+    start._integrate(y)
+    start = right
+  } while (right !== null)
+}
+
+export function getListItemIDByPosition (type, i) {
+  let pos = 0
+  let n = type._start
+  while (n !== null) {
+    if (!n._deleted) {
+      if (pos <= i && i < pos + n._length) {
+        const id = n._id
+        return new ID(id.user, id.clock + i - pos)
+      }
+      pos++
+    }
+    n = n._right
+  }
+}
+
+export default class Type extends Item {
+  constructor () {
+    super()
+    this._map = new Map()
+    this._start = null
+    this._y = null
+    this._eventHandler = new EventHandler()
+    this._deepEventHandler = new EventHandler()
+  }
+  getPathTo (type) {
+    if (type === this) {
+      return []
+    }
+    const path = []
+    const y = this._y
+    while (type._parent !== this && this._parent !== y) {
+      let parent = type._parent
+      if (type._parentSub !== null) {
+        path.push(type._parentSub)
+      } else {
+        // parent is array-ish
+        for (let [i, child] of parent) {
+          if (child === type) {
+            path.push(i)
+            break
+          }
+        }
+      }
+      type = parent
+    }
+    if (this._parent !== this) {
+      throw new Error('The type is not a child of this node')
+    }
+    return path
+  }
+  _callEventHandler (event) {
+    const changedParentTypes = this._y._transaction.changedParentTypes
+    this._eventHandler.callEventListeners(event)
+    let type = this
+    while (type !== this._y) {
+      let events = changedParentTypes.get(type)
+      if (events === undefined) {
+        events = []
+        changedParentTypes.set(type, events)
+      }
+      events.push(event)
+      type = type._parent
+    }
+  }
+  _copy (undeleteChildren) {
+    let copy = super._copy()
+    let map = new Map()
+    copy._map = map
+    for (let [key, value] of this._map) {
+      if (undeleteChildren.has(value) || !value.deleted) {
+        let _item = value._copy(undeleteChildren)
+        _item._parent = copy
+        map.set(key, value._copy(undeleteChildren))
+      }
+    }
+    let prevUndeleted = null
+    copy._start = null
+    let item = this._start
+    while (item !== null) {
+      if (undeleteChildren.has(item) || !item.deleted) {
+        let _item = item._copy(undeleteChildren)
+        _item._left = prevUndeleted
+        _item._origin = prevUndeleted
+        _item._right = null
+        _item._right_origin = null
+        _item._parent = copy
+        if (prevUndeleted === null) {
+          copy._start = _item
+        } else {
+          prevUndeleted._right = _item
+        }
+        prevUndeleted = _item
+      }
+      item = item._right
+    }
+    return copy
+  }
+  _transact (f) {
+    const y = this._y
+    if (y !== null) {
+      y.transact(f)
+    } else {
+      f(y)
+    }
+  }
+  observe (f) {
+    this._eventHandler.addEventListener(f)
+  }
+  observeDeep (f) {
+    this._deepEventHandler.addEventListener(f)
+  }
+  unobserve (f) {
+    this._eventHandler.removeEventListener(f)
+  }
+  unobserveDeep (f) {
+    this._deepEventHandler.removeEventListener(f)
+  }
+  _integrate (y) {
+    y._transaction.newTypes.add(this)
+    super._integrate(y)
+    this._y = y
+    // when integrating children we must make sure to
+    // integrate start
+    const start = this._start
+    if (start !== null) {
+      this._start = null
+      integrateChildren(y, start)
+    }
+    // integrate map children
+    const map = this._map
+    this._map = new Map()
+    for (let t of map.values()) {
+      // TODO make sure that right elements are deleted!
+      integrateChildren(y, t)
+    }
+  }
+  _delete (y, createDelete) {
+    super._delete(y, createDelete)
+    y._transaction.changedTypes.delete(this)
+    // delete map types
+    for (let value of this._map.values()) {
+      if (value instanceof Item && !value._deleted) {
+        value._delete(y, false)
+      }
+    }
+    // delete array types
+    let t = this._start
+    while (t !== null) {
+      if (!t._deleted) {
+        t._delete(y, false)
+      }
+      t = t._right
+    }
+  }
+}

src/Type/YArray.js

@@ -0,0 +1,222 @@
+import Type from '../Struct/Type.js'
+import ItemJSON from '../Struct/ItemJSON.js'
+import ItemString from '../Struct/ItemString.js'
+import { logID } from '../MessageHandler/messageToString.js'
+import YEvent from '../Util/YEvent.js'
+
+class YArrayEvent extends YEvent {
+  constructor (yarray, remote) {
+    super(yarray)
+    this.remote = remote
+  }
+}
+
+export default class YArray extends Type {
+  _callObserver (parentSubs, remote) {
+    this._callEventHandler(new YArrayEvent(this, remote))
+  }
+  get (pos) {
+    let n = this._start
+    while (n !== null) {
+      if (!n._deleted) {
+        if (pos < n._length) {
+          if (n.constructor === ItemJSON || n.constructor === ItemString) {
+            return n._content[pos]
+          } else {
+            return n
+          }
+        }
+        pos -= n._length
+      }
+      n = n._right
+    }
+  }
+  toArray () {
+    return this.map(c => c)
+  }
+  toJSON () {
+    return this.map(c => {
+      if (c instanceof Type) {
+        if (c.toJSON !== null) {
+          return c.toJSON()
+        } else {
+          return c.toString()
+        }
+      }
+      return c
+    })
+  }
+  map (f) {
+    const res = []
+    this.forEach((c, i) => {
+      res.push(f(c, i, this))
+    })
+    return res
+  }
+  forEach (f) {
+    let pos = 0
+    let n = this._start
+    while (n !== null) {
+      if (!n._deleted) {
+        if (n instanceof Type) {
+          f(n, pos++, this)
+        } else {
+          const content = n._content
+          const contentLen = content.length
+          for (let i = 0; i < contentLen; i++) {
+            pos++
+            f(content[i], pos, this)
+          }
+        }
+      }
+      n = n._right
+    }
+  }
+  get length () {
+    let length = 0
+    let n = this._start
+    while (n !== null) {
+      if (!n._deleted) {
+        length += n._length
+      }
+      n = n._right
+    }
+    return length
+  }
+  [Symbol.iterator] () {
+    return {
+      next: function () {
+        while (this._item !== null && (this._item._deleted || this._item._length <= this._itemElement)) {
+          // item is deleted or itemElement does not exist (is deleted)
+          this._item = this._item._right
+          this._itemElement = 0
+        }
+        if (this._item === null) {
+          return {
+            done: true
+          }
+        }
+        let content
+        if (this._item instanceof Type) {
+          content = this._item
+        } else {
+          content = this._item._content[this._itemElement++]
+        }
+        return {
+          value: [this._count, content],
+          done: false
+        }
+      },
+      _item: this._start,
+      _itemElement: 0,
+      _count: 0
+    }
+  }
+  delete (pos, length = 1) {
+    this._y.transact(() => {
+      let item = this._start
+      let count = 0
+      while (item !== null && length > 0) {
+        if (!item._deleted) {
+          if (count <= pos && pos < count + item._length) {
+            const diffDel = pos - count
+            item = item._splitAt(this._y, diffDel)
+            item._splitAt(this._y, length)
+            length -= item._length
+            item._delete(this._y)
+            count += diffDel
+          } else {
+            count += item._length
+          }
+        }
+        item = item._right
+      }
+    })
+    if (length > 0) {
+      throw new Error('Delete exceeds the range of the YArray')
+    }
+  }
+  insertAfter (left, content) {
+    this._transact(y => {
+      let right
+      if (left === null) {
+        right = this._start
+      } else {
+        right = left._right
+      }
+      let prevJsonIns = null
+      for (let i = 0; i < content.length; i++) {
+        let c = content[i]
+        if (typeof c === 'function') {
+          c = new c() // eslint-disable-line new-cap
+        }
+        if (c instanceof Type) {
+          if (prevJsonIns !== null) {
+            if (y !== null) {
+              prevJsonIns._integrate(y)
+            }
+            left = prevJsonIns
+            prevJsonIns = null
+          }
+          c._origin = left
+          c._left = left
+          c._right = right
+          c._right_origin = right
+          c._parent = this
+          if (y !== null) {
+            c._integrate(y)
+          } else if (left === null) {
+            this._start = c
+          } else {
+            left._right = c
+          }
+          left = c
+        } else {
+          if (prevJsonIns === null) {
+            prevJsonIns = new ItemJSON()
+            prevJsonIns._origin = left
+            prevJsonIns._left = left
+            prevJsonIns._right = right
+            prevJsonIns._right_origin = right
+            prevJsonIns._parent = this
+            prevJsonIns._content = []
+          }
+          prevJsonIns._content.push(c)
+        }
+      }
+      if (prevJsonIns !== null && y !== null) {
+        prevJsonIns._integrate(y)
+      }
+    })
+  }
+  insert (pos, content) {
+    let left = null
+    let right = this._start
+    let count = 0
+    const y = this._y
+    while (right !== null) {
+      const rightLen = right._deleted ? 0 : (right._length - 1)
+      if (count <= pos && pos <= count + rightLen) {
+        const splitDiff = pos - count
+        right = right._splitAt(y, splitDiff)
+        left = right._left
+        count += splitDiff
+        break
+      }
+      if (!right._deleted) {
+        count += right._length
+      }
+      left = right
+      right = right._right
+    }
+    if (pos > count) {
+      throw new Error('Position exceeds array range!')
+    }
+    this.insertAfter(left, content)
+  }
+  _logString () {
+    const left = this._left !== null ? this._left._lastId : null
+    const origin = this._origin !== null ? this._origin._lastId : null
+    return `YArray(id:${logID(this._id)},start:${logID(this._start)},left:${logID(left)},origin:${logID(origin)},right:${logID(this._right)},parent:${logID(this._parent)},parentSub:${this._parentSub})`
+  }
+}

src/Type/YMap.js

@@ -0,0 +1,114 @@
+import Type from '../Struct/Type.js'
+import Item from '../Struct/Item.js'
+import ItemJSON from '../Struct/ItemJSON.js'
+import { logID } from '../MessageHandler/messageToString.js'
+import YEvent from '../Util/YEvent.js'
+
+class YMapEvent extends YEvent {
+  constructor (ymap, subs, remote) {
+    super(ymap)
+    this.keysChanged = subs
+    this.remote = remote
+  }
+}
+
+export default class YMap extends Type {
+  _callObserver (parentSubs, remote) {
+    this._callEventHandler(new YMapEvent(this, parentSubs, remote))
+  }
+  toJSON () {
+    const map = {}
+    for (let [key, item] of this._map) {
+      if (!item._deleted) {
+        let res
+        if (item instanceof Type) {
+          if (item.toJSON !== undefined) {
+            res = item.toJSON()
+          } else {
+            res = item.toString()
+          }
+        } else {
+          res = item._content[0]
+        }
+        map[key] = res
+      }
+    }
+    return map
+  }
+  keys () {
+    let keys = []
+    for (let [key, value] of this._map) {
+      if (!value._deleted) {
+        keys.push(key)
+      }
+    }
+    return keys
+  }
+  delete (key) {
+    this._transact((y) => {
+      let c = this._map.get(key)
+      if (y !== null && c !== undefined) {
+        c._delete(y)
+      }
+    })
+  }
+  set (key, value) {
+    this._transact(y => {
+      const old = this._map.get(key) || null
+      if (old !== null) {
+        if (old instanceof ItemJSON && old._content[0] === value) {
+          // Trying to overwrite with same value
+          // break here
+          return value
+        }
+        if (y !== null) {
+          old._delete(y)
+        }
+      }
+      let v
+      if (typeof value === 'function') {
+        v = new value() // eslint-disable-line new-cap
+        value = v
+      } else if (value instanceof Item) {
+        v = value
+      } else {
+        v = new ItemJSON()
+        v._content = [value]
+      }
+      v._right = old
+      v._right_origin = old
+      v._parent = this
+      v._parentSub = key
+      if (y !== null) {
+        v._integrate(y)
+      } else {
+        this._map.set(key, v)
+      }
+    })
+    return value
+  }
+  get (key) {
+    let v = this._map.get(key)
+    if (v === undefined || v._deleted) {
+      return undefined
+    }
+    if (v instanceof Type) {
+      return v
+    } else {
+      return v._content[v._content.length - 1]
+    }
+  }
+  has (key) {
+    let v = this._map.get(key)
+    if (v === undefined || v._deleted) {
+      return false
+    } else {
+      return true
+    }
+  }
+  _logString () {
+    const left = this._left !== null ? this._left._lastId : null
+    const origin = this._origin !== null ? this._origin._lastId : null
+    return `YMap(id:${logID(this._id)},mapSize:${this._map.size},left:${logID(left)},origin:${logID(origin)},right:${logID(this._right)},parent:${logID(this._parent)},parentSub:${this._parentSub})`
+  }
+}

src/Type/YText.js

@@ -0,0 +1,67 @@
+import ItemString from '../Struct/ItemString.js'
+import YArray from './YArray.js'
+import { logID } from '../MessageHandler/messageToString.js'
+
+export default class YText extends YArray {
+  constructor (string) {
+    super()
+    if (typeof string === 'string') {
+      const start = new ItemString()
+      start._parent = this
+      start._content = string
+      this._start = start
+    }
+  }
+  toString () {
+    const strBuilder = []
+    let n = this._start
+    while (n !== null) {
+      if (!n._deleted) {
+        strBuilder.push(n._content)
+      }
+      n = n._right
+    }
+    return strBuilder.join('')
+  }
+  insert (pos, text) {
+    this._transact(y => {
+      let left = null
+      let right = this._start
+      let count = 0
+      while (right !== null) {
+        if (count <= pos && pos < count + right._length) {
+          const splitDiff = pos - count
+          right = right._splitAt(this._y, pos - count)
+          left = right._left
+          count += splitDiff
+          break
+        }
+        count += right._length
+        left = right
+        right = right._right
+      }
+      if (pos > count) {
+        throw new Error('Position exceeds array range!')
+      }
+      let item = new ItemString()
+      item._origin = left
+      item._left = left
+      item._right = right
+      item._right_origin = right
+      item._parent = this
+      item._content = text
+      if (y !== null) {
+        item._integrate(this._y)
+      } else if (left === null) {
+        this._start = item
+      } else {
+        left._right = item
+      }
+    })
+  }
+  _logString () {
+    const left = this._left !== null ? this._left._lastId : null
+    const origin = this._origin !== null ? this._origin._lastId : null
+    return `YText(id:${logID(this._id)},start:${logID(this._start)},left:${logID(left)},origin:${logID(origin)},right:${logID(this._right)},parent:${logID(this._parent)},parentSub:${this._parentSub})`
+  }
+}

src/Type/y-xml/YXmlElement.js

@@ -0,0 +1,131 @@
+// import diff from 'fast-diff'
+import { defaultDomFilter } from './utils.js'
+
+import YMap from '../YMap.js'
+import YXmlFragment from './YXmlFragment.js'
+
+export default class YXmlElement extends YXmlFragment {
+  constructor (arg1, arg2) {
+    super()
+    this.nodeName = null
+    this._scrollElement = null
+    if (typeof arg1 === 'string') {
+      this.nodeName = arg1.toUpperCase()
+    } else if (arg1 != null && arg1.nodeType != null && arg1.nodeType === arg1.ELEMENT_NODE) {
+      this.nodeName = arg1.nodeName
+      this._setDom(arg1)
+    } else {
+      this.nodeName = 'UNDEFINED'
+    }
+    if (typeof arg2 === 'function') {
+      this._domFilter = arg2
+    }
+  }
+  _copy (undeleteChildren) {
+    let struct = super._copy(undeleteChildren)
+    struct.nodeName = this.nodeName
+    return struct
+  }
+  _setDom (dom) {
+    if (this._dom != null) {
+      throw new Error('Only call this method if you know what you are doing ;)')
+    } else if (dom._yxml != null) { // TODO do i need to check this? - no.. but for dev purps..
+      throw new Error('Already bound to an YXml type')
+    } else {
+      this._dom = dom
+      dom._yxml = this
+      // tag is already set in constructor
+      // set attributes
+      let attrNames = []
+      for (let i = 0; i < dom.attributes.length; i++) {
+        attrNames.push(dom.attributes[i].name)
+      }
+      attrNames = this._domFilter(dom, attrNames)
+      for (let i = 0; i < attrNames.length; i++) {
+        let attrName = attrNames[i]
+        let attrValue = dom.getAttribute(attrName)
+        this.setAttribute(attrName, attrValue)
+      }
+      this.insertDomElements(0, Array.prototype.slice.call(dom.childNodes))
+      this._bindToDom(dom)
+      return dom
+    }
+  }
+  _fromBinary (y, decoder) {
+    const missing = super._fromBinary(y, decoder)
+    this.nodeName = decoder.readVarString()
+    return missing
+  }
+  _toBinary (encoder) {
+    super._toBinary(encoder)
+    encoder.writeVarString(this.nodeName)
+  }
+  _integrate (y) {
+    if (this.nodeName === null) {
+      throw new Error('nodeName must be defined!')
+    }
+    if (this._domFilter === defaultDomFilter && this._parent instanceof YXmlFragment) {
+      this._domFilter = this._parent._domFilter
+    }
+    super._integrate(y)
+  }
+  /**
+   * Returns the string representation of the XML document.
+   * The attributes are ordered by attribute-name, so you can easily use this
+   * method to compare YXmlElements
+   */
+  toString () {
+    const attrs = this.getAttributes()
+    const stringBuilder = []
+    const keys = []
+    for (let key in attrs) {
+      keys.push(key)
+    }
+    keys.sort()
+    const keysLen = keys.length
+    for (let i = 0; i < keysLen; i++) {
+      const key = keys[i]
+      stringBuilder.push(key + '="' + attrs[key] + '"')
+    }
+    const nodeName = this.nodeName.toLocaleLowerCase()
+    const attrsString = stringBuilder.length > 0 ? ' ' + stringBuilder.join(' ') : ''
+    return `<${nodeName}${attrsString}>${super.toString()}</${nodeName}>`
+  }
+  removeAttribute () {
+    return YMap.prototype.delete.apply(this, arguments)
+  }
+
+  setAttribute () {
+    return YMap.prototype.set.apply(this, arguments)
+  }
+
+  getAttribute () {
+    return YMap.prototype.get.apply(this, arguments)
+  }
+
+  getAttributes () {
+    const obj = {}
+    for (let [key, value] of this._map) {
+      obj[key] = value._content[0]
+    }
+    return obj
+  }
+  getDom (_document) {
+    _document = _document || document
+    let dom = this._dom
+    if (dom == null) {
+      dom = _document.createElement(this.nodeName)
+      this._dom = dom
+      dom._yxml = this
+      let attrs = this.getAttributes()
+      for (let key in attrs) {
+        dom.setAttribute(key, attrs[key])
+      }
+      this.forEach(yxml => {
+        dom.appendChild(yxml.getDom(_document))
+      })
+      this._bindToDom(dom)
+    }
+    return dom
+  }
+}

src/Type/y-xml/YXmlEvent.js

@@ -0,0 +1,17 @@
+import YEvent from '../../Util/YEvent.js'
+
+export default class YXmlEvent extends YEvent {
+  constructor (target, subs, remote) {
+    super(target)
+    this.childListChanged = false
+    this.attributesChanged = new Set()
+    this.remote = remote
+    subs.forEach((sub) => {
+      if (sub === null) {
+        this.childListChanged = true
+      } else {
+        this.attributesChanged.add(sub)
+      }
+    })
+  }
+}

src/Type/y-xml/YXmlFragment.js

@@ -0,0 +1,279 @@
+/* global MutationObserver */
+
+import { defaultDomFilter, applyChangesFromDom, reflectChangesOnDom } from './utils.js'
+import { beforeTransactionSelectionFixer, afterTransactionSelectionFixer } from './selection.js'
+
+import YArray from '../YArray.js'
+import YXmlText from './YXmlText.js'
+import YXmlEvent from './YXmlEvent.js'
+import { logID } from '../../MessageHandler/messageToString.js'
+import diff from 'fast-diff'
+
+function domToYXml (parent, doms) {
+  const types = []
+  doms.forEach(d => {
+    if (d._yxml != null && d._yxml !== false) {
+      d._yxml._unbindFromDom()
+    }
+    if (parent._domFilter(d, []) !== null) {
+      let type
+      if (d.nodeType === d.TEXT_NODE) {
+        type = new YXmlText(d)
+      } else if (d.nodeType === d.ELEMENT_NODE) {
+        type = new YXmlFragment._YXmlElement(d, parent._domFilter)
+      } else {
+        throw new Error('Unsupported node!')
+      }
+      type.enableSmartScrolling(parent._scrollElement)
+      types.push(type)
+    } else {
+      d._yxml = false
+    }
+  })
+  return types
+}
+
+class YXmlTreeWalker {
+  constructor (root, f) {
+    this._filter = f || (() => true)
+    this._root = root
+    this._currentNode = root
+    this._firstCall = true
+  }
+  [Symbol.iterator] () {
+    return this
+  }
+  next () {
+    let n = this._currentNode
+    if (this._firstCall) {
+      this._firstCall = false
+      if (!n._deleted && this._filter(n)) {
+        return { value: n, done: false }
+      }
+    }
+    do {
+      if (!n._deleted && n.constructor === YXmlFragment._YXmlElement && n._start !== null) {
+        // walk down in the tree
+        n = n._start
+      } else {
+        // walk right or up in the tree
+        while (n !== this._root) {
+          if (n._right !== null) {
+            n = n._right
+            break
+          }
+          n = n._parent
+        }
+        if (n === this._root) {
+          n = null
+        }
+      }
+      if (n === this._root) {
+        break
+      }
+    } while (n !== null && (n._deleted || !this._filter(n)))
+    this._currentNode = n
+    if (n === null) {
+      return { done: true }
+    } else {
+      return { value: n, done: false }
+    }
+  }
+}
+
+export default class YXmlFragment extends YArray {
+  constructor () {
+    super()
+    this._dom = null
+    this._domFilter = defaultDomFilter
+    this._domObserver = null
+    // this function makes sure that either the
+    // dom event is executed, or the yjs observer is executed
+    var token = true
+    this._mutualExclude = f => {
+      if (token) {
+        token = false
+        try {
+          f()
+        } catch (e) {
+          console.error(e)
+        }
+        this._domObserver.takeRecords()
+        token = true
+      }
+    }
+  }
+  createTreeWalker (filter) {
+    return new YXmlTreeWalker(this, filter)
+  }
+  /**
+   * Retrieve first element that matches *query*
+   * Similar to DOM's querySelector, but only accepts a subset of its queries
+   *
+   * Query support:
+   *   - tagname
+   * TODO:
+   *   - id
+   *   - attribute
+   */
+  querySelector (query) {
+    query = query.toUpperCase()
+    const iterator = new YXmlTreeWalker(this, element => element.nodeName === query)
+    const next = iterator.next()
+    if (next.done) {
+      return null
+    } else {
+      return next.value
+    }
+  }
+  querySelectorAll (query) {
+    query = query.toUpperCase()
+    return Array.from(new YXmlTreeWalker(this, element => element.nodeName === query))
+  }
+  enableSmartScrolling (scrollElement) {
+    this._scrollElement = scrollElement
+    this.forEach(xml => {
+      xml.enableSmartScrolling(scrollElement)
+    })
+  }
+  setDomFilter (f) {
+    this._domFilter = f
+    this.forEach(xml => {
+      xml.setDomFilter(f)
+    })
+  }
+  _callObserver (parentSubs, remote) {
+    this._callEventHandler(new YXmlEvent(this, parentSubs, remote))
+  }
+  toString () {
+    return this.map(xml => xml.toString()).join('')
+  }
+  _delete (y, createDelete) {
+    this._unbindFromDom()
+    super._delete(y, createDelete)
+  }
+  _unbindFromDom () {
+    if (this._domObserver != null) {
+      this._domObserver.disconnect()
+      this._domObserver = null
+    }
+    if (this._dom != null) {
+      this._dom._yxml = null
+      this._dom = null
+    }
+  }
+  insertDomElementsAfter (prev, doms) {
+    const types = domToYXml(this, doms)
+    this.insertAfter(prev, types)
+    return types
+  }
+  insertDomElements (pos, doms) {
+    const types = domToYXml(this, doms)
+    this.insert(pos, types)
+    return types
+  }
+  getDom () {
+    return this._dom
+  }
+  bindToDom (dom) {
+    if (this._dom != null) {
+      this._unbindFromDom()
+    }
+    if (dom._yxml != null) {
+      dom._yxml._unbindFromDom()
+    }
+    if (MutationObserver == null) {
+      throw new Error('Not able to bind to a DOM element, because MutationObserver is not available!')
+    }
+    dom.innerHTML = ''
+    this._dom = dom
+    dom._yxml = this
+    this.forEach(t => {
+      dom.insertBefore(t.getDom(), null)
+    })
+    this._bindToDom(dom)
+  }
+  // binds to a dom element
+  // Only call if dom and YXml are isomorph
+  _bindToDom (dom) {
+    if (this._parent === null || this._parent._dom != null || typeof MutationObserver === 'undefined') {
+      // only bind if parent did not already bind
+      return
+    }
+    this._y.on('beforeTransaction', () => {
+      this._domObserverListener(this._domObserver.takeRecords())
+    })
+    this._y.on('beforeTransaction', beforeTransactionSelectionFixer)
+    this._y.on('afterTransaction', afterTransactionSelectionFixer)
+    // Apply Y.Xml events to dom
+    this.observeDeep(reflectChangesOnDom.bind(this))
+    // Apply Dom changes on Y.Xml
+    this._domObserverListener = mutations => {
+      this._mutualExclude(() => {
+        this._y.transact(() => {
+          let diffChildren = new Set()
+          mutations.forEach(mutation => {
+            const dom = mutation.target
+            const yxml = dom._yxml
+            if (yxml == null) {
+              // dom element is filtered
+              return
+            }
+            switch (mutation.type) {
+              case 'characterData':
+                var diffs = diff(yxml.toString(), dom.nodeValue)
+                var pos = 0
+                for (var i = 0; i < diffs.length; i++) {
+                  var d = diffs[i]
+                  if (d[0] === 0) { // EQUAL
+                    pos += d[1].length
+                  } else if (d[0] === -1) { // DELETE
+                    yxml.delete(pos, d[1].length)
+                  } else { // INSERT
+                    yxml.insert(pos, d[1])
+                    pos += d[1].length
+                  }
+                }
+                break
+              case 'attributes':
+                let name = mutation.attributeName
+                // check if filter accepts attribute
+                if (this._domFilter(dom, [name]).length > 0 && this.constructor !== YXmlFragment) {
+                  var val = dom.getAttribute(name)
+                  if (yxml.getAttribute(name) !== val) {
+                    if (val == null) {
+                      yxml.removeAttribute(name)
+                    } else {
+                      yxml.setAttribute(name, val)
+                    }
+                  }
+                }
+                break
+              case 'childList':
+                diffChildren.add(mutation.target)
+                break
+            }
+          })
+          for (let dom of diffChildren) {
+            if (dom._yxml != null && dom._yxml !== false) {
+              applyChangesFromDom(dom)
+            }
+          }
+        })
+      })
+    }
+    this._domObserver = new MutationObserver(this._domObserverListener)
+    this._domObserver.observe(dom, {
+      childList: true,
+      attributes: true,
+      characterData: true,
+      subtree: true
+    })
+    return dom
+  }
+  _logString () {
+    const left = this._left !== null ? this._left._lastId : null
+    const origin = this._origin !== null ? this._origin._lastId : null
+    return `YXml(id:${logID(this._id)},left:${logID(left)},origin:${logID(origin)},right:${this._right},parent:${logID(this._parent)},parentSub:${this._parentSub})`
+  }
+}

src/Type/y-xml/YXmlText.js

@@ -0,0 +1,93 @@
+import YText from '../YText.js'
+
+export default class YXmlText extends YText {
+  constructor (arg1) {
+    let dom = null
+    let initialText = null
+    if (arg1 != null) {
+      if (arg1.nodeType != null && arg1.nodeType === arg1.TEXT_NODE) {
+        dom = arg1
+        initialText = dom.nodeValue
+      } else if (typeof arg1 === 'string') {
+        initialText = arg1
+      }
+    }
+    super(initialText)
+    this._dom = null
+    this._domObserver = null
+    this._domObserverListener = null
+    this._scrollElement = null
+    if (dom !== null) {
+      this._setDom(arg1)
+    }
+    /*
+    var token = true
+    this._mutualExclude = f => {
+      if (token) {
+        token = false
+        try {
+          f()
+        } catch (e) {
+          console.error(e)
+        }
+        this._domObserver.takeRecords()
+        token = true
+      }
+    }
+    this.observe(event => {
+      if (this._dom != null) {
+        const dom = this._dom
+        this._mutualExclude(() => {
+          let anchorViewPosition = getAnchorViewPosition(this._scrollElement)
+          let anchorViewFix
+          if (anchorViewPosition !== null && (anchorViewPosition.anchor !== null || getBoundingClientRect(this._dom).top <= 0)) {
+            anchorViewFix = anchorViewPosition
+          } else {
+            anchorViewFix = null
+          }
+          dom.nodeValue = this.toString()
+          fixScrollPosition(this._scrollElement, anchorViewFix)
+        })
+      }
+    })
+    */
+  }
+  setDomFilter () {}
+  enableSmartScrolling (scrollElement) {
+    this._scrollElement = scrollElement
+  }
+  _setDom (dom) {
+    if (this._dom != null) {
+      this._unbindFromDom()
+    }
+    if (dom._yxml != null) {
+      dom._yxml._unbindFromDom()
+    }
+    // set marker
+    this._dom = dom
+    dom._yxml = this
+  }
+  getDom (_document) {
+    _document = _document || document
+    if (this._dom === null) {
+      const dom = _document.createTextNode(this.toString())
+      this._setDom(dom)
+      return dom
+    }
+    return this._dom
+  }
+  _delete (y, createDelete) {
+    this._unbindFromDom()
+    super._delete(y, createDelete)
+  }
+  _unbindFromDom () {
+    if (this._domObserver != null) {
+      this._domObserver.disconnect()
+      this._domObserver = null
+    }
+    if (this._dom != null) {
+      this._dom._yxml = null
+      this._dom = null
+    }
+  }
+}

src/Type/y-xml/selection.js

@@ -0,0 +1,73 @@
+/* globals getSelection */
+
+import { getRelativePosition, fromRelativePosition } from '../../Util/relativePosition.js'
+
+let browserSelection = null
+let relativeSelection = null
+
+export let beforeTransactionSelectionFixer
+if (typeof getSelection !== 'undefined') {
+  beforeTransactionSelectionFixer = function _beforeTransactionSelectionFixer (y, remote) {
+    if (!remote) {
+      return
+    }
+    relativeSelection = { from: null, to: null, fromY: null, toY: null }
+    browserSelection = getSelection()
+    const anchorNode = browserSelection.anchorNode
+    if (anchorNode !== null && anchorNode._yxml != null) {
+      const yxml = anchorNode._yxml
+      relativeSelection.from = getRelativePosition(yxml, browserSelection.anchorOffset)
+      relativeSelection.fromY = yxml._y
+    }
+    const focusNode = browserSelection.focusNode
+    if (focusNode !== null && focusNode._yxml != null) {
+      const yxml = focusNode._yxml
+      relativeSelection.to = getRelativePosition(yxml, browserSelection.focusOffset)
+      relativeSelection.toY = yxml._y
+    }
+  }
+} else {
+  beforeTransactionSelectionFixer = function _fakeBeforeTransactionSelectionFixer () {}
+}
+
+export function afterTransactionSelectionFixer (y, remote) {
+  if (relativeSelection === null || !remote) {
+    return
+  }
+  const to = relativeSelection.to
+  const from = relativeSelection.from
+  const fromY = relativeSelection.fromY
+  const toY = relativeSelection.toY
+  let shouldUpdate = false
+  let anchorNode = browserSelection.anchorNode
+  let anchorOffset = browserSelection.anchorOffset
+  let focusNode = browserSelection.focusNode
+  let focusOffset = browserSelection.focusOffset
+  if (from !== null) {
+    let sel = fromRelativePosition(fromY, from)
+    if (sel !== null) {
+      shouldUpdate = true
+      anchorNode = sel.type.getDom()
+      anchorOffset = sel.offset
+    }
+  }
+  if (to !== null) {
+    let sel = fromRelativePosition(toY, to)
+    if (sel !== null) {
+      focusNode = sel.type.getDom()
+      focusOffset = sel.offset
+      shouldUpdate = true
+    }
+  }
+  if (shouldUpdate) {
+    browserSelection.setBaseAndExtent(
+      anchorNode,
+      anchorOffset,
+      focusNode,
+      focusOffset
+    )
+  }
+  // delete, so the objects can be gc'd
+  relativeSelection = null
+  browserSelection = null
+}

src/Type/y-xml/utils.js

@@ -0,0 +1,246 @@
+import YXmlText from './YXmlText.js'
+
+export function defaultDomFilter (node, attributes) {
+  return attributes
+}
+
+export function getAnchorViewPosition (scrollElement) {
+  if (scrollElement == null) {
+    return null
+  }
+  let anchor = document.getSelection().anchorNode
+  if (anchor != null) {
+    let top = getBoundingClientRect(anchor).top
+    if (top >= 0 && top <= document.documentElement.clientHeight) {
+      return {
+        anchor: anchor,
+        top: top
+      }
+    }
+  }
+  return {
+    anchor: null,
+    scrollTop: scrollElement.scrollTop,
+    scrollHeight: scrollElement.scrollHeight
+  }
+}
+
+// get BoundingClientRect that works on text nodes
+export function getBoundingClientRect (element) {
+  if (element.getBoundingClientRect != null) {
+    // is element node
+    return element.getBoundingClientRect()
+  } else {
+    // is text node
+    if (element.parentNode == null) {
+      // range requires that text nodes have a parent
+      let span = document.createElement('span')
+      span.appendChild(element)
+    }
+    let range = document.createRange()
+    range.selectNode(element)
+    return range.getBoundingClientRect()
+  }
+}
+
+export function fixScrollPosition (scrollElement, fix) {
+  if (scrollElement !== null && fix !== null) {
+    if (fix.anchor === null) {
+      if (scrollElement.scrollTop === fix.scrollTop) {
+        scrollElement.scrollTop = scrollElement.scrollHeight - fix.scrollHeight
+      }
+    } else {
+      scrollElement.scrollTop = getBoundingClientRect(fix.anchor).top - fix.top
+    }
+  }
+}
+
+function iterateUntilUndeleted (item) {
+  while (item !== null && item._deleted) {
+    item = item._right
+  }
+  return item
+}
+
+function _insertNodeHelper (yxml, prevExpectedNode, child) {
+  let insertedNodes = yxml.insertDomElementsAfter(prevExpectedNode, [child])
+  if (insertedNodes.length > 0) {
+    return insertedNodes[0]
+  } else {
+    return prevExpectedNode
+  }
+}
+
+/*
+ * 1. Check if any of the nodes was deleted
+ * 2. Iterate over the children.
+ *    2.1 If a node exists without _yxml property, insert a new node
+ *    2.2 If _contents.length < dom.childNodes.length, fill the
+ *        rest of _content with childNodes
+ *    2.3 If a node was moved, delete it and
+ *       recreate a new yxml element that is bound to that node.
+ *       You can detect that a node was moved because expectedId
+ *       !== actualId in the list
+ */
+export function applyChangesFromDom (dom) {
+  const yxml = dom._yxml
+  const y = yxml._y
+  let knownChildren =
+    new Set(
+      Array.prototype.map.call(dom.childNodes, child => child._yxml)
+      .filter(id => id !== undefined)
+    )
+  // 1. Check if any of the nodes was deleted
+  yxml.forEach(function (childType, i) {
+    if (!knownChildren.has(childType)) {
+      childType._delete(y)
+    }
+  })
+  // 2. iterate
+  let childNodes = dom.childNodes
+  let len = childNodes.length
+  let prevExpectedNode = null
+  let expectedNode = iterateUntilUndeleted(yxml._start)
+  for (let domCnt = 0; domCnt < len; domCnt++) {
+    const child = childNodes[domCnt]
+    const childYXml = child._yxml
+    if (childYXml != null) {
+      if (childYXml === false) {
+        // should be ignored or is going to be deleted
+        continue
+      }
+      if (expectedNode !== null) {
+        if (expectedNode !== childYXml) {
+          // 2.3 Not expected node
+          if (childYXml._parent !== this) {
+            // element is going to be deleted by its previous parent
+            child._yxml = null
+          } else {
+            childYXml._delete(y)
+          }
+          prevExpectedNode = _insertNodeHelper(yxml, prevExpectedNode, child)
+        } else {
+          prevExpectedNode = expectedNode
+          expectedNode = iterateUntilUndeleted(expectedNode._right)
+        }
+        // if this is the expected node id, just continue
+      } else {
+        // 2.2 fill _conten with child nodes
+        prevExpectedNode = _insertNodeHelper(yxml, prevExpectedNode, child)
+      }
+    } else {
+      // 2.1 A new node was found
+      prevExpectedNode = _insertNodeHelper(yxml, prevExpectedNode, child)
+    }
+  }
+}
+
+export function reflectChangesOnDom (events) {
+  // Make sure that no filtered attributes are applied to the structure
+  // if they were, delete them
+  events.forEach(event => {
+    const target = event.target
+    const keys = this._domFilter(target.nodeName, Array.from(event.keysChanged))
+    if (keys === null) {
+      target._delete()
+    } else {
+      const removeKeys = new Set() // is a copy of event.keysChanged
+      event.keysChanged.forEach(key => { removeKeys.add(key) })
+      keys.forEach(key => {
+        // remove all accepted keys from removeKeys
+        removeKeys.delete(key)
+      })
+      // remove the filtered attribute
+      removeKeys.forEach(key => {
+        target.removeAttribute(key)
+      })
+    }
+  })
+  this._mutualExclude(() => {
+    events.forEach(event => {
+      const yxml = event.target
+      const dom = yxml._dom
+      if (dom != null) {
+        // TODO: do this once before applying stuff
+        // let anchorViewPosition = getAnchorViewPosition(yxml._scrollElement)
+        if (yxml.constructor === YXmlText) {
+          yxml._dom.nodeValue = yxml.toString()
+        } else {
+          // update attributes
+          event.attributesChanged.forEach(attributeName => {
+            const value = yxml.getAttribute(attributeName)
+            if (value === undefined) {
+              dom.removeAttribute(attributeName)
+            } else {
+              dom.setAttribute(attributeName, value)
+            }
+          })
+          if (event.childListChanged) {
+            // create fragment of undeleted nodes
+            const fragment = document.createDocumentFragment()
+            yxml.forEach(function (t) {
+              fragment.append(t.getDom())
+            })
+            // remove remainding nodes
+            let lastChild = dom.lastChild
+            while (lastChild !== null) {
+              dom.removeChild(lastChild)
+              lastChild = dom.lastChild
+            }
+            // insert fragment of undeleted nodes
+            dom.append(fragment)
+          }
+        }
+        /* TODO: smartscrolling
+        .. else if (event.type === 'childInserted' || event.type === 'insert') {
+          let nodes = event.values
+          for (let i = nodes.length - 1; i >= 0; i--) {
+            let node = nodes[i]
+            node.setDomFilter(yxml._domFilter)
+            node.enableSmartScrolling(yxml._scrollElement)
+            let dom = node.getDom()
+            let fixPosition = null
+            let nextDom = null
+            if (yxml._content.length > event.index + i + 1) {
+              nextDom = yxml.get(event.index + i + 1).getDom()
+            }
+            yxml._dom.insertBefore(dom, nextDom)
+            if (anchorViewPosition === null) {
+              // nop
+            } else if (anchorViewPosition.anchor !== null) {
+              // no scrolling when current selection
+              if (!dom.contains(anchorViewPosition.anchor) && !anchorViewPosition.anchor.contains(dom)) {
+                fixPosition = anchorViewPosition
+              }
+            } else if (getBoundingClientRect(dom).top <= 0) {
+              // adjust scrolling if modified element is out of view,
+              // there is no anchor element, and the browser did not adjust scrollTop (this is checked later)
+              fixPosition = anchorViewPosition
+            }
+            fixScrollPosition(yxml._scrollElement, fixPosition)
+          }
+        } else if (event.type === 'childRemoved' || event.type === 'delete') {
+          for (let i = event.values.length - 1; i >= 0; i--) {
+            let dom = event.values[i]._dom
+            let fixPosition = null
+            if (anchorViewPosition === null) {
+              // nop
+            } else if (anchorViewPosition.anchor !== null) {
+              // no scrolling when current selection
+              if (!dom.contains(anchorViewPosition.anchor) && !anchorViewPosition.anchor.contains(dom)) {
+                fixPosition = anchorViewPosition
+              }
+            } else if (getBoundingClientRect(dom).top <= 0) {
+              // adjust scrolling if modified element is out of view,
+              // there is no anchor element, and the browser did not adjust scrollTop (this is checked later)
+              fixPosition = anchorViewPosition
+            }
+            dom.remove()
+            fixScrollPosition(yxml._scrollElement, fixPosition)
+          }
+        }
+        */
+      }
+    })
+  })
+}

src/Type/y-xml/y-xml.js

@@ -0,0 +1,9 @@
+
+import YXmlFragment from './YXmlFragment.js'
+import YXmlElement from './YXmlElement.js'
+
+export { default as YXmlFragment } from './YXmlFragment.js'
+export { default as YXmlElement } from './YXmlElement.js'
+export { default as YXmlText } from './YXmlText.js'
+
+YXmlFragment._YXmlElement = YXmlElement

src/Util/EventHandler.js

@@ -0,0 +1,35 @@
+
+export default class EventHandler {
+  constructor () {
+    this.eventListeners = []
+  }
+  destroy () {
+    this.eventListeners = null
+  }
+  addEventListener (f) {
+    this.eventListeners.push(f)
+  }
+  removeEventListener (f) {
+    this.eventListeners = this.eventListeners.filter(function (g) {
+      return f !== g
+    })
+  }
+  removeAllEventListeners () {
+    this.eventListeners = []
+  }
+  callEventListeners (event) {
+    for (var i = 0; i < this.eventListeners.length; i++) {
+      try {
+        const f = this.eventListeners[i]
+        f(event)
+      } catch (e) {
+        /*
+          Your observer threw an error. This error was caught so that Yjs
+          can ensure data consistency! In order to debug this error you
+          have to check "Pause On Caught Exceptions" in developer tools.
+        */
+        console.error(e)
+      }
+    }
+  }
+}

src/Util/ID.js

@@ -0,0 +1,16 @@
+
+export default class ID {
+  constructor (user, clock) {
+    this.user = user
+    this.clock = clock
+  }
+  clone () {
+    return new ID(this.user, this.clock)
+  }
+  equals (id) {
+    return id !== null && id.user === this.user && id.clock === this.clock
+  }
+  lessThan (id) {
+    return this.user < id.user || (this.user === id.user && this.clock < id.clock)
+  }
+}

src/Util/NamedEventHandler.js

@@ -0,0 +1,46 @@
+export default class NamedEventHandler {
+  constructor () {
+    this._eventListener = new Map()
+  }
+  _getListener (name) {
+    let listeners = this._eventListener.get(name)
+    if (listeners === undefined) {
+      listeners = {
+        once: new Set(),
+        on: new Set()
+      }
+      this._eventListener.set(name, listeners)
+    }
+    return listeners
+  }
+  once (name, f) {
+    let listeners = this._getListener(name)
+    listeners.once.add(f)
+  }
+  on (name, f) {
+    let listeners = this._getListener(name)
+    listeners.on.add(f)
+  }
+  off (name, f) {
+    if (name == null || f == null) {
+      throw new Error('You must specify event name and function!')
+    }
+    const listener = this._eventListener.get(name)
+    if (listener !== undefined) {
+      listener.remove(f)
+    }
+  }
+  emit (name, ...args) {
+    const listener = this._eventListener.get(name)
+    if (listener !== undefined) {
+      listener.on.forEach(f => f.apply(null, args))
+      listener.once.forEach(f => f.apply(null, args))
+      listener.once = new Set()
+    } else if (name === 'error') {
+      console.error(args[0])
+    }
+  }
+  destroy () {
+    this._eventListener = null
+  }
+}

src/Util/RootID.js

@@ -0,0 +1,17 @@
+import { getReference } from './structReferences.js'
+
+export const RootFakeUserID = 0xFFFFFF
+
+export default class RootID {
+  constructor (name, typeConstructor) {
+    this.user = RootFakeUserID
+    this.name = name
+    this.type = getReference(typeConstructor)
+  }
+  equals (id) {
+    return id !== null && id.user === this.user && id.name === this.name && id.type === this.type
+  }
+  lessThan (id) {
+    return this.user < id.user || (this.user === id.user && (this.name < id.name || (this.name === id.name && this.type < id.type)))
+  }
+}

src/Util/Tree.js

@@ -0,0 +1,471 @@
+
+class N {
+  // A created node is always red!
+  constructor (val) {
+    this.val = val
+    this.color = true
+    this._left = null
+    this._right = null
+    this._parent = null
+  }
+  isRed () { return this.color }
+  isBlack () { return !this.color }
+  redden () { this.color = true; return this }
+  blacken () { this.color = false; return this }
+  get grandparent () {
+    return this.parent.parent
+  }
+  get parent () {
+    return this._parent
+  }
+  get sibling () {
+    return (this === this.parent.left)
+      ? this.parent.right : this.parent.left
+  }
+  get left () {
+    return this._left
+  }
+  get right () {
+    return this._right
+  }
+  set left (n) {
+    if (n !== null) {
+      n._parent = this
+    }
+    this._left = n
+  }
+  set right (n) {
+    if (n !== null) {
+      n._parent = this
+    }
+    this._right = n
+  }
+  rotateLeft (tree) {
+    var parent = this.parent
+    var newParent = this.right
+    var newRight = this.right.left
+    newParent.left = this
+    this.right = newRight
+    if (parent === null) {
+      tree.root = newParent
+      newParent._parent = null
+    } else if (parent.left === this) {
+      parent.left = newParent
+    } else if (parent.right === this) {
+      parent.right = newParent
+    } else {
+      throw new Error('The elements are wrongly connected!')
+    }
+  }
+  next () {
+    if (this.right !== null) {
+      // search the most left node in the right tree
+      var o = this.right
+      while (o.left !== null) {
+        o = o.left
+      }
+      return o
+    } else {
+      var p = this
+      while (p.parent !== null && p !== p.parent.left) {
+        p = p.parent
+      }
+      return p.parent
+    }
+  }
+  prev () {
+    if (this.left !== null) {
+      // search the most right node in the left tree
+      var o = this.left
+      while (o.right !== null) {
+        o = o.right
+      }
+      return o
+    } else {
+      var p = this
+      while (p.parent !== null && p !== p.parent.right) {
+        p = p.parent
+      }
+      return p.parent
+    }
+  }
+  rotateRight (tree) {
+    var parent = this.parent
+    var newParent = this.left
+    var newLeft = this.left.right
+    newParent.right = this
+    this.left = newLeft
+    if (parent === null) {
+      tree.root = newParent
+      newParent._parent = null
+    } else if (parent.left === this) {
+      parent.left = newParent
+    } else if (parent.right === this) {
+      parent.right = newParent
+    } else {
+      throw new Error('The elements are wrongly connected!')
+    }
+  }
+  getUncle () {
+    // we can assume that grandparent exists when this is called!
+    if (this.parent === this.parent.parent.left) {
+      return this.parent.parent.right
+    } else {
+      return this.parent.parent.left
+    }
+  }
+}
+
+/*
+ * This is a Red Black Tree implementation
+ */
+export default class Tree {
+  constructor () {
+    this.root = null
+    this.length = 0
+  }
+  findNext (id) {
+    var nextID = id.clone()
+    nextID.clock += 1
+    return this.findWithLowerBound(nextID)
+  }
+  findPrev (id) {
+    let prevID = id.clone()
+    prevID.clock -= 1
+    return this.findWithUpperBound(prevID)
+  }
+  findNodeWithLowerBound (from) {
+    var o = this.root
+    if (o === null) {
+      return null
+    } else {
+      while (true) {
+        if (from === null || (from.lessThan(o.val._id) && o.left !== null)) {
+          // o is included in the bound
+          // try to find an element that is closer to the bound
+          o = o.left
+        } else if (from !== null && o.val._id.lessThan(from)) {
+          // o is not within the bound, maybe one of the right elements is..
+          if (o.right !== null) {
+            o = o.right
+          } else {
+            // there is no right element. Search for the next bigger element,
+            // this should be within the bounds
+            return o.next()
+          }
+        } else {
+          return o
+        }
+      }
+    }
+  }
+  findNodeWithUpperBound (to) {
+    if (to === void 0) {
+      throw new Error('You must define from!')
+    }
+    var o = this.root
+    if (o === null) {
+      return null
+    } else {
+      while (true) {
+        if ((to === null || o.val._id.lessThan(to)) && o.right !== null) {
+          // o is included in the bound
+          // try to find an element that is closer to the bound
+          o = o.right
+        } else if (to !== null && to.lessThan(o.val._id)) {
+          // o is not within the bound, maybe one of the left elements is..
+          if (o.left !== null) {
+            o = o.left
+          } else {
+            // there is no left element. Search for the prev smaller element,
+            // this should be within the bounds
+            return o.prev()
+          }
+        } else {
+          return o
+        }
+      }
+    }
+  }
+  findSmallestNode () {
+    var o = this.root
+    while (o != null && o.left != null) {
+      o = o.left
+    }
+    return o
+  }
+  findWithLowerBound (from) {
+    var n = this.findNodeWithLowerBound(from)
+    return n == null ? null : n.val
+  }
+  findWithUpperBound (to) {
+    var n = this.findNodeWithUpperBound(to)
+    return n == null ? null : n.val
+  }
+  iterate (from, to, f) {
+    var o
+    if (from === null) {
+      o = this.findSmallestNode()
+    } else {
+      o = this.findNodeWithLowerBound(from)
+    }
+    while (
+      o !== null &&
+      (
+        to === null || // eslint-disable-line no-unmodified-loop-condition
+        o.val._id.lessThan(to) ||
+        o.val._id.equals(to)
+      )
+    ) {
+      f(o.val)
+      o = o.next()
+    }
+  }
+  find (id) {
+    let n = this.findNode(id)
+    if (n !== null) {
+      return n.val
+    } else {
+      return null
+    }
+  }
+  findNode (id) {
+    var o = this.root
+    if (o === null) {
+      return null
+    } else {
+      while (true) {
+        if (o === null) {
+          return null
+        }
+        if (id.lessThan(o.val._id)) {
+          o = o.left
+        } else if (o.val._id.lessThan(id)) {
+          o = o.right
+        } else {
+          return o
+        }
+      }
+    }
+  }
+  delete (id) {
+    var d = this.findNode(id)
+    if (d == null) {
+      // throw new Error('Element does not exist!')
+      return
+    }
+    this.length--
+    if (d.left !== null && d.right !== null) {
+      // switch d with the greates element in the left subtree.
+      // o should have at most one child.
+      var o = d.left
+      // find
+      while (o.right !== null) {
+        o = o.right
+      }
+      // switch
+      d.val = o.val
+      d = o
+    }
+    // d has at most one child
+    // let n be the node that replaces d
+    var isFakeChild
+    var child = d.left || d.right
+    if (child === null) {
+      isFakeChild = true
+      child = new N(null)
+      child.blacken()
+      d.right = child
+    } else {
+      isFakeChild = false
+    }
+
+    if (d.parent === null) {
+      if (!isFakeChild) {
+        this.root = child
+        child.blacken()
+        child._parent = null
+      } else {
+        this.root = null
+      }
+      return
+    } else if (d.parent.left === d) {
+      d.parent.left = child
+    } else if (d.parent.right === d) {
+      d.parent.right = child
+    } else {
+      throw new Error('Impossible!')
+    }
+    if (d.isBlack()) {
+      if (child.isRed()) {
+        child.blacken()
+      } else {
+        this._fixDelete(child)
+      }
+    }
+    this.root.blacken()
+    if (isFakeChild) {
+      if (child.parent.left === child) {
+        child.parent.left = null
+      } else if (child.parent.right === child) {
+        child.parent.right = null
+      } else {
+        throw new Error('Impossible #3')
+      }
+    }
+  }
+  _fixDelete (n) {
+    function isBlack (node) {
+      return node !== null ? node.isBlack() : true
+    }
+    function isRed (node) {
+      return node !== null ? node.isRed() : false
+    }
+    if (n.parent === null) {
+      // this can only be called after the first iteration of fixDelete.
+      return
+    }
+    // d was already replaced by the child
+    // d is not the root
+    // d and child are black
+    var sibling = n.sibling
+    if (isRed(sibling)) {
+      // make sibling the grandfather
+      n.parent.redden()
+      sibling.blacken()
+      if (n === n.parent.left) {
+        n.parent.rotateLeft(this)
+      } else if (n === n.parent.right) {
+        n.parent.rotateRight(this)
+      } else {
+        throw new Error('Impossible #2')
+      }
+      sibling = n.sibling
+    }
+    // parent, sibling, and children of n are black
+    if (n.parent.isBlack() &&
+      sibling.isBlack() &&
+      isBlack(sibling.left) &&
+      isBlack(sibling.right)
+    ) {
+      sibling.redden()
+      this._fixDelete(n.parent)
+    } else if (n.parent.isRed() &&
+      sibling.isBlack() &&
+      isBlack(sibling.left) &&
+      isBlack(sibling.right)
+    ) {
+      sibling.redden()
+      n.parent.blacken()
+    } else {
+      if (n === n.parent.left &&
+        sibling.isBlack() &&
+        isRed(sibling.left) &&
+        isBlack(sibling.right)
+      ) {
+        sibling.redden()
+        sibling.left.blacken()
+        sibling.rotateRight(this)
+        sibling = n.sibling
+      } else if (n === n.parent.right &&
+        sibling.isBlack() &&
+        isRed(sibling.right) &&
+        isBlack(sibling.left)
+      ) {
+        sibling.redden()
+        sibling.right.blacken()
+        sibling.rotateLeft(this)
+        sibling = n.sibling
+      }
+      sibling.color = n.parent.color
+      n.parent.blacken()
+      if (n === n.parent.left) {
+        sibling.right.blacken()
+        n.parent.rotateLeft(this)
+      } else {
+        sibling.left.blacken()
+        n.parent.rotateRight(this)
+      }
+    }
+  }
+  put (v) {
+    var node = new N(v)
+    if (this.root !== null) {
+      var p = this.root // p abbrev. parent
+      while (true) {
+        if (node.val._id.lessThan(p.val._id)) {
+          if (p.left === null) {
+            p.left = node
+            break
+          } else {
+            p = p.left
+          }
+        } else if (p.val._id.lessThan(node.val._id)) {
+          if (p.right === null) {
+            p.right = node
+            break
+          } else {
+            p = p.right
+          }
+        } else {
+          p.val = node.val
+          return p
+        }
+      }
+      this._fixInsert(node)
+    } else {
+      this.root = node
+    }
+    this.length++
+    this.root.blacken()
+    return node
+  }
+  _fixInsert (n) {
+    if (n.parent === null) {
+      n.blacken()
+      return
+    } else if (n.parent.isBlack()) {
+      return
+    }
+    var uncle = n.getUncle()
+    if (uncle !== null && uncle.isRed()) {
+      // Note: parent: red, uncle: red
+      n.parent.blacken()
+      uncle.blacken()
+      n.grandparent.redden()
+      this._fixInsert(n.grandparent)
+    } else {
+      // Note: parent: red, uncle: black or null
+      // Now we transform the tree in such a way that
+      // either of these holds:
+      //   1) grandparent.left.isRed
+      //     and grandparent.left.left.isRed
+      //   2) grandparent.right.isRed
+      //     and grandparent.right.right.isRed
+      if (n === n.parent.right && n.parent === n.grandparent.left) {
+        n.parent.rotateLeft(this)
+        // Since we rotated and want to use the previous
+        // cases, we need to set n in such a way that
+        // n.parent.isRed again
+        n = n.left
+      } else if (n === n.parent.left && n.parent === n.grandparent.right) {
+        n.parent.rotateRight(this)
+        // see above
+        n = n.right
+      }
+      // Case 1) or 2) hold from here on.
+      // Now traverse grandparent, make parent a black node
+      // on the highest level which holds two red nodes.
+      n.parent.blacken()
+      n.grandparent.redden()
+      if (n === n.parent.left) {
+        // Case 1
+        n.grandparent.rotateRight(this)
+      } else {
+        // Case 2
+        n.grandparent.rotateLeft(this)
+      }
+    }
+  }
+  flush () {}
+}

src/Util/UndoManager.js

@@ -0,0 +1,106 @@
+import ID from './ID.js'
+
+class ReverseOperation {
+  constructor (y) {
+    this.created = new Date()
+    const beforeState = y._transaction.beforeState
+    this.toState = new ID(y.userID, y.ss.getState(y.userID) - 1)
+    if (beforeState.has(y.userID)) {
+      this.fromState = new ID(y.userID, beforeState.get(y.userID))
+    } else {
+      this.fromState = this.toState
+    }
+    this.deletedStructs = y._transaction.deletedStructs
+  }
+}
+
+function isStructInScope (y, struct, scope) {
+  while (struct !== y) {
+    if (struct === scope) {
+      return true
+    }
+    struct = struct._parent
+  }
+  return false
+}
+
+function applyReverseOperation (y, scope, reverseBuffer) {
+  let performedUndo = false
+  y.transact(() => {
+    while (!performedUndo && reverseBuffer.length > 0) {
+      let undoOp = reverseBuffer.pop()
+      // make sure that it is possible to iterate {from}-{to}
+      y.os.getItemCleanStart(undoOp.fromState)
+      y.os.getItemCleanEnd(undoOp.toState)
+      y.os.iterate(undoOp.fromState, undoOp.toState, op => {
+        if (!op._deleted && isStructInScope(y, op, scope)) {
+          performedUndo = true
+          op._delete(y)
+        }
+      })
+      for (let op of undoOp.deletedStructs) {
+        if (
+          isStructInScope(y, op, scope) &&
+          op._parent !== y &&
+          !op._parent._deleted &&
+          (
+            op._parent._id.user !== y.userID ||
+            op._parent._id.clock < undoOp.fromState.clock ||
+            op._parent._id.clock > undoOp.fromState.clock
+          )
+        ) {
+          performedUndo = true
+          op = op._copy(undoOp.deletedStructs)
+          op._integrate(y)
+        }
+      }
+    }
+  })
+  return performedUndo
+}
+
+export default class UndoManager {
+  constructor (scope, options = {}) {
+    this.options = options
+    options.captureTimeout = options.captureTimeout == null ? 500 : options.captureTimeout
+    this._undoBuffer = []
+    this._redoBuffer = []
+    this._scope = scope
+    this._undoing = false
+    this._redoing = false
+    const y = scope._y
+    this.y = y
+    y.on('afterTransaction', (y, remote) => {
+      if (!remote && y._transaction.changedParentTypes.has(scope)) {
+        let reverseOperation = new ReverseOperation(y)
+        if (!this._undoing) {
+          let lastUndoOp = this._undoBuffer.length > 0 ? this._undoBuffer[this._undoBuffer.length - 1] : null
+          if (lastUndoOp !== null && reverseOperation.created - lastUndoOp.created <= options.captureTimeout) {
+            lastUndoOp.created = reverseOperation.created
+            lastUndoOp.toState = reverseOperation.toState
+            reverseOperation.deletedStructs.forEach(lastUndoOp.deletedStructs.add, lastUndoOp.deletedStructs)
+          } else {
+            this._undoBuffer.push(reverseOperation)
+          }
+          if (!this._redoing) {
+            this._redoBuffer = []
+          }
+        } else {
+          this._redoBuffer.push(reverseOperation)
+        }
+      }
+    })
+  }
+  undo () {
+    this._undoing = true
+    const performedUndo = applyReverseOperation(this.y, this._scope, this._undoBuffer)
+    this._undoing = false
+    return performedUndo
+  }
+  redo () {
+    this._redoing = true
+    const performedRedo = applyReverseOperation(this.y, this._scope, this._redoBuffer)
+    this._redoing = false
+    return performedRedo
+  }
+}

src/Util/YEvent.js

@@ -0,0 +1,28 @@
+
+export default class YEvent {
+  constructor (target) {
+    this.target = target
+    this.currentTarget = target
+  }
+  get path () {
+    const path = []
+    let type = this.target
+    const y = type._y
+    while (type !== this.currentTarget && type !== y) {
+      let parent = type._parent
+      if (type._parentSub !== null) {
+        path.unshift(type._parentSub)
+      } else {
+        // parent is array-ish
+        for (let [i, child] of parent) {
+          if (child === type) {
+            path.unshift(i)
+            break
+          }
+        }
+      }
+      type = parent
+    }
+    return path
+  }
+}

src/Util/defragmentItemContent.js

@@ -0,0 +1,57 @@
+
+import ID from '../Util/ID.js'
+import ItemJSON from '../Struct/ItemJSON.js'
+import ItemString from '../Struct/ItemString.js'
+
+/**
+ * Try to merge all items in os with their successors.
+ *
+ * Some transformations (like delete) fragment items.
+ * Item(c: 'ab') + Delete(1,1) + Delete(0, 1) -> Item(c: 'a',deleted);Item(c: 'b',deleted)
+ *
+ * This functions merges the fragmented nodes together:
+ * Item(c: 'a',deleted);Item(c: 'b',deleted) -> Item(c: 'ab', deleted)
+ *
+ * TODO: The Tree implementation does not support deletions in-spot.
+ *       This is why all deletions must be performed after the traversal.
+ *
+ */
+export function defragmentItemContent (y) {
+  const os = y.os
+  if (os.length < 2) {
+    return
+  }
+  let deletes = []
+  let node = os.findSmallestNode()
+  let next = node.next()
+  while (next !== null) {
+    let a = node.val
+    let b = next.val
+    if (
+      (a instanceof ItemJSON || a instanceof ItemString) &&
+      a.constructor === b.constructor &&
+      a._deleted === b._deleted &&
+      a._right === b &&
+      (new ID(a._id.user, a._id.clock + a._length)).equals(b._id)
+    ) {
+      a._right = b._right
+      if (a instanceof ItemJSON) {
+        a._content = a._content.concat(b._content)
+      } else if (a instanceof ItemString) {
+        a._content += b._content
+      }
+      // delete b later
+      deletes.push(b._id)
+      // do not iterate node!
+      // !(node = next)
+    } else {
+      // not able to merge node, get next node
+      node = next
+    }
+    // update next
+    next = next.next()
+  }
+  for (let i = deletes.length - 1; i >= 0; i--) {
+    os.delete(deletes[i])
+  }
+}

src/Util/generateUserID.js

@@ -0,0 +1,16 @@
+/* global crypto */
+
+export function generateUserID () {
+  if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) {
+    // browser
+    let arr = new Uint32Array(1)
+    crypto.getRandomValues(arr)
+    return arr[0]
+  } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) {
+    // node
+    let buf = crypto.randomBytes(4)
+    return new Uint32Array(buf.buffer)[0]
+  } else {
+    return Math.ceil(Math.random() * 0xFFFFFFFF)
+  }
+}

src/Util/relativePosition.js

@@ -0,0 +1,59 @@
+import ID from './ID.js'
+import RootID from './RootID.js'
+
+export function getRelativePosition (type, offset) {
+  if (offset === 0) {
+    return ['startof', type._id.user, type._id.clock || null, type._id.name || null, type._id.type || null]
+  } else {
+    let t = type._start
+    while (t !== null) {
+      if (t._length >= offset) {
+        return [t._id.user, t._id.clock + offset - 1]
+      }
+      if (t._right === null) {
+        return [t._id.user, t._id.clock + t._length - 1]
+      }
+      if (!t._deleted) {
+        offset -= t._length
+      }
+      t = t._right
+    }
+    return null
+  }
+}
+
+export function fromRelativePosition (y, rpos) {
+  if (rpos[0] === 'startof') {
+    let id
+    if (rpos[3] === null) {
+      id = new ID(rpos[1], rpos[2])
+    } else {
+      id = new RootID(rpos[3], rpos[4])
+    }
+    return {
+      type: y.os.get(id),
+      offset: 0
+    }
+  } else {
+    let offset = 0
+    let struct = y.os.findNodeWithUpperBound(new ID(rpos[0], rpos[1])).val
+    const parent = struct._parent
+    if (parent._deleted) {
+      return null
+    }
+    if (!struct._deleted) {
+      offset = rpos[1] - struct._id.clock + 1
+    }
+    struct = struct._left
+    while (struct !== null) {
+      if (!struct._deleted) {
+        offset += struct._length
+      }
+      struct = struct._left
+    }
+    return {
+      type: parent,
+      offset: offset
+    }
+  }
+}

src/Util/structReferences.js

@@ -0,0 +1,37 @@
+import YArray from '../Type/YArray.js'
+import YMap from '../Type/YMap.js'
+import YText from '../Type/YText.js'
+import YXmlFragment from '../Type/y-xml/YXmlFragment.js'
+import YXmlElement from '../Type/y-xml/YXmlElement.js'
+import YXmlText from '../Type/y-xml/YXmlText.js'
+
+import Delete from '../Struct/Delete.js'
+import ItemJSON from '../Struct/ItemJSON.js'
+import ItemString from '../Struct/ItemString.js'
+
+const structs = new Map()
+const references = new Map()
+
+function addStruct (reference, structConstructor) {
+  structs.set(reference, structConstructor)
+  references.set(structConstructor, reference)
+}
+
+export function getStruct (reference) {
+  return structs.get(reference)
+}
+
+export function getReference (typeConstructor) {
+  return references.get(typeConstructor)
+}
+
+addStruct(0, ItemJSON)
+addStruct(1, ItemString)
+addStruct(2, Delete)
+
+addStruct(3, YArray)
+addStruct(4, YMap)
+addStruct(5, YText)
+addStruct(6, YXmlFragment)
+addStruct(7, YXmlElement)
+addStruct(8, YXmlText)

src/Util/writeJSONToType.js

@@ -0,0 +1,33 @@
+
+import YMap from '../Type/YMap'
+import YArray from '../Type/YArray'
+
+export function writeObjectToYMap (object, type) {
+  for (var key in object) {
+    var val = object[key]
+    if (Array.isArray(val)) {
+      type.set(key, YArray)
+      writeArrayToYArray(val, type.get(key))
+    } else if (typeof val === 'object') {
+      type.set(key, YMap)
+      writeObjectToYMap(val, type.get(key))
+    } else {
+      type.set(key, val)
+    }
+  }
+}
+
+export function writeArrayToYArray (array, type) {
+  for (var i = array.length - 1; i >= 0; i--) {
+    var val = array[i]
+    if (Array.isArray(val)) {
+      type.insert(0, [YArray])
+      writeArrayToYArray(val, type.get(0))
+    } else if (typeof val === 'object') {
+      type.insert(0, [YMap])
+      writeObjectToYMap(val, type.get(0))
+    } else {
+      type.insert(0, [val])
+    }
+  }
+}

src/Utils.js

@@ -1,856 +0,0 @@
-/* globals crypto */
-
-import { BinaryDecoder, BinaryEncoder } from './Encoding.js'
-
-/*
-  EventHandler is an helper class for constructing custom types.
-
-  Why: When constructing custom types, you sometimes want your types to work
-  synchronous: E.g.
-  ``` Synchronous
-    mytype.setSomething("yay")
-    mytype.getSomething() === "yay"
-  ```
-  versus
-  ``` Asynchronous
-    mytype.setSomething("yay")
-    mytype.getSomething() === undefined
-    mytype.waitForSomething().then(function(){
-      mytype.getSomething() === "yay"
-    })
-  ```
-
-  The structures usually work asynchronously (you have to wait for the
-  database request to finish). EventHandler helps you to make your type
-  synchronous.
-*/
-
-export default function Utils (Y) {
-  Y.utils = {
-    BinaryDecoder: BinaryDecoder,
-    BinaryEncoder: BinaryEncoder
-  }
-
-  Y.utils.bubbleEvent = function (type, event) {
-    type.eventHandler.callEventListeners(event)
-    event.path = []
-    while (type != null && type._deepEventHandler != null) {
-      type._deepEventHandler.callEventListeners(event)
-      var parent = null
-      if (type._parent != null) {
-        parent = type.os.getType(type._parent)
-      }
-      if (parent != null && parent._getPathToChild != null) {
-        event.path = [parent._getPathToChild(type._model)].concat(event.path)
-        type = parent
-      } else {
-        type = null
-      }
-    }
-  }
-
-  class NamedEventHandler {
-    constructor () {
-      this._eventListener = {}
-    }
-    on (name, f) {
-      if (this._eventListener[name] == null) {
-        this._eventListener[name] = []
-      }
-      this._eventListener[name].push(f)
-    }
-    off (name, f) {
-      if (name == null || f == null) {
-        throw new Error('You must specify event name and function!')
-      }
-      let listener = this._eventListener[name] || []
-      this._eventListener[name] = listener.filter(e => e !== f)
-    }
-    emit (name, value) {
-      (this._eventListener[name] || []).forEach(l => l(value))
-    }
-    destroy () {
-      this._eventListener = null
-    }
-  }
-  Y.utils.NamedEventHandler = NamedEventHandler
-
-  class EventListenerHandler {
-    constructor () {
-      this.eventListeners = []
-    }
-    destroy () {
-      this.eventListeners = null
-    }
-     /*
-      Basic event listener boilerplate...
-    */
-    addEventListener (f) {
-      this.eventListeners.push(f)
-    }
-    removeEventListener (f) {
-      this.eventListeners = this.eventListeners.filter(function (g) {
-        return f !== g
-      })
-    }
-    removeAllEventListeners () {
-      this.eventListeners = []
-    }
-    callEventListeners (event) {
-      for (var i = 0; i < this.eventListeners.length; i++) {
-        try {
-          var _event = {}
-          for (var name in event) {
-            _event[name] = event[name]
-          }
-          this.eventListeners[i](_event)
-        } catch (e) {
-          /*
-            Your observer threw an error. This error was caught so that Yjs
-            can ensure data consistency! In order to debug this error you
-            have to check "Pause On Caught Exceptions" in developer tools.
-          */
-          console.error(e)
-        }
-      }
-    }
-  }
-  Y.utils.EventListenerHandler = EventListenerHandler
-
-  class EventHandler extends EventListenerHandler {
-    /* ::
-    waiting: Array<Insertion | Deletion>;
-    awaiting: number;
-    onevent: Function;
-    eventListeners: Array<Function>;
-    */
-    /*
-      onevent: is called when the structure changes.
-
-      Note: "awaiting opertations" is used to denote operations that were
-      prematurely called. Events for received operations can not be executed until
-      all prematurely called operations were executed ("waiting operations")
-    */
-    constructor (onevent /* : Function */) {
-      super()
-      this.waiting = []
-      this.awaiting = 0
-      this.onevent = onevent
-    }
-    destroy () {
-      super.destroy()
-      this.waiting = null
-      this.onevent = null
-    }
-    /*
-      Call this when a new operation arrives. It will be executed right away if
-      there are no waiting operations, that you prematurely executed
-    */
-    receivedOp (op) {
-      if (this.awaiting <= 0) {
-        this.onevent(op)
-      } else if (op.struct === 'Delete') {
-        var self = this
-        var checkDelete = function checkDelete (d) {
-          if (d.length == null) {
-            throw new Error('This shouldn\'t happen! d.length must be defined!')
-          }
-          // we check if o deletes something in self.waiting
-          // if so, we remove the deleted operation
-          for (var w = 0; w < self.waiting.length; w++) {
-            var i = self.waiting[w]
-            if (i.struct === 'Insert' && i.id[0] === d.target[0]) {
-              var iLength = i.hasOwnProperty('content') ? i.content.length : 1
-              var dStart = d.target[1]
-              var dEnd = d.target[1] + (d.length || 1)
-              var iStart = i.id[1]
-              var iEnd = i.id[1] + iLength
-              // Check if they don't overlap
-              if (iEnd <= dStart || dEnd <= iStart) {
-                // no overlapping
-                continue
-              }
-              // we check all overlapping cases. All cases:
-              /*
-                1)  iiiii
-                      ddddd
-                    --> modify i and d
-                2)  iiiiiii
-                      ddddd
-                    --> modify i, remove d
-                3)  iiiiiii
-                      ddd
-                    --> remove d, modify i, and create another i (for the right hand side)
-                4)  iiiii
-                    ddddddd
-                    --> remove i, modify d
-                5)  iiiiiii
-                    ddddddd
-                    --> remove both i and d (**)
-                6)  iiiiiii
-                    ddddd
-                    --> modify i, remove d
-                7)    iii
-                    ddddddd
-                    --> remove i, create and apply two d with checkDelete(d) (**)
-                8)    iiiii
-                    ddddddd
-                    --> remove i, modify d (**)
-                9)    iiiii
-                    ddddd
-                    --> modify i and d
-                (**) (also check if i contains content or type)
-              */
-              // TODO: I left some debugger statements, because I want to debug all cases once in production. REMEMBER END TODO
-              if (iStart < dStart) {
-                if (dStart < iEnd) {
-                  if (iEnd < dEnd) {
-                    // Case 1
-                    // remove the right part of i's content
-                    i.content.splice(dStart - iStart)
-                    // remove the start of d's deletion
-                    d.length = dEnd - iEnd
-                    d.target = [d.target[0], iEnd]
-                    continue
-                  } else if (iEnd === dEnd) {
-                    // Case 2
-                    i.content.splice(dStart - iStart)
-                    // remove d, we do that by simply ending this function
-                    return
-                  } else { // (dEnd < iEnd)
-                    // Case 3
-                    var newI = {
-                      id: [i.id[0], dEnd],
-                      content: i.content.slice(dEnd - iStart),
-                      struct: 'Insert'
-                    }
-                    self.waiting.push(newI)
-                    i.content.splice(dStart - iStart)
-                    return
-                  }
-                }
-              } else if (dStart === iStart) {
-                if (iEnd < dEnd) {
-                  // Case 4
-                  d.length = dEnd - iEnd
-                  d.target = [d.target[0], iEnd]
-                  i.content = []
-                  continue
-                } else if (iEnd === dEnd) {
-                  // Case 5
-                  self.waiting.splice(w, 1)
-                  return
-                } else { // (dEnd < iEnd)
-                  // Case 6
-                  i.content = i.content.slice(dEnd - iStart)
-                  i.id = [i.id[0], dEnd]
-                  return
-                }
-              } else { // (dStart < iStart)
-                if (iStart < dEnd) {
-                  // they overlap
-                  /*
-                  7)    iii
-                      ddddddd
-                      --> remove i, create and apply two d with checkDelete(d) (**)
-                  8)    iiiii
-                      ddddddd
-                      --> remove i, modify d (**)
-                  9)    iiiii
-                      ddddd
-                      --> modify i and d
-                  */
-                  if (iEnd < dEnd) {
-                    // Case 7
-                    // debugger // TODO: You did not test this case yet!!!! (add the debugger here)
-                    self.waiting.splice(w, 1)
-                    checkDelete({
-                      target: [d.target[0], dStart],
-                      length: iStart - dStart,
-                      struct: 'Delete'
-                    })
-                    checkDelete({
-                      target: [d.target[0], iEnd],
-                      length: iEnd - dEnd,
-                      struct: 'Delete'
-                    })
-                    return
-                  } else if (iEnd === dEnd) {
-                    // Case 8
-                    self.waiting.splice(w, 1)
-                    w--
-                    d.length -= iLength
-                    continue
-                  } else { // dEnd < iEnd
-                    // Case 9
-                    d.length = iStart - dStart
-                    i.content.splice(0, dEnd - iStart)
-                    i.id = [i.id[0], dEnd]
-                    continue
-                  }
-                }
-              }
-            }
-          }
-          // finished with remaining operations
-          self.waiting.push(d)
-        }
-        if (op.key == null) {
-          // deletes in list
-          checkDelete(op)
-        } else {
-          // deletes in map
-          this.waiting.push(op)
-        }
-      } else {
-        this.waiting.push(op)
-      }
-    }
-    /*
-      You created some operations, and you want the `onevent` function to be
-      called right away. Received operations will not be executed untill all
-      prematurely called operations are executed
-    */
-    awaitAndPrematurelyCall (ops) {
-      this.awaiting++
-      ops.map(Y.utils.copyOperation).forEach(this.onevent)
-    }
-    * awaitOps (transaction, f, args) {
-      function notSoSmartSort (array) {
-        // this function sorts insertions in a executable order
-        var result = []
-        while (array.length > 0) {
-          for (var i = 0; i < array.length; i++) {
-            var independent = true
-            for (var j = 0; j < array.length; j++) {
-              if (Y.utils.matchesId(array[j], array[i].left)) {
-                // array[i] depends on array[j]
-                independent = false
-                break
-              }
-            }
-            if (independent) {
-              result.push(array.splice(i, 1)[0])
-              i--
-            }
-          }
-        }
-        return result
-      }
-      var before = this.waiting.length
-      // somehow create new operations
-      yield * f.apply(transaction, args)
-      // remove all appended ops / awaited ops
-      this.waiting.splice(before)
-      if (this.awaiting > 0) this.awaiting--
-      // if there are no awaited ops anymore, we can update all waiting ops, and send execute them (if there are still no awaited ops)
-      if (this.awaiting === 0 && this.waiting.length > 0) {
-        // update all waiting ops
-        for (let i = 0; i < this.waiting.length; i++) {
-          var o = this.waiting[i]
-          if (o.struct === 'Insert') {
-            var _o = yield * transaction.getInsertion(o.id)
-            if (_o.parentSub != null && _o.left != null) {
-              // if o is an insertion of a map struc (parentSub is defined), then it shouldn't be necessary to compute left
-              this.waiting.splice(i, 1)
-              i-- // update index
-            } else if (!Y.utils.compareIds(_o.id, o.id)) {
-              // o got extended
-              o.left = [o.id[0], o.id[1] - 1]
-            } else if (_o.left == null) {
-              o.left = null
-            } else {
-              // find next undeleted op
-              var left = yield * transaction.getInsertion(_o.left)
-              while (left.deleted != null) {
-                if (left.left != null) {
-                  left = yield * transaction.getInsertion(left.left)
-                } else {
-                  left = null
-                  break
-                }
-              }
-              o.left = left != null ? Y.utils.getLastId(left) : null
-            }
-          }
-        }
-        // the previous stuff was async, so we have to check again!
-        // We also pull changes from the bindings, if there exists such a method, this could increase awaiting too
-        if (this._pullChanges != null) {
-          this._pullChanges()
-        }
-        if (this.awaiting === 0) {
-          // sort by type, execute inserts first
-          var ins = []
-          var dels = []
-          this.waiting.forEach(function (o) {
-            if (o.struct === 'Delete') {
-              dels.push(o)
-            } else {
-              ins.push(o)
-            }
-          })
-          this.waiting = []
-          // put in executable order
-          ins = notSoSmartSort(ins)
-          // this.onevent can trigger the creation of another operation
-          // -> check if this.awaiting increased & stop computation if it does
-          for (var i = 0; i < ins.length; i++) {
-            if (this.awaiting === 0) {
-              this.onevent(ins[i])
-            } else {
-              this.waiting = this.waiting.concat(ins.slice(i))
-              break
-            }
-          }
-          for (i = 0; i < dels.length; i++) {
-            if (this.awaiting === 0) {
-              this.onevent(dels[i])
-            } else {
-              this.waiting = this.waiting.concat(dels.slice(i))
-              break
-            }
-          }
-        }
-      }
-    }
-    // TODO: Remove awaitedInserts and awaitedDeletes in favor of awaitedOps, as they are deprecated and do not always work
-    // Do this in one of the coming releases that are breaking anyway
-    /*
-      Call this when you successfully awaited the execution of n Insert operations
-    */
-    awaitedInserts (n) {
-      var ops = this.waiting.splice(this.waiting.length - n)
-      for (var oid = 0; oid < ops.length; oid++) {
-        var op = ops[oid]
-        if (op.struct === 'Insert') {
-          for (var i = this.waiting.length - 1; i >= 0; i--) {
-            let w = this.waiting[i]
-            // TODO: do I handle split operations correctly here? Super unlikely, but yeah..
-            // Also: can this case happen? Can op be inserted in the middle of a larger op that is in $waiting?
-            if (w.struct === 'Insert') {
-              if (Y.utils.matchesId(w, op.left)) {
-                // include the effect of op in w
-                w.right = op.id
-                // exclude the effect of w in op
-                op.left = w.left
-              } else if (Y.utils.compareIds(w.id, op.right)) {
-                // similar..
-                w.left = Y.utils.getLastId(op)
-                op.right = w.right
-              }
-            }
-          }
-        } else {
-          throw new Error('Expected Insert Operation!')
-        }
-      }
-      this._tryCallEvents(n)
-    }
-    /*
-      Call this when you successfully awaited the execution of n Delete operations
-    */
-    awaitedDeletes (n, newLeft) {
-      var ops = this.waiting.splice(this.waiting.length - n)
-      for (var j = 0; j < ops.length; j++) {
-        var del = ops[j]
-        if (del.struct === 'Delete') {
-          if (newLeft != null) {
-            for (var i = 0; i < this.waiting.length; i++) {
-              let w = this.waiting[i]
-              // We will just care about w.left
-              if (w.struct === 'Insert' && Y.utils.compareIds(del.target, w.left)) {
-                w.left = newLeft
-              }
-            }
-          }
-        } else {
-          throw new Error('Expected Delete Operation!')
-        }
-      }
-      this._tryCallEvents(n)
-    }
-    /* (private)
-      Try to execute the events for the waiting operations
-    */
-    _tryCallEvents () {
-      function notSoSmartSort (array) {
-        var result = []
-        while (array.length > 0) {
-          for (var i = 0; i < array.length; i++) {
-            var independent = true
-            for (var j = 0; j < array.length; j++) {
-              if (Y.utils.matchesId(array[j], array[i].left)) {
-                // array[i] depends on array[j]
-                independent = false
-                break
-              }
-            }
-            if (independent) {
-              result.push(array.splice(i, 1)[0])
-              i--
-            }
-          }
-        }
-        return result
-      }
-      if (this.awaiting > 0) this.awaiting--
-      if (this.awaiting === 0 && this.waiting.length > 0) {
-        var ins = []
-        var dels = []
-        this.waiting.forEach(function (o) {
-          if (o.struct === 'Delete') {
-            dels.push(o)
-          } else {
-            ins.push(o)
-          }
-        })
-        ins = notSoSmartSort(ins)
-        ins.forEach(this.onevent)
-        dels.forEach(this.onevent)
-        this.waiting = []
-      }
-    }
-  }
-  Y.utils.EventHandler = EventHandler
-
-  /*
-    Default class of custom types!
-  */
-  class CustomType {
-    getPath () {
-      var parent = null
-      if (this._parent != null) {
-        parent = this.os.getType(this._parent)
-      }
-      if (parent != null && parent._getPathToChild != null) {
-        var firstKey = parent._getPathToChild(this._model)
-        var parentKeys = parent.getPath()
-        parentKeys.push(firstKey)
-        return parentKeys
-      } else {
-        return []
-      }
-    }
-  }
-  Y.utils.CustomType = CustomType
-
-  /*
-    A wrapper for the definition of a custom type.
-    Every custom type must have three properties:
-
-    * struct
-      - Structname of this type
-    * initType
-      - Given a model, creates a custom type
-    * class
-      - the constructor of the custom type (e.g. in order to inherit from a type)
-  */
-  class CustomTypeDefinition { // eslint-disable-line
-    /* ::
-    struct: any;
-    initType: any;
-    class: Function;
-    name: String;
-    */
-    constructor (def) {
-      if (def.struct == null ||
-        def.initType == null ||
-        def.class == null ||
-        def.name == null ||
-        def.createType == null
-      ) {
-        throw new Error('Custom type was not initialized correctly!')
-      }
-      this.struct = def.struct
-      this.initType = def.initType
-      this.createType = def.createType
-      this.class = def.class
-      this.name = def.name
-      if (def.appendAdditionalInfo != null) {
-        this.appendAdditionalInfo = def.appendAdditionalInfo
-      }
-      this.parseArguments = (def.parseArguments || function () {
-        return [this]
-      }).bind(this)
-      this.parseArguments.typeDefinition = this
-    }
-  }
-  Y.utils.CustomTypeDefinition = CustomTypeDefinition
-
-  Y.utils.isTypeDefinition = function isTypeDefinition (v) {
-    if (v != null) {
-      if (v instanceof Y.utils.CustomTypeDefinition) return [v]
-      else if (v.constructor === Array && v[0] instanceof Y.utils.CustomTypeDefinition) return v
-      else if (v instanceof Function && v.typeDefinition instanceof Y.utils.CustomTypeDefinition) return [v.typeDefinition]
-    }
-    return false
-  }
-
-  /*
-    Make a flat copy of an object
-    (just copy properties)
-  */
-  function copyObject (o) {
-    var c = {}
-    for (var key in o) {
-      c[key] = o[key]
-    }
-    return c
-  }
-  Y.utils.copyObject = copyObject
-
-  /*
-    Copy an operation, so that it can be manipulated.
-    Note: You must not change subproperties (except o.content)!
-  */
-  function copyOperation (o) {
-    o = copyObject(o)
-    if (o.content != null) {
-      o.content = o.content.map(function (c) { return c })
-    }
-    return o
-  }
-
-  Y.utils.copyOperation = copyOperation
-
-  /*
-    Defines a smaller relation on Id's
-  */
-  function smaller (a, b) {
-    return a[0] < b[0] || (a[0] === b[0] && (a[1] < b[1] || typeof a[1] < typeof b[1]))
-  }
-  Y.utils.smaller = smaller
-
-  function inDeletionRange (del, ins) {
-    return del.target[0] === ins[0] && del.target[1] <= ins[1] && ins[1] < del.target[1] + (del.length || 1)
-  }
-  Y.utils.inDeletionRange = inDeletionRange
-
-  function compareIds (id1, id2) {
-    if (id1 == null || id2 == null) {
-      return id1 === id2
-    } else {
-      return id1[0] === id2[0] && id1[1] === id2[1]
-    }
-  }
-  Y.utils.compareIds = compareIds
-
-  function matchesId (op, id) {
-    if (id == null || op == null) {
-      return id === op
-    } else {
-      if (id[0] === op.id[0]) {
-        if (op.content == null) {
-          return id[1] === op.id[1]
-        } else {
-          return id[1] >= op.id[1] && id[1] < op.id[1] + op.content.length
-        }
-      }
-    }
-    return false
-  }
-  Y.utils.matchesId = matchesId
-
-  function getLastId (op) {
-    if (op.content == null || op.content.length === 1) {
-      return op.id
-    } else {
-      return [op.id[0], op.id[1] + op.content.length - 1]
-    }
-  }
-  Y.utils.getLastId = getLastId
-
-  function createEmptyOpsArray (n) {
-    var a = new Array(n)
-    for (var i = 0; i < a.length; i++) {
-      a[i] = {
-        id: [null, null]
-      }
-    }
-    return a
-  }
-
-  function createSmallLookupBuffer (Store) {
-    /*
-      This buffer implements a very small buffer that temporarily stores operations
-      after they are read / before they are written.
-      The buffer basically implements FIFO. Often requested lookups will be re-queued every time they are looked up / written.
-
-      It can speed up lookups on Operation Stores and State Stores. But it does not require notable use of memory or processing power.
-
-      Good for os and ss, bot not for ds (because it often uses methods that require a flush)
-
-      I tried to optimize this for performance, therefore no highlevel operations.
-    */
-    class SmallLookupBuffer extends Store {
-      constructor (arg1, arg2) {
-        // super(...arguments) -- do this when this is supported by stable nodejs
-        super(arg1, arg2)
-        this.writeBuffer = createEmptyOpsArray(5)
-        this.readBuffer = createEmptyOpsArray(10)
-      }
-      * find (id, noSuperCall) {
-        var i, r
-        for (i = this.readBuffer.length - 1; i >= 0; i--) {
-          r = this.readBuffer[i]
-          // we don't have to use compareids, because id is always defined!
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            // found r
-            // move r to the end of readBuffer
-            for (; i < this.readBuffer.length - 1; i++) {
-              this.readBuffer[i] = this.readBuffer[i + 1]
-            }
-            this.readBuffer[this.readBuffer.length - 1] = r
-            return r
-          }
-        }
-        var o
-        for (i = this.writeBuffer.length - 1; i >= 0; i--) {
-          r = this.writeBuffer[i]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            o = r
-            break
-          }
-        }
-        if (i < 0 && noSuperCall === undefined) {
-          // did not reach break in last loop
-          // read id and put it to the end of readBuffer
-          o = yield * super.find(id)
-        }
-        if (o != null) {
-          for (i = 0; i < this.readBuffer.length - 1; i++) {
-            this.readBuffer[i] = this.readBuffer[i + 1]
-          }
-          this.readBuffer[this.readBuffer.length - 1] = o
-        }
-        return o
-      }
-      * put (o) {
-        var id = o.id
-        var i, r // helper variables
-        for (i = this.writeBuffer.length - 1; i >= 0; i--) {
-          r = this.writeBuffer[i]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            // is already in buffer
-            // forget r, and move o to the end of writeBuffer
-            for (; i < this.writeBuffer.length - 1; i++) {
-              this.writeBuffer[i] = this.writeBuffer[i + 1]
-            }
-            this.writeBuffer[this.writeBuffer.length - 1] = o
-            break
-          }
-        }
-        if (i < 0) {
-          // did not reach break in last loop
-          // write writeBuffer[0]
-          var write = this.writeBuffer[0]
-          if (write.id[0] !== null) {
-            yield * super.put(write)
-          }
-          // put o to the end of writeBuffer
-          for (i = 0; i < this.writeBuffer.length - 1; i++) {
-            this.writeBuffer[i] = this.writeBuffer[i + 1]
-          }
-          this.writeBuffer[this.writeBuffer.length - 1] = o
-        }
-        // check readBuffer for every occurence of o.id, overwrite if found
-        // whether found or not, we'll append o to the readbuffer
-        for (i = 0; i < this.readBuffer.length - 1; i++) {
-          r = this.readBuffer[i + 1]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            this.readBuffer[i] = o
-          } else {
-            this.readBuffer[i] = r
-          }
-        }
-        this.readBuffer[this.readBuffer.length - 1] = o
-      }
-      * delete (id) {
-        var i, r
-        for (i = 0; i < this.readBuffer.length; i++) {
-          r = this.readBuffer[i]
-          if (r.id[1] === id[1] && r.id[0] === id[0]) {
-            this.readBuffer[i] = {
-              id: [null, null]
-            }
-          }
-        }
-        yield * this.flush()
-        yield * super.delete(id)
-      }
-      * findWithLowerBound (id) {
-        var o = yield * this.find(id, true)
-        if (o != null) {
-          return o
-        } else {
-          yield * this.flush()
-          return yield * super.findWithLowerBound.apply(this, arguments)
-        }
-      }
-      * findWithUpperBound (id) {
-        var o = yield * this.find(id, true)
-        if (o != null) {
-          return o
-        } else {
-          yield * this.flush()
-          return yield * super.findWithUpperBound.apply(this, arguments)
-        }
-      }
-      * findNext () {
-        yield * this.flush()
-        return yield * super.findNext.apply(this, arguments)
-      }
-      * findPrev () {
-        yield * this.flush()
-        return yield * super.findPrev.apply(this, arguments)
-      }
-      * iterate () {
-        yield * this.flush()
-        yield * super.iterate.apply(this, arguments)
-      }
-      * flush () {
-        for (var i = 0; i < this.writeBuffer.length; i++) {
-          var write = this.writeBuffer[i]
-          if (write.id[0] !== null) {
-            yield * super.put(write)
-            this.writeBuffer[i] = {
-              id: [null, null]
-            }
-          }
-        }
-      }
-    }
-    return SmallLookupBuffer
-  }
-  Y.utils.createSmallLookupBuffer = createSmallLookupBuffer
-
-  function generateUserId () {
-    if (typeof crypto !== 'undefined' && crypto.getRandomValue != null) {
-      // browser
-      let arr = new Uint32Array(1)
-      crypto.getRandomValues(arr)
-      return arr[0]
-    } else if (typeof crypto !== 'undefined' && crypto.randomBytes != null) {
-      // node
-      let buf = crypto.randomBytes(4)
-      return new Uint32Array(buf.buffer)[0]
-    } else {
-      return Math.ceil(Math.random() * 0xFFFFFFFF)
-    }
-  }
-  Y.utils.generateUserId = generateUserId
-
-  Y.utils.parseTypeDefinition = function parseTypeDefinition (type, typeArgs) {
-    var args = []
-    try {
-      args = JSON.parse('[' + typeArgs + ']')
-    } catch (e) {
-      throw new Error('Was not able to parse type definition!')
-    }
-    if (type.typeDefinition.parseArguments != null) {
-      args = type.typeDefinition.parseArguments(args[0])[1]
-    }
-    return args
-  }
-}

src/Y.js

@@ -0,0 +1,176 @@
+import DeleteStore from './Store/DeleteStore.js'
+import OperationStore from './Store/OperationStore.js'
+import StateStore from './Store/StateStore.js'
+import { generateUserID } from './Util/generateUserID.js'
+import RootID from './Util/RootID.js'
+import NamedEventHandler from './Util/NamedEventHandler.js'
+import UndoManager from './Util/UndoManager.js'
+
+import { messageToString, messageToRoomname } from './MessageHandler/messageToString.js'
+
+import Connector from './Connector.js'
+import Persistence from './Persistence.js'
+import YArray from './Type/YArray.js'
+import YMap from './Type/YMap.js'
+import YText from './Type/YText.js'
+import { YXmlFragment, YXmlElement, YXmlText } from './Type/y-xml/y-xml.js'
+import BinaryDecoder from './Binary/Decoder.js'
+
+import debug from 'debug'
+import Transaction from './Transaction.js'
+
+export default class Y extends NamedEventHandler {
+  constructor (opts) {
+    super()
+    this._opts = opts
+    this.userID = opts._userID != null ? opts._userID : generateUserID()
+    this.share = {}
+    this.ds = new DeleteStore(this)
+    this.os = new OperationStore(this)
+    this.ss = new StateStore(this)
+    this.connector = new Y[opts.connector.name](this, opts.connector)
+    if (opts.persistence != null) {
+      this.persistence = new Y[opts.persistence.name](this, opts.persistence)
+      this.persistence.retrieveContent()
+    } else {
+      this.persistence = null
+    }
+    this.connected = true
+    this._missingStructs = new Map()
+    this._readyToIntegrate = []
+    this._transaction = null
+  }
+  _beforeChange () {}
+  transact (f, remote = false) {
+    let initialCall = this._transaction === null
+    if (initialCall) {
+      this.emit('beforeTransaction', this, remote)
+      this._transaction = new Transaction(this)
+    }
+    try {
+      f(this)
+    } catch (e) {
+      console.error(e)
+    }
+    if (initialCall) {
+      // emit change events on changed types
+      this._transaction.changedTypes.forEach(function (subs, type) {
+        if (!type._deleted) {
+          type._callObserver(subs, remote)
+        }
+      })
+      this._transaction.changedParentTypes.forEach(function (events, type) {
+        if (!type._deleted) {
+          events = events
+            .filter(event =>
+              !event.target._deleted
+            )
+          events
+            .forEach(event => {
+              event.currentTarget = type
+            })
+          // we don't have to check for events.length
+          // because there is no way events is empty..
+          type._deepEventHandler.callEventListeners(events)
+        }
+      })
+      // when all changes & events are processed, emit afterTransaction event
+      this.emit('afterTransaction', this, remote)
+      this._transaction = null
+    }
+  }
+  // fake _start for root properties (y.set('name', type))
+  get _start () {
+    return null
+  }
+  set _start (start) {
+    return null
+  }
+  get room () {
+    return this._opts.connector.room
+  }
+  define (name, TypeConstructor) {
+    let id = new RootID(name, TypeConstructor)
+    let type = this.os.get(id)
+    if (type === null) {
+      type = new TypeConstructor()
+      type._id = id
+      type._parent = this
+      type._integrate(this)
+      if (this.share[name] !== undefined) {
+        throw new Error('Type is already defined with a different constructor!')
+      }
+    }
+    if (this.share[name] === undefined) {
+      this.share[name] = type
+    }
+    return type
+  }
+  get (name) {
+    return this.share[name]
+  }
+  disconnect () {
+    if (this.connected) {
+      this.connected = false
+      return this.connector.disconnect()
+    } else {
+      return Promise.resolve()
+    }
+  }
+  reconnect () {
+    if (!this.connected) {
+      this.connected = true
+      return this.connector.reconnect()
+    } else {
+      return Promise.resolve()
+    }
+  }
+  destroy () {
+    this.share = null
+    if (this.connector.destroy != null) {
+      this.connector.destroy()
+    } else {
+      this.connector.disconnect()
+    }
+    this.os = null
+    this.ds = null
+    this.ss = null
+  }
+  whenSynced () {
+    return new Promise(resolve => {
+      this.once('synced', () => {
+        resolve()
+      })
+    })
+  }
+}
+
+Y.extend = function extendYjs () {
+  for (var i = 0; i < arguments.length; i++) {
+    var f = arguments[i]
+    if (typeof f === 'function') {
+      f(Y)
+    } else {
+      throw new Error('Expected a function!')
+    }
+  }
+}
+
+// TODO: The following assignments should be moved to yjs-dist
+Y.AbstractConnector = Connector
+Y.Persisence = Persistence
+Y.Array = YArray
+Y.Map = YMap
+Y.Text = YText
+Y.XmlElement = YXmlElement
+Y.XmlFragment = YXmlFragment
+Y.XmlText = YXmlText
+
+Y.utils = {
+  BinaryDecoder,
+  UndoManager
+}
+
+Y.debug = debug
+debug.formatters.Y = messageToString
+debug.formatters.y = messageToRoomname

src/y-dist.cjs.js

@@ -0,0 +1,3 @@
+
+import Y from './Y.js'
+export default Y

src/y.js

@@ -1,259 +0,0 @@
-import extendConnector from './Connector.js'
-import extendPersistence from './Persistence.js'
-import extendDatabase from './Database.js'
-import extendTransaction from './Transaction.js'
-import extendStruct from './Struct.js'
-import extendUtils from './Utils.js'
-import debug from 'debug'
-import { formatYjsMessage, formatYjsMessageType } from './MessageHandler.js'
-
-extendConnector(Y)
-extendPersistence(Y)
-extendDatabase(Y)
-extendTransaction(Y)
-extendStruct(Y)
-extendUtils(Y)
-
-Y.debug = debug
-debug.formatters.Y = formatYjsMessage
-debug.formatters.y = formatYjsMessageType
-
-var requiringModules = {}
-
-Y.requiringModules = requiringModules
-
-Y.extend = function (name, value) {
-  if (arguments.length === 2 && typeof name === 'string') {
-    if (value instanceof Y.utils.CustomTypeDefinition) {
-      Y[name] = value.parseArguments
-    } else {
-      Y[name] = value
-    }
-    if (requiringModules[name] != null) {
-      requiringModules[name].resolve()
-      delete requiringModules[name]
-    }
-  } else {
-    for (var i = 0; i < arguments.length; i++) {
-      var f = arguments[i]
-      if (typeof f === 'function') {
-        f(Y)
-      } else {
-        throw new Error('Expected function!')
-      }
-    }
-  }
-}
-
-Y.requestModules = requestModules
-function requestModules (modules) {
-  var sourceDir
-  if (Y.sourceDir === null) {
-    sourceDir = null
-  } else {
-    sourceDir = Y.sourceDir || '/bower_components'
-  }
-  // determine if this module was compiled for es5 or es6 (y.js vs. y.es6)
-  // if Insert.execute is a Function, then it isnt a generator..
-  // then load the es5(.js) files..
-  var extention = typeof regeneratorRuntime !== 'undefined' ? '.js' : '.es6'
-  var promises = []
-  for (var i = 0; i < modules.length; i++) {
-    var module = modules[i].split('(')[0]
-    var modulename = 'y-' + module.toLowerCase()
-    if (Y[module] == null) {
-      if (requiringModules[module] == null) {
-        // module does not exist
-        if (typeof window !== 'undefined' && window.Y !== 'undefined') {
-          if (sourceDir != null) {
-            var imported = document.createElement('script')
-            imported.src = sourceDir + '/' + modulename + '/' + modulename + extention
-            document.head.appendChild(imported)
-          }
-          let requireModule = {}
-          requiringModules[module] = requireModule
-          requireModule.promise = new Promise(function (resolve) {
-            requireModule.resolve = resolve
-          })
-          promises.push(requireModule.promise)
-        } else {
-          console.info('YJS: Please do not depend on automatic requiring of modules anymore! Extend modules as follows `require(\'y-modulename\')(Y)`')
-          require(modulename)(Y)
-        }
-      } else {
-        promises.push(requiringModules[modules[i]].promise)
-      }
-    }
-  }
-  return Promise.all(promises)
-}
-
-/* ::
-type MemoryOptions = {
-  name: 'memory'
-}
-type IndexedDBOptions = {
-  name: 'indexeddb',
-  namespace: string
-}
-type DbOptions = MemoryOptions | IndexedDBOptions
-
-type WebRTCOptions = {
-  name: 'webrtc',
-  room: string
-}
-type WebsocketsClientOptions = {
-  name: 'websockets-client',
-  room: string
-}
-type ConnectionOptions = WebRTCOptions | WebsocketsClientOptions
-
-type YOptions = {
-  connector: ConnectionOptions,
-  db: DbOptions,
-  types: Array<TypeName>,
-  sourceDir: string,
-  share: {[key: string]: TypeName}
-}
-*/
-
-export default function Y (opts/* :YOptions */) /* :Promise<YConfig> */ {
-  if (opts.hasOwnProperty('sourceDir')) {
-    Y.sourceDir = opts.sourceDir
-  }
-  opts.types = opts.types != null ? opts.types : []
-  var modules = [opts.db.name, opts.connector.name].concat(opts.types)
-  for (var name in opts.share) {
-    modules.push(opts.share[name])
-  }
-  return new Promise(function (resolve, reject) {
-    if (opts == null) reject(new Error('An options object is expected!'))
-    else if (opts.connector == null) reject(new Error('You must specify a connector! (missing connector property)'))
-    else if (opts.connector.name == null) reject(new Error('You must specify connector name! (missing connector.name property)'))
-    else if (opts.db == null) reject(new Error('You must specify a database! (missing db property)'))
-    else if (opts.connector.name == null) reject(new Error('You must specify db name! (missing db.name property)'))
-    else {
-      opts = Y.utils.copyObject(opts)
-      opts.connector = Y.utils.copyObject(opts.connector)
-      opts.db = Y.utils.copyObject(opts.db)
-      opts.share = Y.utils.copyObject(opts.share)
-      Y.requestModules(modules).then(function () {
-        var yconfig = new YConfig(opts)
-        let resolved = false
-        if (opts.timeout != null && opts.timeout >= 0) {
-          setTimeout(function () {
-            if (!resolved) {
-              reject(new Error('Yjs init timeout'))
-              yconfig.destroy()
-            }
-          }, opts.timeout)
-        }
-        yconfig.db.whenUserIdSet(function () {
-          yconfig.init(function () {
-            resolved = true
-            resolve(yconfig)
-          }, reject)
-        })
-      }).catch(reject)
-    }
-  })
-}
-
-class YConfig extends Y.utils.NamedEventHandler {
-  /* ::
-  db: Y.AbstractDatabase;
-  connector: Y.AbstractConnector;
-  share: {[key: string]: any};
-  options: Object;
-  */
-  constructor (opts, callback) {
-    super()
-    this.options = opts
-    this.db = new Y[opts.db.name](this, opts.db)
-    this.connector = new Y[opts.connector.name](this, opts.connector)
-    if (opts.persistence != null) {
-      this.persistence = new Y[opts.persistence.name](this, opts.persistence)
-    } else {
-      this.persistence = null
-    }
-    this.connected = true
-  }
-  init (callback) {
-    var opts = this.options
-    var share = {}
-    this.share = share
-    this.db.requestTransaction(function * requestTransaction () {
-      // create shared object
-      for (var propertyname in opts.share) {
-        var typeConstructor = opts.share[propertyname].split('(')
-        let typeArgs = ''
-        if (typeConstructor.length === 2) {
-          typeArgs = typeConstructor[1].split(')')[0] || ''
-        }
-        var typeName = typeConstructor.splice(0, 1)
-        var type = Y[typeName]
-        var typedef = type.typeDefinition
-        var id = [0xFFFFFF, typedef.struct + '_' + typeName + '_' + propertyname + '_' + typeArgs]
-        let args = Y.utils.parseTypeDefinition(type, typeArgs)
-        share[propertyname] = yield * this.store.initType.call(this, id, args)
-      }
-    })
-    if (this.persistence != null) {
-      this.persistence.retrieveContent()
-        .then(() => this.db.whenTransactionsFinished())
-        .then(callback)
-    } else {
-      this.db.whenTransactionsFinished()
-        .then(callback)
-    }
-  }
-  isConnected () {
-    return this.connector.isSynced
-  }
-  disconnect () {
-    if (this.connected) {
-      this.connected = false
-      return this.connector.disconnect()
-    } else {
-      return Promise.resolve()
-    }
-  }
-  reconnect () {
-    if (!this.connected) {
-      this.connected = true
-      return this.connector.reconnect()
-    } else {
-      return Promise.resolve()
-    }
-  }
-  destroy () {
-    var self = this
-    return this.close().then(function () {
-      if (self.db.deleteDB != null) {
-        return self.db.deleteDB()
-      } else {
-        return Promise.resolve()
-      }
-    }).then(() => {
-      // remove existing event listener
-      super.destroy()
-    })
-  }
-  close () {
-    var self = this
-    this.share = null
-    if (this.connector.destroy != null) {
-      this.connector.destroy()
-    } else {
-      this.connector.disconnect()
-    }
-    return this.db.whenTransactionsFinished().then(function () {
-      self.db.destroyTypes()
-      // make sure to wait for all transactions before destroying the db
-      self.db.requestTransaction(function * () {
-        yield * self.db.destroy()
-      })
-      return self.db.whenTransactionsFinished()
-    })
-  }
-}

test/encode-decode.js

@@ -1,7 +1,8 @@
-import { test } from 'cutest'
+import { test } from '../node_modules/cutest/cutest.mjs'
+import BinaryEncoder from '../src/Binary/Encoder.js'
+import BinaryDecoder from '../src/Binary/Decoder.js'
+import { generateUserID } from '../src/Util/generateUserID.js'
 import Chance from 'chance'
-import Y from '../src/y.js'
-import { BinaryEncoder, BinaryDecoder } from '../src/Encoding.js'
 
 function testEncoding (t, write, read, val) {
   let encoder = new BinaryEncoder()
@@ -42,7 +43,7 @@ test('varUint random', async function varUintRandom (t) {
 
 test('varUint random user id', async function varUintRandomUserId (t) {
   t.getSeed() // enforces that this test is repeated
-  testEncoding(t, writeVarUint, readVarUint, Y.utils.generateUserId())
+  testEncoding(t, writeVarUint, readVarUint, generateUserID())
 })
 
 const writeVarString = (encoder, val) => encoder.writeVarString(val)
@@ -59,120 +60,3 @@ test('varString random', async function varStringRandom (t) {
   const chance = new Chance(t.getSeed() * 1000000000)
   testEncoding(t, writeVarString, readVarString, chance.string())
 })
-
-const writeDelete = Y.Struct.Delete.binaryEncode
-const readDelete = Y.Struct.Delete.binaryDecode
-
-test('encode/decode Delete operation', async function binDelete (t) {
-  let op = {
-    target: [10, 3000],
-    length: 40000,
-    struct: 'Delete'
-  }
-  testEncoding(t, writeDelete, readDelete, op)
-})
-
-const writeInsert = Y.Struct.Insert.binaryEncode
-const readInsert = Y.Struct.Insert.binaryDecode
-
-test('encode/decode Insert operations', async function binInsert (t) {
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: [5, 6],
-    left: [3, 4],
-    origin: [7, 8],
-    parent: [9, 10],
-    struct: 'Insert',
-    content: ['a']
-  })
-
-  t.log('left === origin')
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: [5, 6],
-    left: [3, 4],
-    origin: [3, 4],
-    parent: [9, 10],
-    struct: 'Insert',
-    content: ['a']
-  })
-
-  t.log('parentsub')
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: [5, 6],
-    left: [3, 4],
-    origin: [3, 4],
-    parent: [9, 10],
-    parentSub: 'sub',
-    struct: 'Insert',
-    content: ['a']
-  })
-
-  t.log('opContent')
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: [5, 6],
-    left: [3, 4],
-    origin: [3, 4],
-    parent: [9, 10],
-    struct: 'Insert',
-    opContent: [1000, 10000]
-  })
-
-  t.log('mixed content')
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: [5, 6],
-    left: [3, 4],
-    origin: [3, 4],
-    parent: [9, 10],
-    struct: 'Insert',
-    content: ['a', 1]
-  })
-
-  t.log('origin is null')
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: [5, 6],
-    left: [3, 4],
-    origin: null,
-    parent: [9, 10],
-    struct: 'Insert',
-    content: ['a']
-  })
-
-  t.log('left = origin = right = null')
-  testEncoding(t, writeInsert, readInsert, {
-    id: [1, 2],
-    right: null,
-    left: null,
-    origin: null,
-    parent: [9, 10],
-    struct: 'Insert',
-    content: ['a']
-  })
-})
-
-const writeList = Y.Struct.List.binaryEncode
-const readList = Y.Struct.List.binaryDecode
-
-test('encode/decode List operations', async function binList (t) {
-  testEncoding(t, writeList, readList, {
-    struct: 'List',
-    id: [100, 33],
-    type: 'Array'
-  })
-})
-
-const writeMap = Y.Struct.Map.binaryEncode
-const readMap = Y.Struct.Map.binaryDecode
-
-test('encode/decode Map operations', async function binMap (t) {
-  testEncoding(t, writeMap, readMap, {
-    struct: 'Map',
-    id: [100, 33],
-    type: 'Map',
-    map: {}
-  })
-})

test/index.html

@@ -0,0 +1,8 @@
+<!DOCTYPE html>
+<html>
+  <head>
+  </head>
+  <body>
+    <script type="module" src="./encode-decode.js"></script>
+  </body>
+</html>

test/red-black-tree.js

@@ -0,0 +1,192 @@
+import RedBlackTree from '../src/Util/Tree.js'
+import ID from '../src/Util/ID.js'
+import Chance from 'chance'
+import { test, proxyConsole } from 'cutest'
+
+proxyConsole()
+
+var numberOfRBTreeTests = 10000
+
+function checkRedNodesDoNotHaveBlackChildren (t, tree) {
+  let correct = true
+  function traverse (n) {
+    if (n == null) {
+      return
+    }
+    if (n.isRed()) {
+      if (n.left != null) {
+        correct = correct && !n.left.isRed()
+      }
+      if (n.right != null) {
+        correct = correct && !n.right.isRed()
+      }
+    }
+    traverse(n.left)
+    traverse(n.right)
+  }
+  traverse(tree.root)
+  t.assert(correct, 'Red nodes do not have black children')
+}
+
+function checkBlackHeightOfSubTreesAreEqual (t, tree) {
+  let correct = true
+  function traverse (n) {
+    if (n == null) {
+      return 0
+    }
+    var sub1 = traverse(n.left)
+    var sub2 = traverse(n.right)
+    if (sub1 !== sub2) {
+      correct = false
+    }
+    if (n.isRed()) {
+      return sub1
+    } else {
+      return sub1 + 1
+    }
+  }
+  traverse(tree.root)
+  t.assert(correct, 'Black-height of sub-trees are equal')
+}
+
+function checkRootNodeIsBlack (t, tree) {
+  t.assert(tree.root == null || tree.root.isBlack(), 'root node is black')
+}
+
+test('RedBlack Tree', async function redBlackTree (t) {
+  let tree = new RedBlackTree()
+  tree.put({_id: new ID(8433, 0)})
+  tree.put({_id: new ID(12844, 0)})
+  tree.put({_id: new ID(1795, 0)})
+  tree.put({_id: new ID(30302, 0)})
+  tree.put({_id: new ID(64287)})
+  tree.delete(new ID(8433, 0))
+  tree.put({_id: new ID(28996)})
+  tree.delete(new ID(64287))
+  tree.put({_id: new ID(22721)})
+  checkRootNodeIsBlack(t, tree)
+  checkBlackHeightOfSubTreesAreEqual(t, tree)
+  checkRedNodesDoNotHaveBlackChildren(t, tree)
+})
+
+test(`random tests (${numberOfRBTreeTests})`, async function random (t) {
+  let chance = new Chance(t.getSeed() * 1000000000)
+  let tree = new RedBlackTree()
+  let elements = []
+  for (var i = 0; i < numberOfRBTreeTests; i++) {
+    if (chance.bool({likelihood: 80})) {
+      // 80% chance to insert an element
+      let obj = new ID(chance.integer({min: 0, max: numberOfRBTreeTests}), chance.integer({min: 0, max: 1}))
+      let nodeExists = tree.find(obj)
+      if (nodeExists === null) {
+        if (elements.some(e => e.equals(obj))) {
+          t.assert(false, 'tree and elements contain different results')
+        }
+        elements.push(obj)
+        tree.put({_id: obj})
+      }
+    } else if (elements.length > 0) {
+      // ~20% chance to delete an element
+      var elem = chance.pickone(elements)
+      elements = elements.filter(function (e) {
+        return !e.equals(elem)
+      })
+      tree.delete(elem)
+    }
+  }
+  checkRootNodeIsBlack(t, tree)
+  checkBlackHeightOfSubTreesAreEqual(t, tree)
+  checkRedNodesDoNotHaveBlackChildren(t, tree)
+  // TEST if all nodes exist
+  let allNodesExist = true
+  for (let id of elements) {
+    let node = tree.find(id)
+    if (!node._id.equals(id)) {
+      allNodesExist = false
+    }
+  }
+  t.assert(allNodesExist, 'All inserted nodes exist')
+  // TEST lower bound search
+  let findAllNodesWithLowerBoundSerach = true
+  for (let id of elements) {
+    let node = tree.findWithLowerBound(id)
+    if (!node._id.equals(id)) {
+      findAllNodesWithLowerBoundSerach = false
+    }
+  }
+  t.assert(
+    findAllNodesWithLowerBoundSerach,
+    'Find every object with lower bound search'
+  )
+  // TEST iteration (with lower bound search)
+  let lowerBound = chance.pickone(elements)
+  let expectedResults = elements.filter((e, pos) =>
+    (lowerBound.lessThan(e) || e.equals(lowerBound)) &&
+    elements.indexOf(e) === pos
+  ).length
+  let actualResults = 0
+  tree.iterate(lowerBound, null, function (val) {
+    if (val == null) {
+      t.assert(false, 'val is undefined!')
+    }
+    actualResults++
+  })
+  t.assert(
+    expectedResults === actualResults,
+    'Iterating over a tree with lower bound yields the right amount of results'
+  )
+
+  expectedResults = elements.filter((e, pos) =>
+    elements.indexOf(e) === pos
+  ).length
+  actualResults = 0
+  tree.iterate(null, null, function (val) {
+    if (val == null) {
+      t.assert(false, 'val is undefined!')
+    }
+    actualResults++
+  })
+  t.assert(
+    expectedResults === actualResults,
+    'iterating over a tree without bounds yields the right amount of results'
+  )
+
+  let upperBound = chance.pickone(elements)
+  expectedResults = elements.filter((e, pos) =>
+    (e.lessThan(upperBound) || e.equals(upperBound)) &&
+    elements.indexOf(e) === pos
+  ).length
+  actualResults = 0
+  tree.iterate(null, upperBound, function (val) {
+    if (val == null) {
+      t.assert(false, 'val is undefined!')
+    }
+    actualResults++
+  })
+  t.assert(
+    expectedResults === actualResults,
+    'iterating over a tree with upper bound yields the right amount of results'
+  )
+
+  upperBound = chance.pickone(elements)
+  lowerBound = chance.pickone(elements)
+  if (upperBound.lessThan(lowerBound)) {
+    [lowerBound, upperBound] = [upperBound, lowerBound]
+  }
+  expectedResults = elements.filter((e, pos) =>
+    (lowerBound.lessThan(e) || e.equals(lowerBound)) &&
+    (e.lessThan(upperBound) || e.equals(upperBound)) &&
+    elements.indexOf(e) === pos
+  ).length
+  actualResults = 0
+  tree.iterate(lowerBound, upperBound, function (val) {
+    if (val == null) {
+      t.assert(false, 'val is undefined!')
+    }
+    actualResults++
+  })
+  t.assert(
+    expectedResults === actualResults,
+    'iterating over a tree with upper bound yields the right amount of results'
+  )
+})

test/y-array.tests.js

@@ -1,8 +1,7 @@
-import { wait, initArrays, compareUsers, Y, flushAll, garbageCollectUsers, applyRandomTests } from '../tests-lib/helper.js'
+import { wait, initArrays, compareUsers, Y, flushAll, applyRandomTests } from '../tests-lib/helper.js'
 import { test, proxyConsole } from 'cutest'
 
 proxyConsole()
-
 test('basic spec', async function array0 (t) {
   let { users, array0 } = await initArrays(t, { users: 2 })
 
@@ -11,11 +10,11 @@ test('basic spec', async function array0 (t) {
 
   let throwInvalidPosition = false
   try {
-    array0.delete(1, 0)
+    array0.delete(1, 1)
   } catch (e) {
     throwInvalidPosition = true
   }
-  t.assert(throwInvalidPosition, 'Throws when deleting zero elements with an invalid position')
+  t.assert(throwInvalidPosition, 'Throws when deleting with an invalid position')
 
   array0.insert(0, ['A'])
   array0.delete(1, 0)
@@ -27,9 +26,9 @@ test('basic spec', async function array0 (t) {
 test('insert three elements, try re-get property', async function array1 (t) {
   var { users, array0, array1 } = await initArrays(t, { users: 2 })
   array0.insert(0, [1, 2, 3])
-  t.compare(array0.toArray(), [1, 2, 3], '.toArray() works')
+  t.compare(array0.toJSON(), [1, 2, 3], '.toJSON() works')
   await flushAll(t, users)
-  t.compare(array1.toArray(), [1, 2, 3], '.toArray() works after sync')
+  t.compare(array1.toJSON(), [1, 2, 3], '.toJSON() works after sync')
   await compareUsers(t, users)
 })
 
@@ -38,7 +37,6 @@ test('concurrent insert (handle three conflicts)', async function array2 (t) {
   array0.insert(0, [0])
   array1.insert(0, [1])
   array2.insert(0, [2])
-
   await compareUsers(t, users)
 })
 
@@ -77,8 +75,8 @@ test('disconnect really prevents sending messages', async function array5 (t) {
   array0.insert(1, ['user0'])
   array1.insert(1, ['user1'])
   await wait(1000)
-  t.compare(array0.toArray(), ['x', 'user0', 'y'])
-  t.compare(array1.toArray(), ['x', 'user1', 'y'])
+  t.compare(array0.toJSON(), ['x', 'user0', 'y'])
+  t.compare(array1.toJSON(), ['x', 'user1', 'y'])
   await users[1].reconnect()
   await users[2].reconnect()
   await compareUsers(t, users)
@@ -126,24 +124,15 @@ test('insert & delete events', async function array8 (t) {
   })
   array0.insert(0, [0, 1, 2])
   compareEvent(t, event, {
-    type: 'insert',
-    index: 0,
-    values: [0, 1, 2],
-    length: 3
+    remote: false
   })
   array0.delete(0)
   compareEvent(t, event, {
-    type: 'delete',
-    index: 0,
-    length: 1,
-    values: [0]
+    remote: false
   })
   array0.delete(0, 2)
   compareEvent(t, event, {
-    type: 'delete',
-    index: 0,
-    length: 2,
-    values: [1, 2]
+    remote: false
   })
   await compareUsers(t, users)
 })
@@ -156,19 +145,11 @@ test('insert & delete events for types', async function array9 (t) {
   })
   array0.insert(0, [Y.Array])
   compareEvent(t, event, {
-    type: 'insert',
-    object: array0,
-    index: 0,
-    length: 1
+    remote: false
   })
-  var type = array0.get(0)
-  t.assert(type._model != null, 'Model of type is defined')
   array0.delete(0)
   compareEvent(t, event, {
-    type: 'delete',
-    object: array0,
-    index: 0,
-    length: 1
+    remote: false
   })
   await compareUsers(t, users)
 })
@@ -181,31 +162,19 @@ test('insert & delete events for types (2)', async function array10 (t) {
   })
   array0.insert(0, ['hi', Y.Map])
   compareEvent(t, events[0], {
-    type: 'insert',
-    object: array0,
-    index: 0,
-    length: 1,
-    values: ['hi']
-  })
-  compareEvent(t, events[1], {
-    type: 'insert',
-    object: array0,
-    index: 1,
-    length: 1
+    remote: false
   })
+  t.assert(events.length === 1, 'Event is triggered exactly once for insertion of two elements')
   array0.delete(1)
-  compareEvent(t, events[2], {
-    type: 'delete',
-    object: array0,
-    index: 1,
-    length: 1
+  compareEvent(t, events[1], {
+    remote: false
   })
+  t.assert(events.length === 2, 'Event is triggered exactly once for deletion')
   await compareUsers(t, users)
 })
 
 test('garbage collector', async function gc1 (t) {
   var { users, array0 } = await initArrays(t, { users: 3 })
-
   array0.insert(0, ['x', 'y', 'z'])
   await flushAll(t, users)
   users[0].disconnect()
@@ -213,64 +182,32 @@ test('garbage collector', async function gc1 (t) {
   await wait()
   await users[0].reconnect()
   await flushAll(t, users)
-  await garbageCollectUsers(t, users)
   await compareUsers(t, users)
 })
 
-test('event has correct value when setting a primitive on a YArray (same user)', async function array11 (t) {
-  var { array0, users } = await initArrays(t, { users: 3 })
-
+test('event target is set correctly (local)', async function array11 (t) {
+  let { array0, users } = await initArrays(t, { users: 3 })
   var event
   array0.observe(function (e) {
     event = e
   })
   array0.insert(0, ['stuff'])
-  t.assert(event.values[0] === event.object.get(0), 'compare value with get method')
-  t.assert(event.values[0] === 'stuff', 'check that value is actually present')
-  t.assert(event.values[0] === array0.toArray()[0], '.toArray works as expected')
+  t.assert(event.target === array0, '"target" property is set correctly')
   await compareUsers(t, users)
 })
 
-test('event has correct value when setting a primitive on a YArray (received from another user)', async function array12 (t) {
-  var { users, array0, array1 } = await initArrays(t, { users: 3 })
-
+test('event target is set correctly (remote user)', async function array12 (t) {
+  let { array0, array1, users } = await initArrays(t, { users: 3 })
   var event
   array0.observe(function (e) {
     event = e
   })
   array1.insert(0, ['stuff'])
   await flushAll(t, users)
-  t.assert(event.values[0] === event.object.get(0), 'compare value with get method')
-  t.assert(event.values[0] === 'stuff', 'check that value is actually present')
-  t.assert(event.values[0] === array0.toArray()[0], '.toArray works as expected')
-  await compareUsers(t, users)
-})
-
-test('event has correct value when setting a type on a YArray (same user)', async function array13 (t) {
-  var { array0, users } = await initArrays(t, { users: 3 })
-
-  var event
-  array0.observe(function (e) {
-    event = e
+  compareEvent(t, event, {
+    remote: true
   })
-  array0.insert(0, [Y.Array])
-  t.assert(event.values[0] === event.object.get(0), 'compare value with get method')
-  t.assert(event.values[0] != null, 'event.value exists')
-  t.assert(event.values[0] === array0.toArray()[0], '.toArray works as expected')
-  await compareUsers(t, users)
-})
-test('event has correct value when setting a type on a YArray (ops received from another user)', async function array14 (t) {
-  var { users, array0, array1 } = await initArrays(t, { users: 3 })
-
-  var event
-  array0.observe(function (e) {
-    event = e
-  })
-  array1.insert(0, [Y.Array])
-  await flushAll(t, users)
-  t.assert(event.values[0] === event.object.get(0), 'compare value with get method')
-  t.assert(event.values[0] != null, 'event.value exists')
-  t.assert(event.values[0] === array0.toArray()[0], '.toArray works as expected')
+  t.assert(event.target === array0, '"target" property is set correctly')
   await compareUsers(t, users)
 })
 
@@ -281,56 +218,60 @@ function getUniqueNumber () {
 
 var arrayTransactions = [
   function insert (t, user, chance) {
+    const yarray = user.get('array', Y.Array)
     var uniqueNumber = getUniqueNumber()
     var content = []
     var len = chance.integer({ min: 1, max: 4 })
     for (var i = 0; i < len; i++) {
       content.push(uniqueNumber)
     }
-    var pos = chance.integer({ min: 0, max: user.share.array.length })
-    user.share.array.insert(pos, content)
+    var pos = chance.integer({ min: 0, max: yarray.length })
+    yarray.insert(pos, content)
   },
   function insertTypeArray (t, user, chance) {
-    var pos = chance.integer({ min: 0, max: user.share.array.length })
-    user.share.array.insert(pos, [Y.Array])
-    var array2 = user.share.array.get(pos)
+    const yarray = user.get('array', Y.Array)
+    var pos = chance.integer({ min: 0, max: yarray.length })
+    yarray.insert(pos, [Y.Array])
+    var array2 = yarray.get(pos)
     array2.insert(0, [1, 2, 3, 4])
   },
   function insertTypeMap (t, user, chance) {
-    var pos = chance.integer({ min: 0, max: user.share.array.length })
-    user.share.array.insert(pos, [Y.Map])
-    var map = user.share.array.get(pos)
+    const yarray = user.get('array', Y.Array)
+    var pos = chance.integer({ min: 0, max: yarray.length })
+    yarray.insert(pos, [Y.Map])
+    var map = yarray.get(pos)
     map.set('someprop', 42)
     map.set('someprop', 43)
     map.set('someprop', 44)
   },
   function _delete (t, user, chance) {
-    var length = user.share.array._content.length
+    const yarray = user.get('array', Y.Array)
+    var length = yarray.length
     if (length > 0) {
-      var pos = chance.integer({ min: 0, max: length - 1 })
-      var delLength = chance.integer({ min: 1, max: Math.min(2, length - pos) })
-      if (user.share.array._content[pos].type != null) {
+      var somePos = chance.integer({ min: 0, max: length - 1 })
+      var delLength = chance.integer({ min: 1, max: Math.min(2, length - somePos) })
+      if (yarray instanceof Y.Array) {
         if (chance.bool()) {
-          var type = user.share.array.get(pos)
-          if (type instanceof Y.Array.typeDefinition.class) {
-            if (type._content.length > 0) {
-              pos = chance.integer({ min: 0, max: type._content.length - 1 })
-              delLength = chance.integer({ min: 0, max: Math.min(2, type._content.length - pos) })
-              type.delete(pos, delLength)
-            }
-          } else {
-            type.delete('someprop')
+          var type = yarray.get(somePos)
+          if (type.length > 0) {
+            somePos = chance.integer({ min: 0, max: type.length - 1 })
+            delLength = chance.integer({ min: 0, max: Math.min(2, type.length - somePos) })
+            type.delete(somePos, delLength)
           }
         } else {
-          user.share.array.delete(pos, delLength)
+          yarray.delete(somePos, delLength)
         }
       } else {
-        user.share.array.delete(pos, delLength)
+        yarray.delete(somePos, delLength)
       }
     }
   }
 ]
 
+test('y-array: Random tests (20)', async function randomArray20 (t) {
+  await applyRandomTests(t, arrayTransactions, 20)
+})
+
 test('y-array: Random tests (42)', async function randomArray42 (t) {
   await applyRandomTests(t, arrayTransactions, 42)
 })
@@ -355,20 +296,26 @@ test('y-array: Random tests (47)', async function randomArray47 (t) {
   await applyRandomTests(t, arrayTransactions, 47)
 })
 
-/*
-test('y-array: Random tests (200)', async function randomArray200 (t) {
+test('y-array: Random tests (300)', async function randomArray300 (t) {
   await applyRandomTests(t, arrayTransactions, 200)
 })
 
-test('y-array: Random tests (300)', async function randomArray300 (t) {
+test('y-array: Random tests (500)', async function randomArray500 (t) {
   await applyRandomTests(t, arrayTransactions, 300)
 })
 
-test('y-array: Random tests (400)', async function randomArray400 (t) {
+test('y-array: Random tests (600)', async function randomArray600 (t) {
   await applyRandomTests(t, arrayTransactions, 400)
 })
 
-test('y-array: Random tests (500)', async function randomArray500 (t) {
+test('y-array: Random tests (700)', async function randomArray700 (t) {
   await applyRandomTests(t, arrayTransactions, 500)
 })
-*/
+
+test('y-array: Random tests (1000)', async function randomArray1000 (t) {
+  await applyRandomTests(t, arrayTransactions, 1000)
+})
+
+test('y-array: Random tests (1800)', async function randomArray1800 (t) {
+  await applyRandomTests(t, arrayTransactions, 2000)
+})

test/y-map.tests.js

@@ -10,9 +10,9 @@ test('basic map tests', async function map0 (t) {
   map0.set('number', 1)
   map0.set('string', 'hello Y')
   map0.set('object', { key: { key2: 'value' } })
-  map0.set('y-map', Y.Map)
+  map0.set('y-map', new Y.Map())
   let map = map0.get('y-map')
-  map.set('y-array', Y.Array)
+  map.set('y-array', new Y.Array())
   let array = map.get('y-array')
   array.insert(0, [0])
   array.insert(0, [-1])
@@ -41,20 +41,24 @@ test('basic map tests', async function map0 (t) {
 test('Basic get&set of Map property (converge via sync)', async function map1 (t) {
   let { users, map0 } = await initArrays(t, { users: 2 })
   map0.set('stuff', 'stuffy')
+  map0.set('undefined', undefined)
+  map0.set('null', null)
   t.compare(map0.get('stuff'), 'stuffy')
 
   await flushAll(t, users)
 
   for (let user of users) {
-    var u = user.share.map
+    var u = user.get('map', Y.Map)
     t.compare(u.get('stuff'), 'stuffy')
+    t.assert(u.get('undefined') === undefined, 'undefined')
+    t.compare(u.get('null'), null, 'null')
   }
   await compareUsers(t, users)
 })
 
 test('Map can set custom types (Map)', async function map2 (t) {
   let { users, map0 } = await initArrays(t, { users: 2 })
-  var map = map0.set('Map', Y.Map)
+  var map = map0.set('Map', new Y.Map())
   map.set('one', 1)
   map = map0.get('Map')
   t.compare(map.get('one'), 1)
@@ -63,7 +67,7 @@ test('Map can set custom types (Map)', async function map2 (t) {
 
 test('Map can set custom types (Map) - get also returns the type', async function map3 (t) {
   let { users, map0 } = await initArrays(t, { users: 2 })
-  map0.set('Map', Y.Map)
+  map0.set('Map', new Y.Map())
   var map = map0.get('Map')
   map.set('one', 1)
   map = map0.get('Map')
@@ -73,7 +77,7 @@ test('Map can set custom types (Map) - get also returns the type', async functio
 
 test('Map can set custom types (Array)', async function map4 (t) {
   let { users, map0 } = await initArrays(t, { users: 2 })
-  var array = map0.set('Array', Y.Array)
+  var array = map0.set('Array', new Y.Array())
   array.insert(0, [1, 2, 3])
   array = map0.get('Array')
   t.compare(array.toArray(), [1, 2, 3])
@@ -88,7 +92,7 @@ test('Basic get&set of Map property (converge via update)', async function map5
   await flushAll(t, users)
 
   for (let user of users) {
-    var u = user.share.map
+    var u = user.get('map', Y.Map)
     t.compare(u.get('stuff'), 'stuffy')
   }
   await compareUsers(t, users)
@@ -102,7 +106,7 @@ test('Basic get&set of Map property (handle conflict)', async function map6 (t)
   await flushAll(t, users)
 
   for (let user of users) {
-    var u = user.share.map
+    var u = user.get('map', Y.Map)
     t.compare(u.get('stuff'), 'c0')
   }
   await compareUsers(t, users)
@@ -115,7 +119,7 @@ test('Basic get&set&delete of Map property (handle conflict)', async function ma
   map1.set('stuff', 'c1')
   await flushAll(t, users)
   for (let user of users) {
-    var u = user.share.map
+    var u = user.get('map', Y.Map)
     t.assert(u.get('stuff') === undefined)
   }
   await compareUsers(t, users)
@@ -129,7 +133,7 @@ test('Basic get&set of Map property (handle three conflicts)', async function ma
   map2.set('stuff', 'c3')
   await flushAll(t, users)
   for (let user of users) {
-    var u = user.share.map
+    var u = user.get('map', Y.Map)
     t.compare(u.get('stuff'), 'c0')
   }
   await compareUsers(t, users)
@@ -149,12 +153,13 @@ test('Basic get&set&delete of Map property (handle three conflicts)', async func
   map3.set('stuff', 'c3')
   await flushAll(t, users)
   for (let user of users) {
-    var u = user.share.map
+    var u = user.get('map', Y.Map)
     t.assert(u.get('stuff') === undefined)
   }
   await compareUsers(t, users)
 })
 
+/* TODO reimplement observePath
 test('observePath properties', async function map10 (t) {
   let { users, map0, map1, map2 } = await initArrays(t, { users: 3 })
   let map
@@ -163,56 +168,66 @@ test('observePath properties', async function map10 (t) {
       map.set('yay', 4)
     }
   })
-  map1.set('map', Y.Map)
+  map1.set('map', new Y.Map())
   await flushAll(t, users)
   map = map2.get('map')
   t.compare(map.get('yay'), 4)
   await compareUsers(t, users)
 })
+*/
 
 test('observe deep properties', async function map11 (t) {
   let { users, map1, map2, map3 } = await initArrays(t, { users: 4 })
-  var _map1 = map1.set('map', Y.Map)
+  var _map1 = map1.set('map', new Y.Map())
   var calls = 0
   var dmapid
-  _map1.observe(function (event) {
-    calls++
-    t.compare(event.name, 'deepmap')
-    dmapid = event.object.opContents.deepmap
+  map1.observeDeep(function (events) {
+    events.forEach(event => {
+      calls++
+      t.assert(event.keysChanged.has('deepmap'))
+      t.assert(event.path.length === 1)
+      t.assert(event.path[0] === 'map')
+      dmapid = event.target.get('deepmap')._id
+    })
   })
   await flushAll(t, users)
+  await flushAll(t, users)
   var _map3 = map3.get('map')
-  _map3.set('deepmap', Y.Map)
+  _map3.set('deepmap', new Y.Map())
   await flushAll(t, users)
   var _map2 = map2.get('map')
-  _map2.set('deepmap', Y.Map)
+  _map2.set('deepmap', new Y.Map())
+  await flushAll(t, users)
   await flushAll(t, users)
   var dmap1 = _map1.get('deepmap')
   var dmap2 = _map2.get('deepmap')
   var dmap3 = _map3.get('deepmap')
   t.assert(calls > 0)
-  t.compare(dmap1._model, dmap2._model)
-  t.compare(dmap1._model, dmap3._model)
-  t.compare(dmap1._model, dmapid)
+  t.assert(dmap1._id.equals(dmap2._id))
+  t.assert(dmap1._id.equals(dmap3._id))
+  t.assert(dmap1._id.equals(dmapid))
   await compareUsers(t, users)
 })
 
-test('observes using observePath', async function map12 (t) {
+test('observes using observeDeep', async function map12 (t) {
   let { users, map0 } = await initArrays(t, { users: 2 })
   var pathes = []
   var calls = 0
-  map0.observeDeep(function (event) {
-    pathes.push(event.path)
+  map0.observeDeep(function (events) {
+    events.forEach(event => {
+      pathes.push(event.path)
+    })
     calls++
   })
-  map0.set('map', Y.Map)
-  map0.get('map').set('array', Y.Array)
+  map0.set('map', new Y.Map())
+  map0.get('map').set('array', new Y.Array())
   map0.get('map').get('array').insert(0, ['content'])
   t.assert(calls === 3)
   t.compare(pathes, [[], ['map'], ['map', 'array']])
   await compareUsers(t, users)
 })
 
+/* TODO: Test events in Y.Map
 function compareEvent (t, is, should) {
   for (var key in should) {
     t.assert(should[key] === is[key])
@@ -233,7 +248,7 @@ test('throws add & update & delete events (with type and primitive content)', as
     name: 'stuff'
   })
   // update, oldValue is in contents
-  map0.set('stuff', Y.Array)
+  map0.set('stuff', new Y.Array())
   compareEvent(t, event, {
     type: 'update',
     object: map0,
@@ -255,7 +270,9 @@ test('throws add & update & delete events (with type and primitive content)', as
   })
   await compareUsers(t, users)
 })
+*/
 
+/* reimplement event.value somehow (probably with ss vector)
 test('event has correct value when setting a primitive on a YMap (same user)', async function map14 (t) {
   let { users, map0 } = await initArrays(t, { users: 3 })
   var event
@@ -264,7 +281,7 @@ test('event has correct value when setting a primitive on a YMap (same user)', a
     event = e
   })
   map0.set('stuff', 2)
-  t.compare(event.value, event.object.get(event.name))
+  t.compare(event.value, event.target.get(event.name))
   await compareUsers(t, users)
 })
 
@@ -277,46 +294,22 @@ test('event has correct value when setting a primitive on a YMap (received from
   })
   map1.set('stuff', 2)
   await flushAll(t, users)
-  t.compare(event.value, event.object.get(event.name))
-  await compareUsers(t, users)
-})
-
-test('event has correct value when setting a type on a YMap (same user)', async function map16 (t) {
-  let { users, map0 } = await initArrays(t, { users: 3 })
-  var event
-  await flushAll(t, users)
-  map0.observe(function (e) {
-    event = e
-  })
-  map0.set('stuff', Y.Map)
-  t.compare(event.value._model, event.object.get(event.name)._model)
-  await compareUsers(t, users)
-})
-
-test('event has correct value when setting a type on a YMap (ops received from another user)', async function map17 (t) {
-  let { users, map0, map1 } = await initArrays(t, { users: 3 })
-  var event
-  await flushAll(t, users)
-  map0.observe(function (e) {
-    event = e
-  })
-  map1.set('stuff', Y.Map)
-  await flushAll(t, users)
-  t.compare(event.value._model, event.object.get(event.name)._model)
+  t.compare(event.value, event.target.get(event.name))
   await compareUsers(t, users)
 })
+*/
 
 var mapTransactions = [
   function set (t, user, chance) {
     let key = chance.pickone(['one', 'two'])
     var value = chance.string()
-    user.share.map.set(key, value)
+    user.get('map', Y.Map).set(key, value)
   },
   function setType (t, user, chance) {
     let key = chance.pickone(['one', 'two'])
-    var value = chance.pickone([Y.Array, Y.Map])
-    let type = user.share.map.set(key, value)
-    if (value === Y.Array) {
+    var type = chance.pickone([new Y.Array(), new Y.Map()])
+    user.get('map', Y.Map).set(key, type)
+    if (type instanceof Y.Array) {
       type.insert(0, [1, 2, 3, 4])
     } else {
       type.set('deepkey', 'deepvalue')
@@ -324,7 +317,7 @@ var mapTransactions = [
   },
   function _delete (t, user, chance) {
     let key = chance.pickone(['one', 'two'])
-    user.share.map.delete(key)
+    user.get('map', Y.Map).delete(key)
   }
 ]
 
@@ -352,7 +345,6 @@ test('y-map: Random tests (47)', async function randomMap47 (t) {
   await applyRandomTests(t, mapTransactions, 47)
 })
 
-/*
 test('y-map: Random tests (200)', async function randomMap200 (t) {
   await applyRandomTests(t, mapTransactions, 200)
 })
@@ -361,11 +353,14 @@ test('y-map: Random tests (300)', async function randomMap300 (t) {
   await applyRandomTests(t, mapTransactions, 300)
 })
 
-test('y-map: Random tests (400)', async function randomMap400 (t) {
-  await applyRandomTests(t, mapTransactions, 400)
-})
-
 test('y-map: Random tests (500)', async function randomMap500 (t) {
   await applyRandomTests(t, mapTransactions, 500)
 })
-*/
+
+test('y-map: Random tests (1000)', async function randomMap1000 (t) {
+  await applyRandomTests(t, mapTransactions, 1000)
+})
+
+test('y-map: Random tests (1800)', async function randomMap1800 (t) {
+  await applyRandomTests(t, mapTransactions, 1800)
+})

test/y-xml.tests.js

@@ -10,6 +10,7 @@ test('set property', async function xml0 (t) {
   await compareUsers(t, users)
 })
 
+/* TODO: Test YXml events!
 test('events', async function xml1 (t) {
   var { users, xml0, xml1 } = await initArrays(t, { users: 2 })
   var event
@@ -50,7 +51,7 @@ test('events', async function xml1 (t) {
     type: 'childInserted',
     index: 0
   }
-  xml0.insert(0, [Y.XmlText('some text')])
+  xml0.insert(0, [new Y.XmlText('some text')])
   t.compare(event, expectedEvent, 'child inserted event')
   await flushAll(t, users)
   t.compare(remoteEvent, expectedEvent, 'child inserted event (remote)')
@@ -65,6 +66,7 @@ test('events', async function xml1 (t) {
   t.compare(remoteEvent, expectedEvent, 'child deleted event (remote)')
   await compareUsers(t, users)
 })
+*/
 
 test('attribute modifications (y -> dom)', async function xml2 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
@@ -110,8 +112,8 @@ test('element insert (dom -> y)', async function xml4 (t) {
 test('element insert (y -> dom)', async function xml5 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
   let dom0 = xml0.getDom()
-  xml0.insert(0, [Y.XmlText('some text')])
-  xml0.insert(1, [Y.XmlElement('p')])
+  xml0.insert(0, [new Y.XmlText('some text')])
+  xml0.insert(1, [new Y.XmlElement('p')])
   t.assert(dom0.childNodes[0].textContent === 'some text', 'Retrieve Text node')
   t.assert(dom0.childNodes[1].nodeName === 'P', 'Retrieve Element node')
   await compareUsers(t, users)
@@ -132,7 +134,7 @@ test('y on insert, then delete (dom -> y)', async function xml6 (t) {
 test('y on insert, then delete (y -> dom)', async function xml7 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
   let dom0 = xml0.getDom()
-  xml0.insert(0, [Y.XmlElement('p')])
+  xml0.insert(0, [new Y.XmlElement('p')])
   t.assert(dom0.childNodes[0].nodeName === 'P', 'Get inserted element from dom')
   xml0.delete(0, 1)
   t.assert(dom0.childNodes.length === 0, '#childNodes is empty after delete')
@@ -142,7 +144,7 @@ test('y on insert, then delete (y -> dom)', async function xml7 (t) {
 test('delete consecutive (1) (Text)', async function xml8 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
   let dom0 = xml0.getDom()
-  xml0.insert(0, ['1', '2', '3'].map(Y.XmlText))
+  xml0.insert(0, [new Y.XmlText('1'), new Y.XmlText('2'), new Y.XmlText('3')])
   await wait()
   xml0.delete(1, 2)
   await wait()
@@ -155,7 +157,7 @@ test('delete consecutive (1) (Text)', async function xml8 (t) {
 test('delete consecutive (2) (Text)', async function xml9 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
   let dom0 = xml0.getDom()
-  xml0.insert(0, ['1', '2', '3'].map(Y.XmlText))
+  xml0.insert(0, [new Y.XmlText('1'), new Y.XmlText('2'), new Y.XmlText('3')])
   await wait()
   xml0.delete(0, 1)
   xml0.delete(1, 1)
@@ -169,7 +171,7 @@ test('delete consecutive (2) (Text)', async function xml9 (t) {
 test('delete consecutive (1) (Element)', async function xml10 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
   let dom0 = xml0.getDom()
-  xml0.insert(0, [Y.XmlElement('A'), Y.XmlElement('B'), Y.XmlElement('C')])
+  xml0.insert(0, [new Y.XmlElement('A'), new Y.XmlElement('B'), new Y.XmlElement('C')])
   await wait()
   xml0.delete(1, 2)
   await wait()
@@ -182,7 +184,7 @@ test('delete consecutive (1) (Element)', async function xml10 (t) {
 test('delete consecutive (2) (Element)', async function xml11 (t) {
   var { users, xml0 } = await initArrays(t, { users: 3 })
   let dom0 = xml0.getDom()
-  xml0.insert(0, [Y.XmlElement('A'), Y.XmlElement('B'), Y.XmlElement('C')])
+  xml0.insert(0, [new Y.XmlElement('A'), new Y.XmlElement('B'), new Y.XmlElement('C')])
   await wait()
   xml0.delete(0, 1)
   xml0.delete(1, 1)
@@ -198,8 +200,8 @@ test('Receive a bunch of elements (with disconnect)', async function xml12 (t) {
   let dom0 = xml0.getDom()
   let dom1 = xml1.getDom()
   users[1].disconnect()
-  xml0.insert(0, [Y.XmlElement('A'), Y.XmlElement('B'), Y.XmlElement('C')])
-  xml0.insert(0, [Y.XmlElement('X'), Y.XmlElement('Y'), Y.XmlElement('Z')])
+  xml0.insert(0, [new Y.XmlElement('A'), new Y.XmlElement('B'), new Y.XmlElement('C')])
+  xml0.insert(0, [new Y.XmlElement('X'), new Y.XmlElement('Y'), new Y.XmlElement('Z')])
   await users[1].reconnect()
   await flushAll(t, users)
   t.assert(xml0.length === 6, 'check length (y)')
@@ -209,30 +211,130 @@ test('Receive a bunch of elements (with disconnect)', async function xml12 (t) {
   await compareUsers(t, users)
 })
 
+test('move element to a different position', async function xml13 (t) {
+  var { users, xml0, xml1 } = await initArrays(t, { users: 3 })
+  let dom0 = xml0.getDom()
+  let dom1 = xml1.getDom()
+  dom0.append(document.createElement('div'))
+  dom0.append(document.createElement('h1'))
+  await flushAll(t, users)
+  dom1.insertBefore(dom1.childNodes[0], null)
+  t.assert(dom1.childNodes[0].nodeName === 'H1', 'div was deleted (user 0)')
+  t.assert(dom1.childNodes[1].nodeName === 'DIV', 'div was moved to the correct position (user 0)')
+  t.assert(dom1.childNodes[0].nodeName === 'H1', 'div was deleted (user 1)')
+  t.assert(dom1.childNodes[1].nodeName === 'DIV', 'div was moved to the correct position (user 1)')
+  await compareUsers(t, users)
+})
+
+test('filter node', async function xml14 (t) {
+  var { users, xml0, xml1 } = await initArrays(t, { users: 3 })
+  let dom0 = xml0.getDom()
+  let dom1 = xml1.getDom()
+  let domFilter = (node, attrs) => {
+    if (node.nodeName === 'H1') {
+      return null
+    } else {
+      return attrs
+    }
+  }
+  xml0.setDomFilter(domFilter)
+  xml1.setDomFilter(domFilter)
+  dom0.append(document.createElement('div'))
+  dom0.append(document.createElement('h1'))
+  await flushAll(t, users)
+  t.assert(dom1.childNodes.length === 1, 'Only one node was not transmitted')
+  t.assert(dom1.childNodes[0].nodeName === 'DIV', 'div node was transmitted')
+  await compareUsers(t, users)
+})
+
+test('filter attribute', async function xml15 (t) {
+  var { users, xml0, xml1 } = await initArrays(t, { users: 3 })
+  let dom0 = xml0.getDom()
+  let dom1 = xml1.getDom()
+  let domFilter = (node, attrs) => {
+    return attrs.filter(name => name !== 'hidden')
+  }
+  xml0.setDomFilter(domFilter)
+  xml1.setDomFilter(domFilter)
+  dom0.setAttribute('hidden', 'true')
+  dom0.setAttribute('style', 'height: 30px')
+  dom0.setAttribute('data-me', '77')
+  await flushAll(t, users)
+  t.assert(dom0.getAttribute('hidden') === 'true', 'User 0 still has the attribute')
+  t.assert(dom1.getAttribute('hidden') == null, 'User 1 did not receive update')
+  t.assert(dom1.getAttribute('style') === 'height: 30px', 'User 1 received style update')
+  t.assert(dom1.getAttribute('data-me') === '77', 'User 1 received data update')
+  await compareUsers(t, users)
+})
+
+test('deep element insert', async function xml16 (t) {
+  var { users, xml0, xml1 } = await initArrays(t, { users: 3 })
+  let dom0 = xml0.getDom()
+  let dom1 = xml1.getDom()
+  let deepElement = document.createElement('p')
+  let boldElement = document.createElement('b')
+  let attrElement = document.createElement('img')
+  attrElement.setAttribute('src', 'http:localhost:8080/nowhere')
+  boldElement.append(document.createTextNode('hi'))
+  deepElement.append(boldElement)
+  deepElement.append(attrElement)
+  dom0.append(deepElement)
+  console.log(dom0.outerHTML)
+  let str0 = dom0.outerHTML
+  await flushAll(t, users)
+  let str1 = dom1.outerHTML
+  t.compare(str0, str1, 'Dom string representation matches')
+  await compareUsers(t, users)
+})
+
+test('treeWalker', async function xml17 (t) {
+  var { users, xml0 } = await initArrays(t, { users: 3 })
+  let paragraph1 = new Y.XmlElement('p')
+  let paragraph2 = new Y.XmlElement('p')
+  let text1 = new Y.Text('init')
+  let text2 = new Y.Text('text')
+  paragraph1.insert(0, [text1, text2])
+  xml0.insert(0, [paragraph1, paragraph2, new Y.XmlElement('img')])
+  let allParagraphs = xml0.querySelectorAll('p')
+  t.assert(allParagraphs.length === 2, 'found exactly two paragraphs')
+  t.assert(allParagraphs[0] === paragraph1, 'querySelectorAll found paragraph1')
+  t.assert(allParagraphs[1] === paragraph2, 'querySelectorAll found paragraph2')
+  t.assert(xml0.querySelector('p') === paragraph1, 'querySelector found paragraph1')
+  await compareUsers(t, users)
+})
+
 // TODO: move elements
 var xmlTransactions = [
   function attributeChange (t, user, chance) {
-    user.share.xml.getDom().setAttribute(chance.word(), chance.word())
+    user.get('xml', Y.XmlElement).getDom().setAttribute(chance.word(), chance.word())
+  },
+  function attributeChangeHidden (t, user, chance) {
+    user.get('xml', Y.XmlElement).getDom().setAttribute('hidden', chance.word())
   },
   function insertText (t, user, chance) {
-    let dom = user.share.xml.getDom()
+    let dom = user.get('xml', Y.XmlElement).getDom()
     var succ = dom.children.length > 0 ? chance.pickone(dom.children) : null
     dom.insertBefore(document.createTextNode(chance.word()), succ)
   },
+  function insertHiddenDom (t, user, chance) {
+    let dom = user.get('xml', Y.XmlElement).getDom()
+    var succ = dom.children.length > 0 ? chance.pickone(dom.children) : null
+    dom.insertBefore(document.createElement('hidden'), succ)
+  },
   function insertDom (t, user, chance) {
-    let dom = user.share.xml.getDom()
+    let dom = user.get('xml', Y.XmlElement).getDom()
     var succ = dom.children.length > 0 ? chance.pickone(dom.children) : null
     dom.insertBefore(document.createElement(chance.word()), succ)
   },
   function deleteChild (t, user, chance) {
-    let dom = user.share.xml.getDom()
+    let dom = user.get('xml', Y.XmlElement).getDom()
     if (dom.childNodes.length > 0) {
       var d = chance.pickone(dom.childNodes)
       d.remove()
     }
   },
   function insertTextSecondLayer (t, user, chance) {
-    let dom = user.share.xml.getDom()
+    let dom = user.get('xml', Y.XmlElement).getDom()
     if (dom.children.length > 0) {
       let dom2 = chance.pickone(dom.children)
       let succ = dom2.childNodes.length > 0 ? chance.pickone(dom2.childNodes) : null
@@ -240,7 +342,7 @@ var xmlTransactions = [
     }
   },
   function insertDomSecondLayer (t, user, chance) {
-    let dom = user.share.xml.getDom()
+    let dom = user.get('xml', Y.XmlElement).getDom()
     if (dom.children.length > 0) {
       let dom2 = chance.pickone(dom.children)
       let succ = dom2.childNodes.length > 0 ? chance.pickone(dom2.childNodes) : null
@@ -248,7 +350,7 @@ var xmlTransactions = [
     }
   },
   function deleteChildSecondLayer (t, user, chance) {
-    let dom = user.share.xml.getDom()
+    let dom = user.get('xml', Y.XmlElement).getDom()
     if (dom.children.length > 0) {
       let dom2 = chance.pickone(dom.children)
       if (dom2.childNodes.length > 0) {
@@ -259,30 +361,46 @@ var xmlTransactions = [
   }
 ]
 
-test('y-xml: Random tests (10)', async function randomXml10 (t) {
+test('y-xml: Random tests (10)', async function xmlRandom10 (t) {
   await applyRandomTests(t, xmlTransactions, 10)
 })
 
-test('y-xml: Random tests (42)', async function randomXml42 (t) {
+test('y-xml: Random tests (42)', async function xmlRandom42 (t) {
   await applyRandomTests(t, xmlTransactions, 42)
 })
 
-test('y-xml: Random tests (43)', async function randomXml43 (t) {
+test('y-xml: Random tests (43)', async function xmlRandom43 (t) {
   await applyRandomTests(t, xmlTransactions, 43)
 })
 
-test('y-xml: Random tests (44)', async function randomXml44 (t) {
+test('y-xml: Random tests (44)', async function xmlRandom44 (t) {
   await applyRandomTests(t, xmlTransactions, 44)
 })
 
-test('y-xml: Random tests (45)', async function randomXml45 (t) {
+test('y-xml: Random tests (45)', async function xmlRandom45 (t) {
   await applyRandomTests(t, xmlTransactions, 45)
 })
 
-test('y-xml: Random tests (46)', async function randomXml46 (t) {
+test('y-xml: Random tests (46)', async function xmlRandom46 (t) {
   await applyRandomTests(t, xmlTransactions, 46)
 })
 
-test('y-xml: Random tests (47)', async function randomXml47 (t) {
+test('y-xml: Random tests (47)', async function xmlRandom47 (t) {
   await applyRandomTests(t, xmlTransactions, 47)
 })
+
+test('y-xml: Random tests (100)', async function xmlRandom100 (t) {
+  await applyRandomTests(t, xmlTransactions, 100)
+})
+
+test('y-xml: Random tests (200)', async function xmlRandom200 (t) {
+  await applyRandomTests(t, xmlTransactions, 200)
+})
+
+test('y-xml: Random tests (500)', async function xmlRandom500 (t) {
+  await applyRandomTests(t, xmlTransactions, 500)
+})
+
+test('y-xml: Random tests (1000)', async function xmlRandom1000 (t) {
+  await applyRandomTests(t, xmlTransactions, 1000)
+})

tests-lib/helper.js

@@ -1,37 +1,32 @@
 
-import _Y from '../../yjs/src/y.js'
-
-import yMemory from '../../y-memory/src/y-memory.js'
-import yArray from '../../y-array/src/y-array.js'
-import yText from '../../y-text/src/Text.js'
-import yMap from '../../y-map/src/Map.js'
-import yXml from '../../y-xml/src/y-xml.js'
+import _Y from '../src/Y.js'
 import yTest from './test-connector.js'
 
 import Chance from 'chance'
+import ItemJSON from '../src/Struct/ItemJSON.js'
+import ItemString from '../src/Struct/ItemString.js'
+import { defragmentItemContent } from '../src/Util/defragmentItemContent.js'
 
-export let Y = _Y
+export const Y = _Y
 
-Y.extend(yMemory, yArray, yText, yMap, yTest, yXml)
+Y.extend(yTest)
 
-export var database = { name: 'memory' }
-export var connector = { name: 'test', url: 'http://localhost:1234' }
+export const database = { name: 'memory' }
+export const connector = { name: 'test', url: 'http://localhost:1234' }
 
-function * getStateSet () {
-  var ss = {}
-  yield * this.ss.iterate(this, null, null, function * (n) {
-    var user = n.id[0]
-    var clock = n.clock
+function getStateSet (y) {
+  let ss = {}
+  for (let [user, clock] of y.ss.state) {
     ss[user] = clock
-  })
+  }
   return ss
 }
 
-function * getDeleteSet () {
+function getDeleteSet (y) {
   var ds = {}
-  yield * this.ds.iterate(this, null, null, function * (n) {
-    var user = n.id[0]
-    var counter = n.id[1]
+  y.ds.iterate(null, null, function (n) {
+    var user = n._id.user
+    var counter = n._id.clock
     var len = n.len
     var gc = n.gc
     var dv = ds[user]
@@ -44,16 +39,17 @@ function * getDeleteSet () {
   return ds
 }
 
-export async function garbageCollectUsers (t, users) {
-  await flushAll(t, users)
-  await Promise.all(users.map(u => u.db.emptyGarbageCollector()))
-}
-
-export function attrsToObject (attrs) {
+export function attrsObject (dom) {
+  let keys = []
+  let yxml = dom._yxml
+  for (let i = 0; i < dom.attributes.length; i++) {
+    keys.push(dom.attributes[i].name)
+  }
+  keys = yxml._domFilter(dom, keys)
   let obj = {}
-  for (var i = 0; i < attrs.length; i++) {
-    let attr = attrs[i]
-    obj[attr.name] = attr.value
+  for (let i = 0; i < keys.length; i++) {
+    let key = keys[i]
+    obj[key] = dom.getAttribute(key)
   }
   return obj
 }
@@ -62,8 +58,10 @@ export function domToJson (dom) {
   if (dom.nodeType === document.TEXT_NODE) {
     return dom.textContent
   } else if (dom.nodeType === document.ELEMENT_NODE) {
-    let attributes = attrsToObject(dom.attributes)
-    let children = Array.from(dom.childNodes.values()).map(domToJson)
+    let attributes = attrsObject(dom)
+    let children = Array.from(dom.childNodes.values())
+      .filter(d => d._yxml !== false)
+      .map(domToJson)
     return {
       name: dom.nodeName,
       children: children,
@@ -89,116 +87,86 @@ export async function compareUsers (t, users) {
   await flushAll(t, users)
   await wait()
   await flushAll(t, users)
+  await wait()
+  await flushAll(t, users)
+  await wait()
+  await flushAll(t, users)
 
-  var userArrayValues = users.map(u => u.share.array._content.map(c => c.val || JSON.stringify(c.type)))
-  function valueToComparable (v) {
-    if (v != null && v._model != null) {
-      return v._model
-    } else {
-      return v || null
-    }
-  }
-  var userMapOneValues = users.map(u => u.share.map.get('one')).map(valueToComparable)
-  var userMapTwoValues = users.map(u => u.share.map.get('two')).map(valueToComparable)
-  var userXmlValues = users.map(u => u.share.xml.getDom()).map(domToJson)
+  var userArrayValues = users.map(u => u.get('array', Y.Array).toJSON().map(val => JSON.stringify(val)))
+  var userMapValues = users.map(u => u.get('map', Y.Map).toJSON())
+  var userXmlValues = users.map(u => u.get('xml', Y.Xml).toString())
 
-  await users[0].db.garbageCollect()
-  await users[0].db.garbageCollect()
-
-  // disconnect all except user 0
-  await Promise.all(users.slice(1).map(async u =>
-    u.disconnect()
-  ))
-  if (users[0].connector.testRoom == null) {
-    await wait(100)
-  }
-  // reconnect all
-  await Promise.all(users.map(u => u.reconnect()))
-  if (users[0].connector.testRoom == null) {
-    await wait(100)
-  }
-  await users[0].connector.testRoom.flushAll(users)
-  await Promise.all(users.map(u =>
-    new Promise(function (resolve) {
-      u.connector.whenSynced(resolve)
-    })
-  ))
-  let filterDeletedOps = users.every(u => u.db.gc === false)
-  var data = await Promise.all(users.map(async (u) => {
+  var data = users.map(u => {
+    defragmentItemContent(u)
     var data = {}
-    u.db.requestTransaction(function * () {
-      let ops = []
-      yield * this.os.iterate(this, null, null, function * (op) {
-        ops.push(Y.Struct[op.struct].encode(op))
-      })
-
-      data.os = {}
-      for (let i = 0; i < ops.length; i++) {
-        let op = ops[i]
-        op = Y.Struct[op.struct].encode(op)
-        delete op.origin
-        /*
-          If gc = false, it is necessary to filter deleted ops
-          as they might have been split up differently..
-         */
-        if (filterDeletedOps) {
-          let opIsDeleted = yield * this.isDeleted(op.id)
-          if (!opIsDeleted) {
-            data.os[JSON.stringify(op.id)] = op
-          }
-        } else {
-          data.os[JSON.stringify(op.id)] = op
-        }
+    let ops = []
+    u.os.iterate(null, null, function (op) {
+      const json = {
+        id: op._id,
+        left: op._left === null ? null : op._left._lastId,
+        right: op._right === null ? null : op._right._id,
+        length: op._length,
+        deleted: op._deleted,
+        parent: op._parent._id
       }
-      data.ds = yield * getDeleteSet.apply(this)
-      data.ss = yield * getStateSet.apply(this)
+      if (op instanceof ItemJSON || op instanceof ItemString) {
+        json.content = op._content
+      }
+      ops.push(json)
     })
-    await u.db.whenTransactionsFinished()
+    data.os = ops
+    data.ds = getDeleteSet(u)
+    data.ss = getStateSet(u)
     return data
-  }))
+  })
   for (var i = 0; i < data.length - 1; i++) {
     await t.asyncGroup(async () => {
       t.compare(userArrayValues[i], userArrayValues[i + 1], 'array types')
-      t.compare(userMapOneValues[i], userMapOneValues[i + 1], 'map types (propery "one")')
-      t.compare(userMapTwoValues[i], userMapTwoValues[i + 1], 'map types (propery "two")')
+      t.compare(userMapValues[i], userMapValues[i + 1], 'map types')
       t.compare(userXmlValues[i], userXmlValues[i + 1], 'xml types')
       t.compare(data[i].os, data[i + 1].os, 'os')
       t.compare(data[i].ds, data[i + 1].ds, 'ds')
       t.compare(data[i].ss, data[i + 1].ss, 'ss')
     }, `Compare user${i} with user${i + 1}`)
   }
-  await Promise.all(users.map(async (u) => {
-    await u.close()
-  }))
+  users.map(u => u.destroy())
 }
 
 export async function initArrays (t, opts) {
   var result = {
     users: []
   }
-  var share = Object.assign({ flushHelper: 'Map', array: 'Array', map: 'Map', xml: 'XmlElement("div")' }, opts.share)
   var chance = opts.chance || new Chance(t.getSeed() * 1000000000)
   var conn = Object.assign({ room: 'debugging_' + t.name, generateUserId: false, testContext: t, chance }, connector)
   for (let i = 0; i < opts.users; i++) {
-    let dbOpts
     let connOpts
     if (i === 0) {
-      // Only one instance can gc!
-      dbOpts = Object.assign({ gc: false }, database)
       connOpts = Object.assign({ role: 'master' }, conn)
     } else {
-      dbOpts = Object.assign({ gc: false }, database)
       connOpts = Object.assign({ role: 'slave' }, conn)
     }
-    let y = await Y({
-      connector: connOpts,
-      db: dbOpts,
-      share: share
+    let y = new Y({
+      _userID: i, // evil hackery, don't try this at home
+      connector: connOpts
     })
     result.users.push(y)
-    for (let name in share) {
-      result[name + i] = y.share[name]
-    }
+    result['array' + i] = y.define('array', Y.Array)
+    result['map' + i] = y.define('map', Y.Map)
+    result['xml' + i] = y.define('xml', Y.XmlElement)
+    y.get('xml', Y.Xml).setDomFilter(function (d, attrs) {
+      if (d.nodeName === 'HIDDEN') {
+        return null
+      } else {
+        return attrs.filter(a => a !== 'hidden')
+      }
+    })
+    y.on('afterTransaction', function () {
+      for (let missing of y._missingStructs.values()) {
+        if (Array.from(missing.values()).length > 0) {
+          console.error(new Error('Test check in "afterTransaction": missing should be empty!'))
+        }
+      }
+    })
   }
   result.array0.delete(0, result.array0.length)
   if (result.users[0].connector.testRoom != null) {
@@ -219,15 +187,12 @@ export async function flushAll (t, users) {
   if (users.length === 0) {
     return
   }
-  await wait(0)
+  await wait(10)
   if (users[0].connector.testRoom != null) {
     // use flushAll method specified in Test Connector
     await users[0].connector.testRoom.flushAll(users)
   } else {
-    // flush for any connector
-    await Promise.all(users.map(u => { return u.db.whenTransactionsFinished() }))
-
-    var flushCounter = users[0].share.flushHelper.get('0') || 0
+    var flushCounter = users[0].get('flushHelper', Y.Map).get('0') || 0
     flushCounter++
     await Promise.all(users.map(async (u, i) => {
       // wait for all users to set the flush counter to the same value
@@ -235,7 +200,7 @@ export async function flushAll (t, users) {
         function observer () {
           var allUsersReceivedUpdate = true
           for (var i = 0; i < users.length; i++) {
-            if (u.share.flushHelper.get(i + '') !== flushCounter) {
+            if (u.get('flushHelper', Y.Map).get(i + '') !== flushCounter) {
               allUsersReceivedUpdate = false
               break
             }
@@ -244,8 +209,8 @@ export async function flushAll (t, users) {
             resolve()
           }
         }
-        u.share.flushHelper.observe(observer)
-        u.share.flushHelper.set(i + '', flushCounter)
+        u.get('flushHelper', Y.Map).observe(observer)
+        u.get('flushHelper').set(i + '', flushCounter)
       })
     }))
   }
@@ -295,7 +260,7 @@ export async function applyRandomTests (t, mods, iterations) {
       // TODO: We do not gc all users as this does not work yet
       // await garbageCollectUsers(t, users)
       await flushAll(t, users)
-      await users[0].db.emptyGarbageCollector()
+      // await users[0].db.emptyGarbageCollector()
       await flushAll(t, users)
     } else if (chance.bool({likelihood: 10})) {
       // 20%*!prev chance to flush some operations

tests-lib/test-connector.js

@@ -1,6 +1,6 @@
 /* global Y */
-import { wait } from './helper.js'
-import { formatYjsMessage } from '../src/MessageHandler.js'
+import { wait } from './helper'
+import { messageToString } from '../src/MessageHandler/messageToString'
 
 var rooms = {}
 
@@ -8,24 +8,28 @@ export class TestRoom {
   constructor (roomname) {
     this.room = roomname
     this.users = new Map()
-    this.nextUserId = 0
   }
   join (connector) {
-    if (connector.userId == null) {
-      connector.setUserId(this.nextUserId++)
-    }
-    this.users.forEach((user, uid) => {
-      if (user.role === 'master' || connector.role === 'master') {
-        this.users.get(uid).userJoined(connector.userId, connector.role)
-        connector.userJoined(uid, this.users.get(uid).role)
+    const userID = connector.y.userID
+    this.users.set(userID, connector)
+    for (let [uid, user] of this.users) {
+      if (uid !== userID && (user.role === 'master' || connector.role === 'master')) {
+        // The order is important because there is no timeout in send/receiveMessage
+        // (the user that receives a sync step must already now about the sender)
+        if (user.role === 'master') {
+          connector.userJoined(uid, user.role)
+          user.userJoined(userID, connector.role)
+        } else if (connector.role === 'master') {
+          user.userJoined(userID, connector.role)
+          connector.userJoined(uid, user.role)
+        }
       }
-    })
-    this.users.set(connector.userId, connector)
+    }
   }
   leave (connector) {
-    this.users.delete(connector.userId)
+    this.users.delete(connector.y.userID)
     this.users.forEach(user => {
-      user.userLeft(connector.userId)
+      user.userLeft(connector.y.userID)
     })
   }
   send (sender, receiver, m) {
@@ -41,13 +45,13 @@ export class TestRoom {
   }
   async flushAll (users) {
     let flushing = true
-    let allUserIds = Array.from(this.users.keys())
+    let allUsers = Array.from(this.users.values())
     if (users == null) {
-      users = allUserIds.map(id => this.users.get(id).y)
+      users = allUsers.map(user => user.y)
     }
     while (flushing) {
       await wait(10)
-      let res = await Promise.all(allUserIds.map(id => this.users.get(id)._flushAll(users)))
+      let res = await Promise.all(allUsers.map(user => user._flushAll(users)))
       flushing = res.some(status => status === 'flushing')
     }
   }
@@ -82,25 +86,28 @@ export default function extendTestConnector (Y) {
       return super.disconnect()
     }
     logBufferParsed () {
-      console.log(' === Logging buffer of user ' + this.userId + ' === ')
+      console.log(' === Logging buffer of user ' + this.y.userID + ' === ')
       for (let [user, conn] of this.connections) {
         console.log(` ${user}:`)
         for (let i = 0; i < conn.buffer.length; i++) {
-          console.log(formatYjsMessage(conn.buffer[i]))
+          console.log(messageToString(conn.buffer[i]))
         }
       }
     }
     reconnect () {
       this.testRoom.join(this)
-      return super.reconnect()
+      super.reconnect()
+      return new Promise(resolve => {
+        this.whenSynced(resolve)
+      })
     }
     send (uid, message) {
       super.send(uid, message)
-      this.testRoom.send(this.userId, uid, message)
+      this.testRoom.send(this.y.userID, uid, message)
     }
     broadcast (message) {
       super.broadcast(message)
-      this.testRoom.broadcast(this.userId, message)
+      this.testRoom.broadcast(this.y.userID, message)
     }
     async whenSynced (f) {
       var synced = false
@@ -119,7 +126,7 @@ export default function extendTestConnector (Y) {
       })
     }
     receiveMessage (sender, m) {
-      if (this.userId !== sender && this.connections.has(sender)) {
+      if (this.y.userID !== sender && this.connections.has(sender)) {
         var buffer = this.connections.get(sender).buffer
         if (buffer == null) {
           buffer = this.connections.get(sender).buffer = []
@@ -135,30 +142,27 @@ export default function extendTestConnector (Y) {
       }
     }
     async _flushAll (flushUsers) {
-      if (flushUsers.some(u => u.connector.userId === this.userId)) {
+      if (flushUsers.some(u => u.connector.y.userID === this.y.userID)) {
         // this one needs to sync with every other user
         flushUsers = Array.from(this.connections.keys()).map(uid => this.testRoom.users.get(uid).y)
       }
-      var finished = []
       for (let i = 0; i < flushUsers.length; i++) {
-        let userId = flushUsers[i].connector.userId
-        if (userId !== this.userId && this.connections.has(userId)) {
-          let buffer = this.connections.get(userId).buffer
+        let userID = flushUsers[i].connector.y.userID
+        if (userID !== this.y.userID && this.connections.has(userID)) {
+          let buffer = this.connections.get(userID).buffer
           if (buffer != null) {
             var messages = buffer.splice(0)
             for (let j = 0; j < messages.length; j++) {
-              let p = super.receiveMessage(userId, messages[j])
-              finished.push(p)
+              super.receiveMessage(userID, messages[j])
             }
           }
         }
       }
-      await Promise.all(finished)
-      await this.y.db.whenTransactionsFinished()
-      return finished.length > 0 ? 'flushing' : 'done'
+      return 'done'
     }
   }
-  Y.extend('test', TestConnector)
+  // TODO: this should be moved to a separate module (dont work on Y)
+  Y.test = TestConnector
 }
 
 if (typeof Y !== 'undefined') {