integrate ydb client and adapt some demos

YdbClient/TODO.md

@@ -0,0 +1,56 @@
+Implement default dom filter..
+
+But requires more explicit filtering of src attributes
+
+e.g. src="java\nscript:alert(0)"
+
+function domFilter (nodeName, attributes) {
+        // Filter all attributes that start with on*. E.g. onclick does execute code
+        // If key is 'href' or 'src', filter everything but 'http*', 'blob*', or 'data:image*' urls
+        attributes.forEach(function (value, key) {
+            key = key.toLowerCase();
+            value = value.toLowerCase();
+            if (key != null && (
+                // filter all attributes starting with 'on'
+                key.substr(0, 2) === 'on' ||
+                // if key is 'href' or 'src', filter everything but http, blob, or data:image
+                (
+                    (key === 'href' || key === 'src') &&
+                    value.substr(0, 4) !== 'http' &&
+                    value.substr(0, 4) !== 'blob' &&
+                    value.substr(0, 10) !== 'data:image'
+                )
+            )) {
+                attributes.delete(key);
+            }
+        });
+        switch (nodeName) {
+        case 'SCRIPT':
+            return null;
+        case 'EN-ADORNMENTS':
+            // TODO: Remove EN-ADORNMENTS check when merged into master branch!
+            return null;
+        case 'EN-TABLE':
+            attributes.delete('class');
+            return attributes;
+        case 'EN-COMMENT':
+            attributes.delete('style');
+            attributes.delete('class');
+            return attributes;
+        case 'SPAN':
+            return (attributes.get('id') || '').substr(0, 5) === 'goog_' ? null : attributes;
+        case 'TD':
+            attributes.delete('class');
+            return attributes;
+        case 'EMBED':
+            attributes.delete('src');
+            attributes.delete('style');
+            attributes.delete('data-reference');
+            return attributes;
+        case 'FORM':
+            attributes.delete('action');
+            return attributes;
+        default:
+            return (nodeName || '').substr(0, 3) === 'UI-' ? null : attributes;
+        }
+    }

YdbClient/YdbClient.js

@@ -7,6 +7,9 @@ import * as encoding from './encoding.js'
 import * as logging from './logging.js'
 import * as idb from './idb.js'
 import Y from '../src/Y.js'
+import BinaryDecoder from '../src/Util/Binary/Decoder.js'
+import { integrateRemoteStruct } from '../src/MessageHandler/integrateRemoteStructs.js'
+import { createMutualExclude } from '../src/Util/mutualExclude.js'
 
 export class YdbClient {
   constructor (url, db) {
@@ -24,9 +27,20 @@ export class YdbClient {
    */
   getY (roomname) {
     const y = new Y(roomname)
-    y.on('afterTransaction', function () {
-      debugger
-    })
+    const mutex = createMutualExclude()
+    y.on('afterTransaction', (y, transaction) => mutex(() => {
+      if (transaction.encodedStructsLen > 0) {
+        update(this, roomname, transaction.encodedStructs.createBuffer())
+      }
+    }))
+    subscribe(this, roomname, update => mutex(() => {
+      y.transact(() => {
+        const decoder = new BinaryDecoder(update)
+        while (decoder.hasContent()) {
+          integrateRemoteStruct(y, decoder)
+        }
+      }, true)
+    }))
     return y
   }
 }
@@ -111,7 +125,7 @@ export const update = (ydb, room, update) => {
   const t = idbactions.createTransaction(ydb.db)
   logging.log(`Write Unconfirmed Update. room "${room}", ${JSON.stringify(update)}`)
   return idbactions.writeClientUnconfirmed(t, room, update).then(clientConf => {
-    logging.log(`Send Unconfirmed Update. connected ${ydb.connected} room "${room}", clientConf ${clientConf}, ${logging.arrayBufferToString(update)}`)
+    logging.log(`Send Unconfirmed Update. connected ${ydb.connected} room "${room}", clientConf ${clientConf}`)
     send(ydb, message.createUpdate(room, update, clientConf))
   })
 }
@@ -133,3 +147,19 @@ export const subscribe = (ydb, room, f) => {
     }
   })
 }
+
+export const subscribeRooms = (ydb, rooms) => {
+  const t = idbactions.createTransaction(ydb.db)
+  const subs = []
+  return globals.pall(rooms.map(room => idbactions.getRoomMeta(t, room).then(meta => {
+    if (meta === undefined) {
+      subs.push(room)
+      return idbactions.writeUnconfirmedSubscription(t, room)
+    }
+  }))).then(() => {
+    // write all sub messages when all unconfirmed subs are writted to idb
+    if (subs.length > 0) {
+      send(ydb, message.createSub(rooms.map(room => ({room, offset: 0}))))
+    }
+  })
+}

YdbClient/YdbClient.test.js

@@ -1,7 +1,7 @@
 /* eslint-env browser */
 
 import * as test from './test.js'
-import * as ydbClient from './ydb-client.js'
+import * as ydbClient from './YdbClient.js'
 import * as globals from './globals.js'
 import * as idbactions from './idbactions.js'
 import * as logging from './logging.js'

YdbClient/idbactions.js

@@ -186,7 +186,7 @@ export const writeHostUnconfirmedByClient = (t, clientConf, offset) => idb.get(g
  * @param {ArrayBuffer} update
  * @return {Promise}
  */
-export const writeHostUnconfirmed = (t, room, offset, update) => idb.add(getStoreHU(t), update, encodeHUKey(room, offset))
+export const writeHostUnconfirmed = (t, room, offset, update) => idb.put(getStoreHU(t), update, encodeHUKey(room, offset))
 
 /**
  * The host confirms that it persisted updates up until (including) offset. updates may be moved from HU to Co.
@@ -199,9 +199,11 @@ export const writeConfirmedByHost = (t, room, offset) => {
   const co = getStoreCo(t)
   return globals.pall([idb.get(co, getCoDataKey(room)), idb.get(co, getCoMetaKey(room))]).then(async arr => {
     const data = arr[0]
-    const meta = arr[1]
-    const metaSessionId = decodeMetaValue(meta).roomsid
+    const meta = decodeMetaValue(arr[1])
     const dataEncoder = encoding.createEncoder()
+    if (meta.offset >= offset) {
+      return // nothing to do
+    }
     encoding.writeArrayBuffer(dataEncoder, data)
     const hu = getStoreHU(t)
     const huKeyRange = idb.createIDBKeyRangeBound(encodeHUKey(room, 0), encodeHUKey(room, offset), false, false)
@@ -210,9 +212,9 @@ export const writeConfirmedByHost = (t, room, offset) => {
       if (key.room === room && key.offset <= offset) {
         encoding.writeArrayBuffer(dataEncoder, value)
       }
-    }).then(() =>
-      globals.pall([idb.put(co, encodeMetaValue(metaSessionId, offset), getCoMetaKey(room)), idb.put(co, encoding.toBuffer(dataEncoder), getCoDataKey(room)), idb.del(hu, huKeyRange)])
-    )
+    }).then(() => {
+      globals.pall([idb.put(co, encodeMetaValue(meta.roomsid, offset), getCoMetaKey(room)), idb.put(co, encoding.toBuffer(dataEncoder), getCoDataKey(room)), idb.del(hu, huKeyRange)])
+    })
   })
 }
 
@@ -290,10 +292,23 @@ const encodeMetaValue = (roomsid, offset) => {
  * @param {number} offset
  * @return {Promise<void>}
  */
-export const confirmSubscription = (t, room, roomsessionid, offset) => globals.pall([
-  idb.put(getStoreCo(t), encodeMetaValue(roomsessionid, offset), getCoMetaKey(room)),
-  idb.put(getStoreCo(t), globals.createArrayBufferFromArray([]), getCoDataKey(room))
-]).then(() => idb.del(getStoreUS(t), room))
+export const confirmSubscription = (t, room, roomsessionid, offset) => idb.get(getStoreCo(t), getCoMetaKey(room)).then(metaval => {
+  if (metaval === undefined) {
+    return globals.pall([
+      idb.put(getStoreCo(t), encodeMetaValue(roomsessionid, offset), getCoMetaKey(room)),
+      idb.put(getStoreCo(t), globals.createArrayBufferFromArray([]), getCoDataKey(room))
+    ]).then(() => idb.del(getStoreUS(t), room))
+  }
+  const meta = decodeMetaValue(metaval)
+  if (meta.roomsid !== roomsessionid) {
+    // TODO: upload all unconfirmed updates
+    // or do a Yjs sync with server
+  } else if (meta.roomsid < offset) {
+    return writeConfirmedByHost(t, room, offset)
+  } else {
+    // nothing needs to happen
+  }
+})
 
 export const writeUnconfirmedSubscription = (t, room) => idb.put(getStoreUS(t), true, room)
 

YdbClient/index.js

@@ -1,4 +1,4 @@
-import * as ydbclient from './ydb-client.js'
+import * as ydbclient from './YdbClient.js'
 
 /**
  * @param {string} url

YdbClient/message.js

@@ -25,7 +25,7 @@ export const readMessage = (ydb, message) => {
         const offset = decoding.readVarUint(decoder)
         const room = decoding.readVarString(decoder)
         const update = decoding.readPayload(decoder)
-        logging.log(`Received Update. room "${room}", offset ${offset}, ${logging.arrayBufferToString(update)}`)
+        logging.log(`Received Update. room "${room}", offset ${offset}`)
         idbactions.writeHostUnconfirmed(t, room, offset, update)
         bc.publish(room, update)
         break
@@ -36,7 +36,7 @@ export const readMessage = (ydb, message) => {
           const room = decoding.readVarString(decoder)
           const offset = decoding.readVarUint(decoder)
           const roomsid = decoding.readVarUint(decoder) // TODO: SID
-          logging.log(`Received Sub Conf. room "${room}", offset ${offset}, roomsid ${roomsid}`)
+          // logging.log(`Received Sub Conf. room "${room}", offset ${offset}, roomsid ${roomsid}`)
           idbactions.confirmSubscription(t, room, roomsid, offset)
         }
         break

examples/notes/index.html

@@ -2,7 +2,16 @@
 <html>
 </head>
   <script src="./index.js" type="module"></script>
+  <link rel="stylesheet" type="text/css" href="style.css">
 </head>
-<body contenteditable="true">
+<body>
+    <div class="sidebar">
+      <h3 id="createNoteButton">+ Create Note</h3>
+      <div class="notelist"></div>
+    </div>
+    <div class="main">
+      <h1 id="headline"></h1>
+      <div id="editor" contenteditable="true"></div>
+    </div>
 </body>
-</html>
+</html>

examples/notes/index.js

@@ -0,0 +1,88 @@
+/* eslint-env browser */
+
+import { createYdbClient } from '../../YdbClient/index.js'
+import Y from '../../src/Y.dist.js'
+import * as ydb from '../../YdbClient/YdbClient.js'
+import DomBinding from '../../src/Bindings/DomBinding/DomBinding.js'
+
+const uuidv4 = () => 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {
+  const r = Math.random() * 16 | 0
+  return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16)
+})
+
+createYdbClient('ws://localhost:8899/ws').then(ydbclient => {
+  const y = ydbclient.getY('notelist')
+  let ynotelist = y.define('notelist', Y.Array)
+  const domNoteList = document.querySelector('.notelist')
+
+  // utils
+  const addEventListener = (element, eventname, f) => element.addEventListener(eventname, f)
+
+  // create note button
+  const createNoteButton = event => {
+    ynotelist.insert(0, [{
+      guid: uuidv4(),
+      title: 'Note #' + ynotelist.length
+    }])
+  }
+  addEventListener(document.querySelector('#createNoteButton'), 'click', createNoteButton)
+  window.createNote = createNoteButton
+  window.createNotes = n => {
+    y.transact(() => {
+      for (let i = 0; i < n; i++) {
+        createNoteButton()
+      }
+    })
+  }
+
+  // clear note list function
+  window.clearNotes = () => ynotelist.delete(0, ynotelist.length)
+
+  // update editor and editor title
+  let domBinding = null
+  const updateEditor = () => {
+    domNoteList.querySelectorAll('a').forEach(a => a.classList.remove('selected'))
+    const domNote = document.querySelector('.notelist').querySelector(`[href="${location.hash}"]`)
+    if (domNote !== null) {
+      domNote.classList.add('selected')
+      const note = ynotelist.toArray().find(note => note.guid === location.hash.slice(1))
+      if (note !== undefined) {
+        const ydoc = ydbclient.getY(note.guid)
+        const ycontent = ydoc.define('content', Y.XmlFragment)
+        if (domBinding !== null) {
+          domBinding.destroy()
+        }
+        domBinding = new DomBinding(ycontent, document.querySelector('#editor'))
+        document.querySelector('#headline').innerText = note.title
+        document.querySelector('#editor').focus()
+      }
+    }
+  }
+
+  // listen to url-hash changes
+  addEventListener(window, 'hashchange', updateEditor)
+  updateEditor()
+
+  // render note list
+  const renderNoteList = addedElements => {
+    const fragment = document.createDocumentFragment()
+    addedElements.forEach(note => {
+      const a = document.createElement('a')
+      a.setAttribute('href', '#' + note.guid)
+      a.innerText = note.title
+      fragment.insertBefore(a, null)
+    })
+    domNoteList.insertBefore(fragment, domNoteList.firstChild)
+  }
+  renderNoteList(ynotelist.toArray())
+  ydb.subscribeRooms(ydbclient, ynotelist.map(note => note.guid))
+  ynotelist.observe(event => {
+    const addedNotes = []
+    event.addedElements.forEach(itemJson => itemJson._content.forEach(json => addedNotes.push(json)))
+    // const arr = ynotelist.toArray().filter(note => event.addedElements.has(note))
+    renderNoteList(addedNotes.reverse())
+    if (domBinding === null) {
+      updateEditor()
+    }
+  })
+})

examples/notes/index.mjs

@@ -1,48 +0,0 @@
-
-import IndexedDBPersistence from '../../src/Persistences/IndexeddbPersistence.js'
-import YWebsocketsConnector from '../../src/Connectors/WebsocketsConnector/WebsocketsConnector.js'
-import Y from '../../src/Y.js'
-import YXmlFragment from '../../src/Types/YXml/YXmlFragment.js'
-
-const yCollection = new YCollection(new YWebsocketsConnector(), new IndexedDBPersistence())
-
-const y = yCollection.getDocument('my-notes')
-
-
-
-
-
-persistence.addConnector(persistence)
-
-const y = new Y()
-await persistence.persistY(y)
-
-
-connector.connectY('html-editor', y)
-persistence.connectY('html-editor', y)
-
-
-
-
-window.connector = connector
-
-window.onload = function () {
-  window.domBinding = new DomBinding(window.yXmlType, document.body, { scrollingElement: document.scrollingElement })
-}
-
-window.y = y
-window.yXmlType = y.define('xml', YXmlFragment)
-window.undoManager = new UndoManager(window.yXmlType, {
-  captureTimeout: 500
-})
-
-document.onkeydown = function interceptUndoRedo (e) {
-  if (e.keyCode === 90 && (e.metaKey || e.ctrlKey)) {
-    if (!e.shiftKey) {
-      window.undoManager.undo()
-    } else {
-      window.undoManager.redo()
-    }
-    e.preventDefault()
-  }
-}

examples/notes/style.css

@@ -0,0 +1,57 @@
+.sidebar {
+    height: 100%; /* Full-height: remove this if you want "auto" height */
+    width: 180px; /* Set the width of the sidebar */
+    position: fixed; /* Fixed Sidebar (stay in place on scroll) */
+    z-index: 1; /* Stay on top */
+    top: 0; /* Stay at the top */
+    left: 0;
+    background-color: #111; /* Black */
+    overflow-x: hidden; /* Disable horizontal scroll */
+    padding-top: 20px;
+    color: #50abff;
+}
+
+#createNoteButton {
+    padding-left: .5em;
+    padding-top: .5em;
+    padding-bottom: .7em;
+    margin: 0;
+    cursor: pointer;
+}
+
+.sidebar a {
+    padding: 6px 8px 6px 16px;
+    text-decoration: none;
+    font-size: 13px;
+    color: #818181;
+    display: block;
+}
+
+.sidebar a.selected {
+    border-style: outset;
+}
+
+/* When you mouse over the navigation links, change their color */
+.sidebar a:hover {
+    color: #f1f1f1;
+}
+
+/* Style page content */
+.main {
+    margin-left: 180px; /* Same as the width of the sidebar */
+    padding: 0px 10px;
+}
+
+/* On smaller screens, where height is less than 450px, change the style of the sidebar (less padding and a smaller font size) */
+@media screen and (max-height: 450px) {
+    .sidebar {padding-top: 15px;}
+    .sidebar a {font-size: 18px;}
+}
+
+#editor {
+    min-height: 400px;
+}
+
+[contenteditable]:focus {
+    outline: 0px solid transparent;
+}

examples/textarea/index.js

@@ -1,4 +1,4 @@
-import { createYdbClient } from '../../ydb/index.js'
+import { createYdbClient } from '../../YdbClient/index.js'
 import Y from '../../src/Y.dist.js'
 
 createYdbClient('ws://localhost:8899/ws').then(ydbclient => {

src/Bindings/DomBinding/DomBinding.js

@@ -69,9 +69,10 @@ export default class DomBinding extends Binding {
       subtree: true
     })
     this._currentSel = null
-    document.addEventListener('selectionchange', () => {
+    this._selectionchange = () => {
       this._currentSel = getCurrentRelativeSelection(this)
-    })
+    }
+    document.addEventListener('selectionchange', this._selectionchange)
     const y = type._y
     this.y = y
     // Force flush dom changes before Type changes are applied (they might
@@ -193,6 +194,7 @@ export default class DomBinding extends Binding {
     y.off('beforeTransaction', this._beforeTransactionHandler)
     y.off('beforeObserverCalls', this._beforeObserverCallsHandler)
     y.off('afterTransaction', this._afterTransactionHandler)
+    document.removeEventListener('selectionchange', this._selectionchange)
     super.destroy()
   }
 }

src/MessageHandler/integrateRemoteStructs.js

@@ -107,3 +107,35 @@ export function integrateRemoteStructs (y, decoder) {
     }
   }
 }
+
+// TODO: use this above / refactor
+export function integrateRemoteStruct (y, decoder) {
+  let reference = decoder.readVarUint()
+  let Constr = getStruct(reference)
+  let struct = new Constr()
+  let decoderPos = decoder.pos
+  let missing = struct._fromBinary(y, decoder)
+  if (missing.length === 0) {
+    while (struct != null) {
+      _integrateRemoteStructHelper(y, struct)
+      struct = y._readyToIntegrate.shift()
+    }
+  } else {
+    let _decoder = new BinaryDecoder(decoder.uint8arr)
+    _decoder.pos = decoderPos
+    let missingEntry = new MissingEntry(_decoder, missing, struct)
+    let missingStructs = y._missingStructs
+    for (let i = missing.length - 1; i >= 0; i--) {
+      let m = missing[i]
+      if (!missingStructs.has(m.user)) {
+        missingStructs.set(m.user, new Map())
+      }
+      let msu = missingStructs.get(m.user)
+      if (!msu.has(m.clock)) {
+        msu.set(m.clock, [])
+      }
+      let mArray = msu = msu.get(m.clock)
+      mArray.push(missingEntry)
+    }
+  }
+}

src/Transaction.js

@@ -60,18 +60,13 @@ export default class Transaction {
      */
     this.changedParentTypes = new Map()
     this.encodedStructsLen = 0
-    this._encodedStructs = new BinaryEncoder()
-    this._encodedStructs.writeUint32(0)
-  }
-  get encodedStructs () {
-    this._encodedStructs.setUint32(0, this.encodedStructsLen)
-    return this._encodedStructs
+    this.encodedStructs = new BinaryEncoder()
   }
 }
 
 export function writeStructToTransaction (transaction, struct) {
   transaction.encodedStructsLen++
-  struct._toBinary(transaction._encodedStructs)
+  struct._toBinary(transaction.encodedStructs)
 }
 
 /**

src/Y.js

@@ -36,21 +36,8 @@ export default class Y extends NamedEventHandler {
      * @type {String}
      */
     this.room = room
-<<<<<<< HEAD:src/Y.js
-    if (opts != null && opts.connector != null) {
-      opts.connector.room = room
-    }
-    this._contentReady = false
-    this._opts = opts
-    if (opts == null || typeof opts.userID !== 'number') {
-      this.userID = generateRandomUint32()
-    } else {
-      this.userID = opts.userID
-    }
-=======
     this._contentReady = false
     this.userID = generateRandomUint32()
->>>>>>> experimental-connectors:src/Y.mjs
     // TODO: This should be a Map so we can use encodables as keys
     this.share = {}
     this.ds = new DeleteStore(this)