Merge pull request #239 from yjs/subdocs

implemented first subdocuments draft #234

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "yjs",
-  "version": "13.3.2",
+  "version": "13.4.0-0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -31,9 +31,9 @@
       }
     },
     "@babel/parser": {
-      "version": "7.10.2",
+      "version": "7.11.5",
-      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.10.2.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.11.5.tgz",
-      "integrity": "sha512-PApSXlNMJyB4JiGVhCOlzKIif+TKFTvu0aQAhnTvfP/z3vVSN6ZypH5bfUNwFXXjRQtUEBNFd2PtmCmG2Py3qQ==",
+      "integrity": "sha512-X9rD8qqm695vgmeaQ4fvz/o3+Wk4ZzQvSHkDBgpYKxpD4qTAUm88ZKtHkVqIOsYFFbIQ6wQYhC6q7pjqVK0E0Q==",
       "dev": true
     },
     "@rollup/plugin-commonjs": {
@@ -1459,9 +1459,9 @@
       }
     },
     "jsdoc": {
-      "version": "3.6.4",
+      "version": "3.6.5",
-      "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.4.tgz",
+      "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.5.tgz",
-      "integrity": "sha512-3G9d37VHv7MFdheviDCjUfQoIjdv4TC5zTTf5G9VODLtOnVS6La1eoYBDlbWfsRT3/Xo+j2MIqki2EV12BZfwA==",
+      "integrity": "sha512-SbY+i9ONuxSK35cgVHaI8O9senTE4CDYAmGSDJ5l3+sfe62Ff4gy96osy6OW84t4K4A8iGnMrlRrsSItSNp3RQ==",
       "dev": true,
       "requires": {
         "@babel/parser": "^7.9.4",
@@ -1548,9 +1548,9 @@
       }
     },
     "lib0": {
-      "version": "0.2.32",
+      "version": "0.2.33",
-      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.32.tgz",
+      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.33.tgz",
-      "integrity": "sha512-cHHKhHTojtvFSsthTk+CKuD17jMHIxuZxYpTzXj9TeQLPNoGNDPl6ax+J6eFETVe3ZvPMh3V0nGfJgGo6QgSvA==",
+      "integrity": "sha512-Pnm8FzjUr+aTYkEu2A20c1EfVHla8GbVX+GXn6poxx0gcmEuCs+XszjLmtEbI9xYOoI/83xVi7VOIoyHgOO87w==",
       "requires": {
         "isomorphic.js": "^0.1.3"
       }
@@ -2786,9 +2786,9 @@
       "dev": true
     },
     "typescript": {
-      "version": "3.9.6",
+      "version": "3.9.7",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.6.tgz",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz",
-      "integrity": "sha512-Pspx3oKAPJtjNwE92YS05HQoY7z2SFyOpHo9MqJor3BXAGNaPUs83CuVp9VISFkSjyRfiTpmKuAYGJB7S7hOxw==",
+      "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==",
       "dev": true
     },
     "uc.micro": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yjs",
-  "version": "13.3.2",
+  "version": "13.4.0-0",
   "description": "Shared Editing Library",
   "main": "./dist/yjs.cjs",
   "module": "./dist/yjs.mjs",
@@ -61,20 +61,20 @@
   },
   "homepage": "https://yjs.dev",
   "dependencies": {
-    "lib0": "^0.2.32"
+    "lib0": "^0.2.33"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^11.1.0",
     "@rollup/plugin-node-resolve": "^7.1.3",
     "concurrently": "^3.6.1",
     "http-server": "^0.12.3",
-    "jsdoc": "^3.6.4",
+    "jsdoc": "^3.6.5",
     "markdownlint-cli": "^0.23.2",
     "rollup": "^1.32.1",
     "rollup-cli": "^1.0.9",
     "standard": "^14.3.4",
     "tui-jsdoc-template": "^1.2.2",
-    "typescript": "^3.9.6",
+    "typescript": "^3.9.7",
     "y-protocols": "^0.2.3"
   }
 }

src/internals.js

@@ -31,6 +31,7 @@ export * from './structs/AbstractStruct.js'
 export * from './structs/GC.js'
 export * from './structs/ContentBinary.js'
 export * from './structs/ContentDeleted.js'
+export * from './structs/ContentDoc.js'
 export * from './structs/ContentEmbed.js'
 export * from './structs/ContentFormat.js'
 export * from './structs/ContentJSON.js'

src/structs/ContentDoc.js

@@ -0,0 +1,135 @@
+
+import {
+  Doc, AbstractUpdateDecoder, AbstractUpdateEncoder, StructStore, Transaction, Item // eslint-disable-line
+} from '../internals.js'
+
+import * as error from 'lib0/error.js'
+
+/**
+ * @private
+ */
+export class ContentDoc {
+  /**
+   * @param {Doc} doc
+   */
+  constructor (doc) {
+    if (doc._item) {
+      console.error('This document was already integrated as a sub-document. You should create a second instance instead with the same guid.')
+    }
+    /**
+     * @type {Doc}
+     */
+    this.doc = doc
+    /**
+     * @type {any}
+     */
+    const opts = {}
+    this.opts = opts
+    if (!doc.gc) {
+      opts.gc = false
+    }
+    if (doc.autoLoad) {
+      opts.autoLoad = true
+    }
+    if (doc.meta !== null) {
+      opts.meta = doc.meta
+    }
+  }
+
+  /**
+   * @return {number}
+   */
+  getLength () {
+    return 1
+  }
+
+  /**
+   * @return {Array<any>}
+   */
+  getContent () {
+    return [this.doc]
+  }
+
+  /**
+   * @return {boolean}
+   */
+  isCountable () {
+    return true
+  }
+
+  /**
+   * @return {ContentDoc}
+   */
+  copy () {
+    return new ContentDoc(this.doc)
+  }
+
+  /**
+   * @param {number} offset
+   * @return {ContentDoc}
+   */
+  splice (offset) {
+    throw error.methodUnimplemented()
+  }
+
+  /**
+   * @param {ContentDoc} right
+   * @return {boolean}
+   */
+  mergeWith (right) {
+    return false
+  }
+
+  /**
+   * @param {Transaction} transaction
+   * @param {Item} item
+   */
+  integrate (transaction, item) {
+    // this needs to be reflected in doc.destroy as well
+    this.doc._item = item
+    transaction.subdocsAdded.add(this.doc)
+    if (this.doc.shouldLoad) {
+      transaction.subdocsLoaded.add(this.doc)
+    }
+  }
+
+  /**
+   * @param {Transaction} transaction
+   */
+  delete (transaction) {
+    if (transaction.subdocsAdded.has(this.doc)) {
+      transaction.subdocsAdded.delete(this.doc)
+    } else {
+      transaction.subdocsRemoved.add(this.doc)
+    }
+  }
+
+  /**
+   * @param {StructStore} store
+   */
+  gc (store) { }
+
+  /**
+   * @param {AbstractUpdateEncoder} encoder
+   * @param {number} offset
+   */
+  write (encoder, offset) {
+    encoder.writeString(this.doc.guid)
+    encoder.writeAny(this.opts)
+  }
+
+  /**
+   * @return {number}
+   */
+  getRef () {
+    return 9
+  }
+}
+
+/**
+ * @private
+ *
+ * @param {AbstractUpdateDecoder} decoder
+ * @return {ContentDoc}
+ */
+export const readContentDoc = decoder => new ContentDoc(new Doc({ guid: decoder.readString(), ...decoder.readAny() }))

src/structs/Item.js

@@ -17,6 +17,7 @@ import {
   readContentAny,
   readContentString,
   readContentEmbed,
+  readContentDoc,
   createID,
   readContentFormat,
   readContentType,
@@ -672,14 +673,15 @@ export const readItemContent = (decoder, info) => contentRefs[info & binary.BITS
  */
 export const contentRefs = [
   () => { throw error.unexpectedCase() }, // GC is not ItemContent
-  readContentDeleted,
+  readContentDeleted, // 1
-  readContentJSON,
+  readContentJSON, // 2
-  readContentBinary,
+  readContentBinary, // 3
-  readContentString,
+  readContentString, // 4
-  readContentEmbed,
+  readContentEmbed, // 5
-  readContentFormat,
+  readContentFormat, // 6
-  readContentType,
+  readContentType, // 7
-  readContentAny
+  readContentAny, // 8
+  readContentDoc // 9
 ]
 
 /**

src/types/AbstractType.js

@@ -11,7 +11,7 @@ import {
   ContentAny,
   ContentBinary,
   getItemCleanStart,
-  YText, YArray, AbstractUpdateEncoder, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
+  ContentDoc, YText, YArray, AbstractUpdateEncoder, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, // eslint-disable-line
 } from '../internals.js'
 
 import * as map from 'lib0/map.js'
@@ -611,6 +611,10 @@ export const typeListInsertGenericsAfter = (transaction, parent, referenceItem,
             left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentBinary(new Uint8Array(/** @type {Uint8Array} */ (c))))
             left.integrate(transaction, 0)
             break
+          case Doc:
+            left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentDoc(/** @type {Doc} */ (c)))
+            left.integrate(transaction, 0)
+            break
           default:
             if (c instanceof AbstractType) {
               left = new Item(createID(ownClientId, getState(store, ownClientId)), left, left && left.lastId, right, right && right.id, parent, null, new ContentType(c))
@@ -761,6 +765,9 @@ export const typeMapSet = (transaction, parent, key, value) => {
       case Uint8Array:
         content = new ContentBinary(/** @type {Uint8Array} */ (value))
         break
+      case Doc:
+        content = new ContentDoc(/** @type {Doc} */ (value))
+        break
       default:
         if (value instanceof AbstractType) {
           content = new ContentType(value)

src/utils/Doc.js

@@ -10,30 +10,39 @@ import {
   YMap,
   YXmlFragment,
   transact,
-  Item, Transaction, YEvent // eslint-disable-line
+  ContentDoc, Item, Transaction, YEvent // eslint-disable-line
 } from '../internals.js'
 
 import { Observable } from 'lib0/observable.js'
 import * as random from 'lib0/random.js'
 import * as map from 'lib0/map.js'
+import * as array from 'lib0/array.js'
 
 export const generateNewClientId = random.uint32
 
+/**
+ * @typedef {Object} DocOpts
+ * @property {boolean} [DocOpts.gc=true] Disable garbage collection (default: gc=true)
+ * @property {function(Item):boolean} [DocOpts.gcFilter] Will be called before an Item is garbage collected. Return false to keep the Item.
+ * @property {string} [DocOpts.guid] Define a globally unique identifier for this document
+ * @property {any} [DocOpts.meta] Any kind of meta information you want to associate with this document. If this is a subdocument, remote peers will store the meta information as well.
+ * @property {boolean} [DocOpts.autoLoad] If a subdocument, automatically load document. If this is a subdocument, remote peers will load the document as well automatically.
+ */
+
 /**
  * A Yjs instance handles the state of shared data.
  * @extends Observable<string>
  */
 export class Doc extends Observable {
   /**
-   * @param {Object} conf configuration
+   * @param {DocOpts} [opts] configuration
-   * @param {boolean} [conf.gc] Disable garbage collection (default: gc=true)
-   * @param {function(Item):boolean} [conf.gcFilter] Will be called before an Item is garbage collected. Return false to keep the Item.
    */
-  constructor ({ gc = true, gcFilter = () => true } = {}) {
+  constructor ({ guid = random.uuidv4(), gc = true, gcFilter = () => true, meta = null, autoLoad = false } = {}) {
     super()
     this.gc = gc
     this.gcFilter = gcFilter
     this.clientID = generateNewClientId()
+    this.guid = guid
     /**
      * @type {Map<string, AbstractType<YEvent>>}
      */
@@ -47,6 +56,43 @@ export class Doc extends Observable {
      * @type {Array<Transaction>}
      */
     this._transactionCleanups = []
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocs = new Set()
+    /**
+     * If this document is a subdocument - a document integrated into another document - then _item is defined.
+     * @type {Item?}
+     */
+    this._item = null
+    this.shouldLoad = autoLoad
+    this.autoLoad = autoLoad
+    this.meta = meta
+  }
+
+  /**
+   * Notify the parent document that you request to load data into this subdocument (if it is a subdocument).
+   *
+   * `load()` might be used in the future to request any provider to load the most current data.
+   *
+   * It is safe to call `load()` multiple times.
+   */
+  load () {
+    const item = this._item
+    if (item !== null && !this.shouldLoad) {
+      transact(/** @type {any} */ (item.parent).doc, transaction => {
+        transaction.subdocsLoaded.add(this)
+      }, null, true)
+    }
+    this.shouldLoad = true
+  }
+
+  getSubdocs () {
+    return this.subdocs
+  }
+
+  getSubdocGuids () {
+    return new Set(Array.from(this.subdocs).map(doc => doc.guid))
   }
 
   /**
@@ -191,13 +237,32 @@ export class Doc extends Observable {
    * Emit `destroy` event and unregister all event handlers.
    */
   destroy () {
+    array.from(this.subdocs).forEach(subdoc => subdoc.destroy())
+    const item = this._item
+    if (item !== null) {
+      this._item = null
+      const content = /** @type {ContentDoc} */ (item.content)
+      if (item.deleted) {
+        // @ts-ignore
+        content.doc = null
+      } else {
+        content.doc = new Doc({ guid: this.guid, ...content.opts })
+        content.doc._item = item
+      }
+      transact(/** @type {any} */ (item).parent.doc, transaction => {
+        if (!item.deleted) {
+          transaction.subdocsAdded.add(content.doc)
+        }
+        transaction.subdocsRemoved.add(this)
+      }, null, true)
+    }
     this.emit('destroyed', [true])
     super.destroy()
   }
 
   /**
    * @param {string} eventName
-   * @param {function} f
+   * @param {function(...any):any} f
    */
   on (eventName, f) {
     super.on(eventName, f)

src/utils/Transaction.js

@@ -102,6 +102,18 @@ export class Transaction {
      * @type {boolean}
      */
     this.local = local
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocsAdded = new Set()
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocsRemoved = new Set()
+    /**
+     * @type {Set<Doc>}
+     */
+    this.subdocsLoaded = new Set()
   }
 }
 
@@ -335,6 +347,12 @@ const cleanupTransactions = (transactionCleanups, i) => {
           doc.emit('updateV2', [encoder.toUint8Array(), transaction.origin, doc])
         }
       }
+      transaction.subdocsAdded.forEach(subdoc => doc.subdocs.add(subdoc))
+      transaction.subdocsRemoved.forEach(subdoc => doc.subdocs.delete(subdoc))
+
+      doc.emit('subdocs', [{ loaded: transaction.subdocsLoaded, added: transaction.subdocsAdded, removed: transaction.subdocsRemoved }])
+      transaction.subdocsRemoved.forEach(subdoc => subdoc.destroy())
+
       if (transactionCleanups.length <= i + 1) {
         doc._transactionCleanups = []
         doc.emit('afterAllTransactions', [doc, transactionCleanups])

tests/doc.tests.js

@@ -57,3 +57,70 @@ export const testToJSON = tc => {
     }
   }, 'doc.toJSON has array and recursive map')
 }
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testSubdoc = tc => {
+  const doc = new Y.Doc()
+  doc.load() // doesn't do anything
+  {
+    /**
+     * @type {Array<any>|null}
+     */
+    let event = /** @type {any} */ (null)
+    doc.on('subdocs', subdocs => {
+      event = [Array.from(subdocs.added).map(x => x.guid), Array.from(subdocs.removed).map(x => x.guid), Array.from(subdocs.loaded).map(x => x.guid)]
+    })
+    const subdocs = doc.getMap('mysubdocs')
+    const docA = new Y.Doc({ guid: 'a' })
+    docA.load()
+    subdocs.set('a', docA)
+    t.compare(event, [['a'], [], ['a']])
+
+    event = null
+    subdocs.get('a').load()
+    t.assert(event === null)
+
+    event = null
+    subdocs.get('a').destroy()
+    t.compare(event, [['a'], ['a'], []])
+    subdocs.get('a').load()
+    t.compare(event, [[], [], ['a']])
+
+    subdocs.set('b', new Y.Doc({ guid: 'a' }))
+    t.compare(event, [['a'], [], []])
+    subdocs.get('b').load()
+    t.compare(event, [[], [], ['a']])
+
+    const docC = new Y.Doc({ guid: 'c' })
+    docC.load()
+    subdocs.set('c', docC)
+    t.compare(event, [['c'], [], ['c']])
+
+    t.compare(Array.from(doc.getSubdocGuids()), ['a', 'c'])
+  }
+
+  const doc2 = new Y.Doc()
+  {
+    t.compare(Array.from(doc2.getSubdocs()), [])
+    /**
+     * @type {Array<any>|null}
+     */
+    let event = /** @type {any} */ (null)
+    doc2.on('subdocs', subdocs => {
+      event = [Array.from(subdocs.added).map(d => d.guid), Array.from(subdocs.removed).map(d => d.guid), Array.from(subdocs.loaded).map(d => d.guid)]
+    })
+    Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc))
+    t.compare(event, [['a', 'a', 'c'], [], []])
+
+    doc2.getMap('mysubdocs').get('a').load()
+    t.compare(event, [[], [], ['a']])
+
+    t.compare(Array.from(doc2.getSubdocGuids()), ['a', 'c'])
+
+    doc2.getMap('mysubdocs').delete('a')
+    t.compare(event, [[], ['a'], []])
+    t.compare(Array.from(doc2.getSubdocGuids()), ['a', 'c'])
+  }
+}

tests/encoding.tests.js

@@ -9,14 +9,15 @@ import {
   readContentEmbed,
   readContentType,
   readContentFormat,
-  readContentAny
+  readContentAny,
+  readContentDoc
 } from '../src/internals.js'
 
 /**
  * @param {t.TestCase} tc
  */
 export const testStructReferences = tc => {
-  t.assert(contentRefs.length === 9)
+  t.assert(contentRefs.length === 10)
   t.assert(contentRefs[1] === readContentDeleted)
   t.assert(contentRefs[2] === readContentJSON) // TODO: deprecate content json?
   t.assert(contentRefs[3] === readContentBinary)
@@ -25,4 +26,5 @@ export const testStructReferences = tc => {
   t.assert(contentRefs[6] === readContentFormat)
   t.assert(contentRefs[7] === readContentType)
   t.assert(contentRefs[8] === readContentAny)
+  t.assert(contentRefs[9] === readContentDoc)
 }