Merge branch '159-create-doc-from-snapshot-2' of git://github.com/calibr/yjs into calibr-159-create-doc-from-snapshot-2

src/index.js

@@ -47,6 +47,7 @@ export {
   findRootTypeKey,
   typeListToArraySnapshot,
   typeMapGetSnapshot,
+  createDocFromSnapshot,
   iterateDeletedStructs,
   applyUpdate,
   applyUpdateV2,

src/utils/DeleteSet.js

@@ -278,37 +278,13 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => {
     const state = getState(store, client)
     for (let i = 0; i < numberOfDeletes; i++) {
       const clock = decoder.readDsClock()
-      const clockEnd = clock + decoder.readDsLen()
+      const len = decoder.readDsLen()
+      const clockEnd = clock + len
       if (clock < state) {
         if (state < clockEnd) {
           addToDeleteSet(unappliedDS, client, state, clockEnd - state)
         }
-        let index = findIndexSS(structs, clock)
+        applyDeleteItem(transaction, structs, { clock, len })
-        /**
-         * We can ignore the case of GC and Delete structs, because we are going to skip them
-         * @type {Item}
-         */
-        // @ts-ignore
-        let struct = structs[index]
-        // split the first item if necessary
-        if (!struct.deleted && struct.id.clock < clock) {
-          structs.splice(index + 1, 0, splitItem(transaction, struct, clock - struct.id.clock))
-          index++ // increase we now want to use the next struct
-        }
-        while (index < structs.length) {
-          // @ts-ignore
-          struct = structs[index++]
-          if (struct.id.clock < clockEnd) {
-            if (!struct.deleted) {
-              if (clockEnd < struct.id.clock + struct.length) {
-                structs.splice(index, 0, splitItem(transaction, struct, clockEnd - struct.id.clock))
-              }
-              struct.delete(transaction)
-            }
-          } else {
-            break
-          }
-        }
       } else {
         addToDeleteSet(unappliedDS, client, clock, clockEnd - clock)
       }
@@ -321,3 +297,43 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => {
     store.pendingDeleteReaders.push(new DSDecoderV2(decoding.createDecoder((unappliedDSEncoder.toUint8Array()))))
   }
 }
+
+/**
+ * Applies a DeleteItem on a document
+ *
+ * @param {Transaction} transaction
+ * @param {Array<GC|Item>} structs
+ * @param {DeleteItem} deleteItem
+ *
+ * @private
+ * @function
+ */
+export const applyDeleteItem = (transaction, structs, { clock, len }) => {
+  const clockEnd = clock + len
+  let index = findIndexSS(structs, clock)
+  /**
+   * We can ignore the case of GC and Delete structs, because we are going to skip them
+   * @type {Item}
+   */
+  // @ts-ignore
+  let struct = structs[index]
+  // split the first item if necessary
+  if (!struct.deleted && struct.id.clock < clock) {
+    structs.splice(index + 1, 0, splitItem(transaction, struct, clock - struct.id.clock))
+    index++ // increase we now want to use the next struct
+  }
+  while (index < structs.length) {
+    // @ts-ignore
+    struct = structs[index++]
+    if (struct.id.clock < clockEnd) {
+      if (!struct.deleted) {
+        if (clockEnd < struct.id.clock + struct.length) {
+          structs.splice(index, 0, splitItem(transaction, struct, clockEnd - struct.id.clock))
+        }
+        struct.delete(transaction)
+      }
+    } else {
+      break
+    }
+  }
+}

src/utils/Snapshot.js

@@ -3,6 +3,7 @@ import {
   isDeleted,
   createDeleteSetFromStructStore,
   getStateVector,
+  getItem,
   getItemCleanStart,
   iterateDeletedStructs,
   writeDeleteSet,
@@ -11,7 +12,11 @@ import {
   readStateVector,
   createDeleteSet,
   createID,
+  ID,
   getState,
+  findIndexCleanStart,
+  AbstractStruct,
+  applyDeleteItem,
   AbstractDSDecoder, AbstractDSEncoder, DSEncoderV1, DSEncoderV2, DSDecoderV1, DSDecoderV2, Transaction, Doc, DeleteSet, Item // eslint-disable-line
 } from '../internals.js'
 
@@ -148,3 +153,124 @@ export const splitSnapshotAffectedStructs = (transaction, snapshot) => {
     meta.add(snapshot)
   }
 }
+
+/**
+ * @param {Doc} originDoc
+ * @param {Snapshot} snapshot
+ * @return {Doc}
+ */
+export const createDocFromSnapshot = (originDoc, snapshot) => {
+  if (originDoc.gc) {
+    // we should not try to restore a GC-ed document, because some of the restored items might have their content deleted
+    throw new Error('originDoc must not be garbage collected')
+  }
+  const { sv, ds } = snapshot
+  const needState = new Map(sv)
+
+  let len = 0
+  const tempStructs = []
+  /**
+   * State Map
+   * @type any[]
+   */
+  const itemsToIntegrate = []
+  originDoc.transact(transaction => {
+    for (let user of needState.keys()) {
+      let clock = needState.get(user) || 0
+      const userItems = originDoc.store.clients.get(user)
+      if (!userItems) {
+        continue
+      }
+
+      let lastIndex
+      const lastItem = userItems[userItems.length - 1]
+      if (clock === lastItem.id.clock + lastItem.length) {
+        lastIndex = lastItem.id.clock + lastItem.length + 1
+      } else {
+        lastIndex = findIndexCleanStart(transaction, userItems, clock)
+      }
+      for (let i = 0; i < lastIndex; i++) {
+        const item = userItems[i]
+        if (item instanceof Item) {
+          itemsToIntegrate.push({
+            id: item.id,
+            left: item.left ? item.left.id : null,
+            right: item.right ? item.right.id : null,
+            origin: item.origin ? createID(item.origin.client, item.origin.clock) : null,
+            rightOrigin: item.rightOrigin ? createID(item.rightOrigin.client, item.rightOrigin.clock) : null,
+            parent: item.parent,
+            parentSub: item.parentSub,
+            content: item.content.copy()
+          })
+        }
+      }
+    }
+  })
+
+  const newDoc = new Doc()
+
+  // copy root types
+  const sharedKeysByValue = new Map()
+  for (const [key, t] of originDoc.share) {
+    const Constructor = t.constructor
+    newDoc.get(key, Constructor)
+    sharedKeysByValue.set(t, key)
+  }
+
+  let lastId = new Map()
+  /**
+   * @param {ID} id
+   * @return {Item|null}
+   */
+  const getItemSafe = (id) => {
+    if (!lastId.has(id.client)) {
+      return null
+    }
+    if (lastId.get(id.client) < id.clock) {
+      return null
+    }
+    return getItem(newDoc.store, id)
+  }
+  newDoc.transact(transaction => {
+    for (const item of itemsToIntegrate) {
+      let parent = null
+      let left = null
+      let right = null
+      const sharedKey = sharedKeysByValue.get(item.parent)
+      if (sharedKey) {
+        parent = newDoc.get(sharedKey)
+      } else if (item.parent) {
+        parent = getItem(newDoc.store, item.parent._item.id).content.type
+      }
+      if (item.left) {
+        left = getItemSafe(item.left)
+      }
+      if (item.right) {
+        right = getItemSafe(item.right)
+      }
+      lastId.set(item.id.client, item.id.clock)
+      const newItem = new Item(
+        item.id,
+        left,
+        item.origin,
+        right,
+        item.rightOrigin,
+        parent, // not sure
+        item.parentSub,
+        item.content
+      )
+      newItem.integrate(transaction, 0)
+    }
+
+    for (const [client, deleteItems] of ds.clients) {
+      for (const deleteItem of deleteItems) {
+        const items = newDoc.store.clients.get(client)
+        if (items) {
+          applyDeleteItem(transaction, items, deleteItem)
+        }
+      }
+    }
+  })
+
+  return newDoc
+}

tests/index.js

@@ -7,6 +7,7 @@ import * as encoding from './encoding.tests.js'
 import * as undoredo from './undo-redo.tests.js'
 import * as compatibility from './compatibility.tests.js'
 import * as doc from './doc.tests.js'
+import * as snapshot from './snapshot.tests.js'
 
 import { runTests } from 'lib0/testing.js'
 import { isBrowser, isNode } from 'lib0/environment.js'
@@ -16,7 +17,7 @@ if (isBrowser) {
   log.createVConsole(document.body)
 }
 runTests({
-  doc, map, array, text, xml, encoding, undoredo, compatibility
+  doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot
 }).then(success => {
   /* istanbul ignore next */
   if (isNode) {

tests/snapshot.tests.js

@@ -0,0 +1,110 @@
+import { createDocFromSnapshot, Doc, snapshot, YMap } from '../src/internals'
+import * as t from 'lib0/testing.js'
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testBasicRestoreSnapshot = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['hello'])
+  const snap = snapshot(doc)
+  doc.getArray('array').insert(1, ['world'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['hello'])
+  t.compare(doc.getArray('array').toArray(), ['hello', 'world'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRestoreSnapshotWithSubType = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, [new YMap()])
+  const subMap = doc.getArray('array').get(0)
+  subMap.set('key1', 'value1')
+
+  const snap = snapshot(doc)
+  subMap.set('key2', 'value2')
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toJSON(), [{
+    key1: 'value1'
+  }])
+  t.compare(doc.getArray('array').toJSON(), [{
+    key1: 'value1',
+    key2: 'value2'
+  }])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRestoreDeletedItem1 = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1', 'item2'])
+
+  const snap = snapshot(doc)
+  doc.getArray('array').delete(0)
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['item1', 'item2'])
+  t.compare(doc.getArray('array').toArray(), ['item2'])
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRestoreLeftItem = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1'])
+  doc.getMap('map').set('test', 1)
+  doc.getArray('array').insert(0, ['item0'])
+
+  const snap = snapshot(doc)
+  doc.getArray('array').delete(1)
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['item0', 'item1'])
+  t.compare(doc.getArray('array').toArray(), ['item0'])
+}
+
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testDeletedItemsBase = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1'])
+  doc.getArray('array').delete(0)
+  const snap = snapshot(doc)
+  doc.getArray('array').insert(0, ['item0'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), [])
+  t.compare(doc.getArray('array').toArray(), ['item0'])
+}
+
+
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testDeletedItems2 = tc => {
+  const doc = new Doc({ gc: false })
+  doc.getArray('array').insert(0, ['item1', 'item2', 'item3'])
+  doc.getArray('array').delete(1)
+  const snap = snapshot(doc)
+  doc.getArray('array').insert(0, ['item0'])
+
+  const docRestored = createDocFromSnapshot(doc, snap)
+
+  t.compare(docRestored.getArray('array').toArray(), ['item1', 'item3'])
+  t.compare(doc.getArray('array').toArray(), ['item0', 'item1', 'item3'])
+}
+