From 3a0694c35c5d9d9e53e736afad1d520d24abb28a Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 29 Jan 2019 00:54:58 +0100 Subject: [PATCH] added utilities to make and recover snapshots --- bindings/prosemirror.js | 132 +++++++++++++++++++----- examples/prosemirror-history.js | 113 ++++++++++++++++++-- examples/prosemirror.html | 33 +++++- examples/prosemirror.js | 8 +- index.js | 1 - package-lock.json | 3 +- persistence/leveldb.js | 36 +++---- protocols/awareness.js | 32 +++--- protocols/history.js | 25 +++-- provider/websocket/WebSocketProvider.js | 10 +- provider/websocket/server.js | 6 +- tmp/000005.ldb | Bin 4834 -> 0 bytes tmp/000006.log | Bin 5003 -> 0 bytes tmp/CURRENT | 1 - tmp/LOCK | 0 tmp/LOG | 5 - tmp/LOG.old | 1 - tmp/MANIFEST-000004 | Bin 129 -> 0 bytes types/YMap.js | 8 +- types/YText.js | 20 +++- types/YXmlElement.js | 6 +- utils/DeleteStore.js | 1 - utils/snapshot.js | 5 +- 23 files changed, 337 insertions(+), 109 deletions(-) delete mode 100644 tmp/000005.ldb delete mode 100644 tmp/000006.log delete mode 100644 tmp/CURRENT delete mode 100644 tmp/LOCK delete mode 100644 tmp/LOG delete mode 100644 tmp/LOG.old delete mode 100644 tmp/MANIFEST-000004 diff --git a/bindings/prosemirror.js b/bindings/prosemirror.js index 984a7dcc..7bd15c72 100644 --- a/bindings/prosemirror.js +++ b/bindings/prosemirror.js @@ -11,6 +11,8 @@ import { Plugin, PluginKey, EditorState, TextSelection } from 'prosemirror-state import * as math from '../lib/math.js' import * as object from '../lib/object.js' import * as YPos from '../utils/relativePosition.js' +import { isVisible } from '../utils/snapshot.js' +import { simpleDiff } from '../lib/diff.js' /** * @typedef {Map} ProsemirrorMapping @@ -33,6 +35,9 @@ export const prosemirrorPluginKey = new PluginKey('yjs') export const prosemirrorPlugin = yXmlFragment => { let changedInitialContent = false const plugin = new Plugin({ + props: { + editable: (state) => prosemirrorPluginKey.getState(state).snapshot == null + }, key: prosemirrorPluginKey, state: { init: (initargs, state) => { @@ -55,7 +60,16 @@ export const prosemirrorPlugin = yXmlFragment => { if (change !== undefined && change.snapshot !== undefined) { // snapshot changed, rerender next setTimeout(() => { - pluginState.binding._renderSnapshot(change.snapshot) + if (change.restore == null) { + pluginState.binding._renderSnapshot(change.snapshot, change.prevSnapshot) + } else { + pluginState.binding._renderSnapshot(change.snapshot, change.snapshot) + // reset to current prosemirror state + delete pluginState.restore + delete pluginState.snapshot + delete pluginState.prevSnapshot + pluginState.binding._prosemirrorChanged(pluginState.binding.prosemirrorView.state.doc) + } }, 0) } else if (pluginState.snapshot == null) { // only apply if no snapshot active @@ -112,7 +126,14 @@ export const cursorPlugin = new Plugin({ const y = ystate.y const awareness = y.getAwarenessInfo() const decorations = [] + if (ystate.snapshot != null) { + // do not render cursors while snapshot is active + return + } awareness.forEach((aw, userID) => { + if (userID === y.userID) { + return + } if (aw.cursor != null) { let user = aw.user || {} if (user.color == null) { @@ -154,7 +175,7 @@ export const cursorPlugin = new Plugin({ } const updateCursorInfo = () => { const current = y.getLocalAwarenessInfo() - if (view.hasFocus()) { + if (view.hasFocus() && ystate.binding !== null) { const anchor = absolutePositionToRelativePosition(view.state.selection.anchor, ystate.type, ystate.binding.mapping) const head = absolutePositionToRelativePosition(view.state.selection.head, ystate.type, ystate.binding.mapping) if (current.cursor == null || !YPos.equal(current.cursor.anchor, anchor) || !YPos.equal(current.cursor.head, head)) { @@ -321,11 +342,24 @@ export class ProsemirrorBinding { }) yXmlFragment.observeDeep(this._observeFunction) } - _renderSnapshot (snapshot) { + _forceRerender () { + this.mapping = new Map() + this.mux(() => { + const fragmentContent = this.type.toArray().map(t => createNodeFromYElement(t, this.prosemirrorView.state.schema, this.mapping)).filter(n => n !== null) + const tr = this.prosemirrorView.state.tr.replace(0, this.prosemirrorView.state.doc.content.size, new PModel.Slice(new PModel.Fragment(fragmentContent), 0, 0)) + this.prosemirrorView.dispatch(tr) + }) + } + /** + * + * @param {*} snapshot + * @param {*} prevSnapshot + */ + _renderSnapshot (snapshot, prevSnapshot) { // clear mapping because we are going to rerender this.mapping = new Map() this.mux(() => { - const fragmentContent = this.type.toArray(snapshot).map(t => createNodeFromYElement(t, this.prosemirrorView.state.schema, new Map(), snapshot)).filter(n => n !== null) + const fragmentContent = this.type.toArray({ sm: snapshot.sm, ds: prevSnapshot.ds}).map(t => createNodeFromYElement(t, this.prosemirrorView.state.schema, new Map(), snapshot, prevSnapshot)).filter(n => n !== null) const tr = this.prosemirrorView.state.tr.replace(0, this.prosemirrorView.state.doc.content.size, new PModel.Slice(new PModel.Fragment(fragmentContent), 0, 0)) this.prosemirrorView.dispatch(tr) }) @@ -370,12 +404,13 @@ export class ProsemirrorBinding { * @param {PModel.Schema} schema * @param {ProsemirrorMapping} mapping * @param {HistorySnapshot} [snapshot] + * @param {HistorySnapshot} [prevSnapshot] * @return {PModel.Node} */ -export const createNodeIfNotExists = (el, schema, mapping, snapshot) => { +export const createNodeIfNotExists = (el, schema, mapping, snapshot, prevSnapshot) => { const node = mapping.get(el) if (node === undefined) { - return createNodeFromYElement(el, schema, mapping, snapshot) + return createNodeFromYElement(el, schema, mapping, snapshot, prevSnapshot) } return node } @@ -385,19 +420,31 @@ export const createNodeIfNotExists = (el, schema, mapping, snapshot) => { * @param {YXmlElement} el * @param {PModel.Schema} schema * @param {ProsemirrorMapping} mapping - * @param {import('../protocols/history.js').HistorySnapshot} snapshot + * @param {import('../protocols/history.js').HistorySnapshot} [snapshot] + * @param {import('../protocols/history.js').HistorySnapshot} [prevSnapshot] * @return {PModel.Node | null} Returns node if node could be created. Otherwise it deletes the yjs type and returns null */ -export const createNodeFromYElement = (el, schema, mapping, snapshot) => { +export const createNodeFromYElement = (el, schema, mapping, snapshot, prevSnapshot) => { + let _snapshot = snapshot + let _prevSnapshot = prevSnapshot + if (snapshot !== undefined && prevSnapshot !== undefined) { + if (!isVisible(el, snapshot)) { + // if this element is already rendered as deleted (ychange), then do not render children as deleted + _snapshot = {sm: snapshot.sm, ds: prevSnapshot.ds} + _prevSnapshot = _snapshot + } else if (!isVisible(el, prevSnapshot)) { + _prevSnapshot = _snapshot + } + } const children = [] - el.toArray(snapshot).forEach(type => { + const createChildren = type => { if (type.constructor === YXmlElement) { - const n = createNodeIfNotExists(type, schema, mapping, snapshot) + const n = createNodeIfNotExists(type, schema, mapping, _snapshot, _prevSnapshot) if (n !== null) { children.push(n) } } else { - const ns = createTextNodesFromYText(type, schema, mapping, snapshot) + const ns = createTextNodesFromYText(type, schema, mapping, _snapshot, _prevSnapshot) if (ns !== null) { ns.forEach(textchild => { if (textchild !== null) { @@ -406,16 +453,31 @@ export const createNodeFromYElement = (el, schema, mapping, snapshot) => { }) } } - }) + } + if (snapshot === undefined || prevSnapshot === undefined) { + el.toArray().forEach(createChildren) + } else { + el.toArray({sm: snapshot.sm, ds: prevSnapshot.ds}).forEach(createChildren) + } let node try { - node = schema.node(el.nodeName.toLowerCase(), el.getAttributes(snapshot), children) + const attrs = el.getAttributes(_snapshot) + if (snapshot !== undefined) { + if (!isVisible(el, snapshot)) { + attrs.ychange = { user: el._id.user, state: 'removed' } + } else if (!isVisible(el, prevSnapshot)) { + attrs.ychange = { user: el._id.user, state: 'added' } + } + } + node = schema.node(el.nodeName.toLowerCase(), attrs, children) } catch (e) { // an error occured while creating the node. This is probably a result because of a concurrent action. - // delete the node and do not push to children + // ignore the node while rendering + /* do not delete anymore el._y.transact(() => { el._delete(el._y, true) }) + */ return null } mapping.set(el, node) @@ -427,12 +489,13 @@ export const createNodeFromYElement = (el, schema, mapping, snapshot) => { * @param {YText} text * @param {PModel.Schema} schema * @param {ProsemirrorMapping} mapping - * @param {HistorySnapshot} [snapshot] + * @param {HistorySnapshot} [snapshot] + * @param {HistorySnapshot} [prevSnapshot] * @return {Array} */ -export const createTextNodesFromYText = (text, schema, mapping, snapshot) => { +export const createTextNodesFromYText = (text, schema, mapping, snapshot, prevSnapshot) => { const nodes = [] - const deltas = text.toDelta(snapshot) + const deltas = text.toDelta(snapshot, prevSnapshot) try { for (let i = 0; i < deltas.length; i++) { const delta = deltas[i] @@ -446,9 +509,11 @@ export const createTextNodesFromYText = (text, schema, mapping, snapshot) => { mapping.set(text, nodes[0]) // only map to first child, all following children are also considered bound to this type } } catch (e) { + /* text._y.transact(() => { text._delete(text._y, true) }) + */ return null } return nodes @@ -465,13 +530,17 @@ export const createTypeFromNode = (node, mapping) => { if (node.isText) { type = new YText() const attrs = {} - node.marks.forEach(mark => { attrs[mark.type.name] = mark.attrs }) + node.marks.forEach(mark => { + if (mark.type.name !== 'ychange') { + attrs[mark.type.name] = mark.attrs + } + }) type.insert(0, node.text, attrs) } else { type = new YXmlElement(node.type.name) for (let key in node.attrs) { const val = node.attrs[key] - if (val !== null) { + if (val !== null && key !== 'ychange') { type.setAttribute(key, val) } } @@ -490,7 +559,9 @@ const equalAttrs = (pattrs, yattrs) => { let eq = keys.length === Object.keys(yattrs).filter(key => yattrs[key] === null).length for (let i = 0; i < keys.length && eq; i++) { const key = keys[i] - eq = pattrs[key] === yattrs[key] + const l = pattrs[key] + const r = yattrs[key] + eq = key === 'ychange' || l === r || (typeof l === 'object' && typeof r === 'object' && equalAttrs(l, r)) } return eq } @@ -554,7 +625,7 @@ const updateYFragment = (yDomFragment, pContent, mapping) => { const pAttrs = pContent.attrs for (let key in pAttrs) { if (pAttrs[key] !== null) { - if (yDomAttrs[key] !== pAttrs[key]) { + if (yDomAttrs[key] !== pAttrs[key] && key !== 'ychange') { yDomFragment.setAttribute(key, pAttrs[key]) } } else { @@ -610,8 +681,23 @@ const updateYFragment = (yDomFragment, pContent, mapping) => { const rightP = pContent.child(pChildCnt - right - 1) if (leftY.constructor === YText && leftP.isText) { if (!equalYTextPText(leftY, leftP)) { - yDomFragment.delete(left, 1) - yDomFragment.insert(left, [createTypeFromNode(leftP, mapping)]) + // try to apply diff. Only if attrs don't match, delete insert + // TODO: use a single ytext to hold all following Prosemirror Text nodes + const pattrs = {} + leftP.marks.forEach(mark => { + if (mark.type.name !== 'ychange') { + pattrs[mark.type.name] = mark.attrs + } + }) + const delta = leftY.toDelta() + if (delta.length === 1 && delta[0].insert && equalAttrs(pattrs, delta[0].attributes || {})) { + const diff = simpleDiff(delta[0].insert, leftP.text) + leftY.delete(diff.pos, diff.remove) + leftY.insert(diff.pos, diff.insert) + } else { + yDomFragment.delete(left, 1) + yDomFragment.insert(left, [createTypeFromNode(leftP, mapping)]) + } } left += 1 } else { diff --git a/examples/prosemirror-history.js b/examples/prosemirror-history.js index c3da82a9..36cffc7b 100644 --- a/examples/prosemirror-history.js +++ b/examples/prosemirror-history.js @@ -1,5 +1,5 @@ -import {Plugin} from "prosemirror-state" +import {Plugin} from 'prosemirror-state' import crel from 'crel' import * as Y from '../index.js' import { prosemirrorPluginKey } from '../bindings/prosemirror.js' @@ -7,21 +7,53 @@ import * as encoding from '../lib/encoding.js' import * as decoding from '../lib/decoding.js' import * as historyProtocol from '../protocols/history.js' +const niceColors = ['#3cb44b', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#008080', '#9a6324', '#800000', '#808000', '#000075', '#808080'] + +const createUserCSS = (userid, username, color = 'rgb(250, 129, 0)', color2 = 'rgba(250, 129, 0, .41)') => ` + [ychange_state][ychange_user="${userid}"]:hover::before { + content: "${username}" !important; + background-color: ${color} !important; + } + [ychange_state="added"][ychange_user="${userid}"] { + background-color: ${color2} !important; + } + [ychange_state="removed"][ychange_user="${userid}"] { + color: ${color} !important; + } +` + export const noteHistoryPlugin = new Plugin({ - view (editorView) { return new NoteHistoryPlugin(editorView) } + state: { + init (initargs, state) { + return new NoteHistoryPlugin() + }, + apply (tr, pluginState) { + return pluginState + } + }, + view (editorView) { + const hstate = noteHistoryPlugin.getState(editorView.state) + hstate.init(editorView) + return { + destroy: hstate.destroy.bind(hstate) + } + } }) const createWrapper = () => { - const wrapper = crel('div', { style: 'display: flex' }) + const wrapper = crel('div', { style: 'display: flex;' }) const historyContainer = crel('div', { style: 'align-self: baseline; flex-basis: 250px;', class: 'shared-history' }) wrapper.insertBefore(historyContainer, null) - return { wrapper, historyContainer } + const userStyleContainer = crel('style') + wrapper.insertBefore(userStyleContainer, null) + return { wrapper, historyContainer, userStyleContainer } } class NoteHistoryPlugin { - constructor(editorView) { + init (editorView) { this.editorView = editorView - const { historyContainer, wrapper } = createWrapper() + const { historyContainer, wrapper, userStyleContainer } = createWrapper() + this.userStyleContainer = userStyleContainer this.wrapper = wrapper this.historyContainer = historyContainer const n = editorView.dom.parentNode.parentNode @@ -33,35 +65,94 @@ class NoteHistoryPlugin { const history = y.define('history', Y.Array) history.observe(this.render.bind(this)) } + destroy () { + this.wrapper.parentNode.replaceChild(this.wrapper.firstChild, this.wrapper) + const y = prosemirrorPluginKey.getState(this.editorView.state).y + const history = y.define('history', Y.Array) + history.unobserve(this.render) + } render () { const y = prosemirrorPluginKey.getState(this.editorView.state).y const history = y.define('history', Y.Array).toArray() const fragment = document.createDocumentFragment() const snapshotBtn = crel('button', { type: 'button' }, ['snapshot']) - snapshotBtn.addEventListener('click', this.snapshot.bind(this)) fragment.insertBefore(snapshotBtn, null) + let _prevSnap = null // empty + snapshotBtn.addEventListener('click', () => { + const awareness = y.getAwarenessInfo() + const userMap = new Map() + const aw = y.getLocalAwarenessInfo() + userMap.set(y.userID, aw.name || 'unknown') + awareness.forEach((a, userID) => { + userMap.set(userID, a.name || 'Unknown') + }) + this.snapshot(userMap) + }) history.forEach(buf => { const decoder = decoding.createDecoder(buf) const snapshot = historyProtocol.readHistorySnapshot(decoder) const date = new Date(decoding.readUint32(decoder) * 1000) + const restoreBtn = crel('button', { type: 'button' }, ['restore']) const a = crel('a', [ - '• '+ date.toUTCString() + '• ' + date.toUTCString(), restoreBtn ]) const el = crel('div', [ a ]) + let prevSnapshot = _prevSnap // rebind to new variable + restoreBtn.addEventListener('click', event => { + if (prevSnapshot === null) { + prevSnapshot = { ds: snapshot.ds, sm: new Map() } + } + this.editorView.dispatch(this.editorView.state.tr.setMeta(prosemirrorPluginKey, { snapshot, prevSnapshot, restore: true })) + event.stopPropagation() + }) a.addEventListener('click', () => { console.log('setting snapshot') - this.editorView.dispatch(this.editorView.state.tr.setMeta(prosemirrorPluginKey, { snapshot })) + if (prevSnapshot === null) { + prevSnapshot = { ds: snapshot.ds, sm: new Map() } + } + this.renderSnapshot(snapshot, prevSnapshot) }) fragment.insertBefore(el, null) + _prevSnap = snapshot }) this.historyContainer.innerHTML = '' this.historyContainer.insertBefore(fragment, null) } - snapshot () { + renderSnapshot (snapshot, prevSnapshot) { + this.editorView.dispatch(this.editorView.state.tr.setMeta(prosemirrorPluginKey, { snapshot, prevSnapshot })) + /** + * @type {Array} + */ + let colors = niceColors.slice() + let style = '' + snapshot.userMap.forEach((name, userid) => { + /** + * @type {any} + */ + const randInt = name.split('').map(s => s.charCodeAt(0)).reduce((a, b) => a + b) + let color = null + let i = 0 + for (; i < colors.length && color === null; i++) { + color = colors[(randInt + i) % colors.length] + } + if (color === null) { + colors = niceColors.slice() + i = 0 + color = colors[randInt % colors.length] + } + colors[randInt % colors.length] = null + style += createUserCSS(userid, name, color, color + '69') + }) + this.userStyleContainer.innerHTML = style + } + /** + * @param {Map} [updatedUserMap] Maps from userid (yjs model) to account name (e.g. mail address) + */ + snapshot (updatedUserMap = new Map()) { const y = prosemirrorPluginKey.getState(this.editorView.state).y const history = y.define('history', Y.Array) const encoder = encoding.createEncoder() - historyProtocol.writeHistorySnapshot(encoder, y) + historyProtocol.writeHistorySnapshot(encoder, y, updatedUserMap) encoding.writeUint32(encoder, Math.floor(Date.now() / 1000)) history.push([encoding.toBuffer(encoder)]) } diff --git a/examples/prosemirror.html b/examples/prosemirror.html index 2aa4fefb..2c15392a 100644 --- a/examples/prosemirror.html +++ b/examples/prosemirror.html @@ -2,7 +2,7 @@ Yjs Prosemirror Example - + diff --git a/examples/prosemirror.js b/examples/prosemirror.js index f546b4a4..0017a0c5 100644 --- a/examples/prosemirror.js +++ b/examples/prosemirror.js @@ -6,19 +6,19 @@ import * as conf from './exampleConfig.js' import { EditorState } from 'prosemirror-state' import { EditorView } from 'prosemirror-view' -import { DOMParser } from 'prosemirror-model' -import { schema } from 'prosemirror-schema-basic' +import { DOMParser, Schema } from 'prosemirror-model' +import { schema } from './prosemirror-schema.js' import { exampleSetup } from 'prosemirror-example-setup' import { noteHistoryPlugin } from './prosemirror-history.js' const provider = new WebsocketProvider(conf.serverAddress) -const ydocument = provider.get('prosemirror') +const ydocument = provider.get('prosemirror', { gc: false }) const type = ydocument.define('prosemirror', Y.XmlFragment) const prosemirrorView = new EditorView(document.querySelector('#editor'), { state: EditorState.create({ doc: DOMParser.fromSchema(schema).parse(document.querySelector('#content')), - plugins: exampleSetup({schema}).concat([prosemirrorPlugin(type), cursorPlugin, noteHistoryPlugin]) + plugins: exampleSetup({schema}).concat([prosemirrorPlugin(type), cursorPlugin /* noteHistoryPlugin */]) }) }) diff --git a/index.js b/index.js index 09d56b2b..7ed6866e 100644 --- a/index.js +++ b/index.js @@ -55,4 +55,3 @@ registerStruct(10, YXmlText) registerStruct(11, YXmlHook) registerStruct(12, ItemEmbed) registerStruct(13, ItemBinary) - diff --git a/package-lock.json b/package-lock.json index 9e8bd726..a79f2e8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2173,7 +2173,8 @@ "crel": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/crel/-/crel-3.1.0.tgz", - "integrity": "sha512-VIGY44ERxx8lXVkOEfcB0A49OkjxkQNK+j+fHvoLy7GsGX1KKgAaQ+p9N0YgvQXu+X+ryUWGDeLx/fSI+w7+eg==" + "integrity": "sha512-VIGY44ERxx8lXVkOEfcB0A49OkjxkQNK+j+fHvoLy7GsGX1KKgAaQ+p9N0YgvQXu+X+ryUWGDeLx/fSI+w7+eg==", + "dev": true }, "cross-spawn": { "version": "5.1.0", diff --git a/persistence/leveldb.js b/persistence/leveldb.js index 86786596..5095922b 100644 --- a/persistence/leveldb.js +++ b/persistence/leveldb.js @@ -9,7 +9,7 @@ const mux = Y.createMutex() /* * Improves the uniqueness of timestamps. - * We gamble with the fact that users won't create more than 10000 changes on a single document + * We gamble with the fact that users won't create more than 10000 changes on a single document * within one millisecond (also assuming clock works correctly). */ let timestampIterator = 0 @@ -22,13 +22,13 @@ const getNextTimestamp = () => { } /** - * @param {string} docName + * @param {string} docName * @return {string} */ const generateEntryKey = docName => `${docName}#${getNextTimestamp()}` /** - * + * * @param {any} db * @param {string} docName * @param {Uint8Array | ArrayBuffer} buf @@ -44,21 +44,21 @@ const readEntry = (arr, ydocument) => mux(() => ) /** - * @param {any} db - * @param {string} docName - * @param {Y.Y} ydocument + * @param {any} db + * @param {string} docName + * @param {Y.Y} ydocument */ -const loadFromPersistence = (db, docName, ydocument) => new Promise((resolve, reject)=> +const loadFromPersistence = (db, docName, ydocument) => new Promise((resolve, reject) => db.createReadStream({ gte: `${docName}#`, lte: `${docName}#Z`, keys: false, values: true }) - .on('data', data => readEntry(data, ydocument)) - .on('error', reject) - .on('end', resolve) - .on('close', resolve) + .on('data', data => readEntry(data, ydocument)) + .on('error', reject) + .on('end', resolve) + .on('close', resolve) ) const persistState = (db, docName, ydocument) => { @@ -68,9 +68,9 @@ const persistState = (db, docName, ydocument) => { const entryPromise = db.put(entryKey, Y.encoding.toBuffer(encoder)) const delOps = [] return new Promise((resolve, reject) => db.createKeyStream({ - gte: `${docName}#`, - lt: entryKey, - }) + gte: `${docName}#`, + lt: entryKey + }) .on('data', key => delOps.push({ type: 'del', key })) .on('error', reject) .on('end', resolve) @@ -92,7 +92,7 @@ exports.LevelDbPersistence = class LevelDbPersistence { * Retrieve all data from LevelDB and automatically persist all document updates to leveldb. * * @param {string} docName - * @param {Y.Y} ydocument + * @param {Y.Y} ydocument */ bindState (docName, ydocument) { // write all updates received from other clients @@ -116,10 +116,10 @@ exports.LevelDbPersistence = class LevelDbPersistence { * Write current state to persistence layer. Deletes all entries that were made before. * Call this method at any time - the recommended time to call this method is before the ydocument is destroyed. * - * @param {string} docName - * @param {Y.Y} ydocument + * @param {string} docName + * @param {Y.Y} ydocument */ writeState (docName, ydocument) { return persistState(this.db, docName, ydocument) } -} \ No newline at end of file +} diff --git a/protocols/awareness.js b/protocols/awareness.js index 4414d0f8..9e935f56 100644 --- a/protocols/awareness.js +++ b/protocols/awareness.js @@ -40,24 +40,22 @@ export const readUsersStateChange = (decoder, y) => { const userID = decoding.readVarUint(decoder) const clock = decoding.readVarUint(decoder) const state = JSON.parse(decoding.readVarString(decoder)) - if (userID !== y.userID) { - const uClock = y.awarenessClock.get(userID) || 0 - y.awarenessClock.set(userID, clock) - if (state === null) { - // only write if clock increases. cannot overwrite - if (y.awareness.has(userID) && uClock < clock) { - y.awareness.delete(userID) - removed.push(userID) - } - } else if (uClock <= clock) { // allow to overwrite (e.g. when client was on, then offline) - if (y.awareness.has(userID)) { - updated.push(userID) - } else { - added.push(userID) - } - y.awareness.set(userID, state) - y.awarenessClock.set(userID, clock) + const uClock = y.awarenessClock.get(userID) || 0 + y.awarenessClock.set(userID, clock) + if (state === null) { + // only write if clock increases. cannot overwrite + if (y.awareness.has(userID) && uClock < clock) { + y.awareness.delete(userID) + removed.push(userID) } + } else if (uClock <= clock) { // allow to overwrite (e.g. when client was on, then offline) + if (y.awareness.has(userID)) { + updated.push(userID) + } else { + added.push(userID) + } + y.awareness.set(userID, state) + y.awarenessClock.set(userID, clock) } } if (added.length > 0 || updated.length > 0 || removed.length > 0) { diff --git a/protocols/history.js b/protocols/history.js index c74e8531..8ee5c603 100644 --- a/protocols/history.js +++ b/protocols/history.js @@ -1,5 +1,4 @@ - import * as encoding from '../lib/encoding.js' import * as decoding from '../lib/decoding.js' import { Y } from '../utils/Y.js' // eslint-disable-line @@ -10,24 +9,38 @@ import { writeStateMap, readStateMap } from '../utils/StateStore.js' * @typedef {Object} HistorySnapshot * @property {DeleteStore} HistorySnapshot.ds * @property {Map} HistorySnapshot.sm + * @property {Map} HistorySnapshot.userMap */ /** * @param {encoding.Encoder} encoder * @param {Y} y + * @param {Map} userMap */ -export const writeHistorySnapshot = (encoder, y) => { +export const writeHistorySnapshot = (encoder, y, userMap) => { writeDeleteStore(encoder, y.ds) writeStateMap(encoder, y.ss.state) + encoding.writeVarUint(encoder, userMap.size) + userMap.forEach((accountname, userid) => { + encoding.writeVarUint(encoder, userid) + encoding.writeVarString(encoder, accountname) + }) } /** - * + * * @param {decoding.Decoder} decoder * @return {HistorySnapshot} */ -export const readHistorySnapshot = (decoder) => { +export const readHistorySnapshot = decoder => { const ds = readFreshDeleteStore(decoder) const sm = readStateMap(decoder) - return { ds, sm } -} \ No newline at end of file + const size = decoding.readVarUint(decoder) + const userMap = new Map() + for (let i = 0; i < size; i++) { + const userid = decoding.readVarUint(decoder) + const accountname = decoding.readVarString(decoder) + userMap.set(userid, accountname) + } + return { ds, sm, userMap } +} diff --git a/provider/websocket/WebSocketProvider.js b/provider/websocket/WebSocketProvider.js index f10f1ce9..b40cdde6 100644 --- a/provider/websocket/WebSocketProvider.js +++ b/provider/websocket/WebSocketProvider.js @@ -97,8 +97,8 @@ const broadcastUpdate = (y, transaction) => { } class WebsocketsSharedDocument extends Y.Y { - constructor (url) { - super() + constructor (url, opts) { + super(opts) this.url = url this.wsconnected = false this.mux = Y.createMutex() @@ -112,7 +112,7 @@ class WebsocketsSharedDocument extends Y.Y { const encoder = readMessage(this, data) // already muxed this.mux(() => { if (Y.encoding.length(encoder) > 1) { - bc.publish(url, Y.encoding.toBuffer(encoder)) + bc.publish(url, Y.encoding.toBuffer(encoder)) } }) } @@ -173,10 +173,10 @@ export class WebsocketProvider { * @param {string} name * @return {WebsocketsSharedDocument} */ - get (name) { + get (name, opts) { let doc = this.docs.get(name) if (doc === undefined) { - doc = new WebsocketsSharedDocument(this.url + name) + doc = new WebsocketsSharedDocument(this.url + name, opts) } return doc } diff --git a/provider/websocket/server.js b/provider/websocket/server.js index 77e9842c..9c422a5f 100644 --- a/provider/websocket/server.js +++ b/provider/websocket/server.js @@ -12,6 +12,8 @@ const http = require('http') const port = process.env.PORT || 1234 +// disable gc when using snapshots! +const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' const persistenceDir = process.env.YPERSISTENCE let persistence = null if (typeof persistenceDir === 'string') { @@ -44,7 +46,7 @@ const afterTransaction = (doc, transaction) => { class WSSharedDoc extends Y.Y { constructor () { - super({ gc: true }) + super({ gc: gcEnabled }) this.mux = Y.createMutex() /** * Maps from conn to set of controlled user ids. Delete all user ids from awareness when this conn is closed @@ -150,4 +152,4 @@ server.on('upgrade', (request, socket, head) => { server.listen(port) -console.log('running on port', port) \ No newline at end of file +console.log('running on port', port) diff --git a/tmp/000005.ldb b/tmp/000005.ldb deleted file mode 100644 index f4c6f3979a29ef023fdc38e3b55054eb1edf7e52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4834 zcmeHLdu&|Qng6~!_srZg_u&~o#uMX-$Hq1@#0mE`cV^OTu$?%G0TVm50|}325|7KY zjuT8wDP*Y~3JoQtfi``yT9rbfYNIZwD>g!^HUy-$tUzL^`fk=HhE)DnGGrQT7fw6#*; z`IVpwiccxI7VB-{xYSAE6GM_7JoDJS^!D3tC-mI&Lz@F%pffu+=89J#_ zhr9`!_^d-}jNMx5N9>hzl!s7{X3h=OaP1#_BQu|rOi2N^fnF}9119(5PH1_miBjnzSZrwt7HxUA;~z@_c@lq;^mWjAt{)b~8DddacJyIoGG zCx8C>W8-TzS0P~nT5*;1Jx|i&gS)aG#7fj7Cve@bD12FX-Icl3Cls`opHRjg#3El3 zJoWqw*FW{Vvejksk_=F}mGD#n^%nhsVOon*4f8wgfnnb5#Zy(^D>);d1DGx^UNf+p z^5ELd@3aSY^AJ`79I#S_u8LOb`?HnW0m@RWSN`CZ(-~6Y3 z|A&YM_awhM)$PH2u1$`-3Vb;pfO~Evb9l}BQrOA!tb(pN3q=BPYGny@w{VoS3}La&AiT%SsEn>8z8c0D(xYB4!Q ziS!ida*H`d22P}_U|+m-v#JGNwf8;*fV-+=857rr4yMC2&=@}Z3+szpo1n+=m z&RF*t{bjpS!F{Xrz6f$Js=PrOW4~H0YRz`6sO|mlnAth{S}gd7Sfhnv&Y|M#>ldKG z^$#E1B!{3IGjot-1~)b{6W_!jEigEY0Zy2ia4}e$Ff(`l7q2W@uj#LL%CXH>(zsD! ziZQE90)}IP5p^a*ZA}@r;0#k|oO4WT6OT_1Q&AzhStI(AL8KB9-}tlYz2E6QbK$H} z^~ui>j-#gGs;ks+kmWQq@ffh zNlea1o3?kN4fTN12}&4osDMciCSlYe7n6c$>-Lh(ARw3j6a=m#J06f;8!*^Oan4@9wyxihi|PK7D#6j}w? z$^ei}%TNxJ%z5k3046!}{tz5P16qjfMs1<4Q1Qv{rl7IEK1xO*si@y(LFytV7FXZ) z9t$%{l+>-q^29><{Bd;EmKYmqx~CD7W$=ZH7oqxnO&v zrcBgq>{4~Y=;nE}WqkM8ej2{SE@3=$eK?5c)7bdTSdB(6z6o|7ujTzJBEMIn?GgLd zOR%efZSAm74va_a+b+%?U7;H~B@X*N=kC4hRQ{s)uLe~_J+DmHh7#xTzvKV$|9=Jl zv+u<}U{*mrWqTEx7@Hoi(KVLO0`(m_6}ikLME;Uv4!5?wh$7T+DmFZ=9PnpX_bL}_Gkd$K zhi+eL%+L4jvW&oYD!*C}m%eShp6W!czY8gzWmxN5X_SL@39jgS)Q}mFyZHTK1;E9u zLjU;B-}?7h>)i;~V%AUaQW|L=`t*JJkX)xxPlxF4Sk|zD>{EGK_MysVF(Knw;s$uD zyaNgR!*USIcJK_U6bHdemVK_JTF41=i679syn_d zxsUjZKS{rNE_?Wo>iXjYsanPgD8@@R9S&be;raK9WcX0Q4cjBVBt+bgNB6|D$Syf_ zyBuVPZo+1`rY0QJxix6P=l#(AVnn>TNtD9zHJ%OC3EY%Cx+9|A(xfIqm0Sn43}u&m zIucQDZBljNRFZrID)7RRxm2R1NCzJP845IntLePRzYA;LQb2b`viq8{CS(m>fGpmC zAeZSQk!-CgYeClJW3L$isxtm{8P|Y9Glk5)4zI=< zMlq2>HCx9BDISGU5+flnLkf%Izv$Svlr(MFE>DS$eKX%R^D|GIshJ1RXVJHy_nN8F zZ_!^tKaKtndZTwT6++VKeNFRenUUFx^YpiH>3M#43+CCzJ^Ubm;erw!E7rz-OdkW^ zL4J1&4)V6^MZaGz16V;oA1@vrI7%zvJIe2F!BMVQ=~tCXrQ`ueL7DGcJ;3j7*B`*^ zy^oI6?>bUjtk?SfMHZ?~5c+UXp@)ihq`D9e?w7jtvJ`B$hMBw?>bOcLL~A>B;c@YR z)PMa*JN^EtFZ@>hw_8X{&jHsVB|&fxkJ{26mV<&)e;w0fMj9Vgsy z3djZm%O!qvb(Z$dq6FLY7;^uw<3!W?`7-!FUint``}x`A>6FRw6K`JmOFg;5Tf0O` zS^4ssuCwh%fzF{^Jlgu@V`^a(_3O#jtu^oImrmZEKYk3qpwO9<>hiLpS)<2$iFSPS z^>c^S<&WogX)-gMpNtRtrm@0fPhpp$C8 z{j8GSGfIW*88xeIQ32?-Q%17Aw||*>@49c+eCo-EmDaP9anw&Im%l!pDLixg_R)?$ z6?Y-J3H{sXI+d<|VHfqObB*~U^~DYI^*ihHv->xUB0n5yER4<{I&^SbL~_5~_i|VU zMYRxpe(2|~Jj(Ob6T84KNO%i4{I=wFde)%4x@&;6YT%>?hn_Ru5_FjwC7+IMsW6WE z9evr>-y}$?OX<|jPj@|n7YE^qL3ADZ0($iJIQ@mJeWKqv*Z*OBxy0WMZ!JI6zqI|Q Fe*hZuXQ==H diff --git a/tmp/000006.log b/tmp/000006.log deleted file mode 100644 index 9f199c34208d8a553cabb5f8094b432ce89906ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5003 zcmeHKU1%It6uxJEc6MjdWVdRzh-nHnvZV@l{&psB#+OP}XdA=_>BH*o!`9feYZkQ+ z5v(FY(JH8kF;%N5ZLMjwNcBOHO3?>HK~YpxMDRfcr4WR^#BV88%c z<=C_5r^ea*{JiC)?6bLDLwkmH>>1j1r$rV(W^}9RmLZTS1|TY9NXjOjD94mkUQjks zg^Z+%g31%M$&ysRpo&Co$x5musD7fhI!R;g^I#DVGU{u8-#VR6*dXuMpigO*i8Iegfof%EMY70 zpCg=ws#^vtAe>L)+eo;O z#CILxjZhQ$e!lz4?g0i36o%D6B z)9m!kN3c%QsoXEElgE=ia%tz>aKPYL$`)>VaJ0I-K?!(VjO*cEt35VWY1Aso2_FREAXd(GcdK0e_s+yW1}Q4LU`%H`@gQf# znMQY-WW||Bce-Td8FzBz`+G_ZPNo82D+Ayj4{AkrOaWac_552UT^*5 zEk*XX81H+j09wSGB&_hvM^9j%cRFR>`hz5hTv1Ke@AP?{K5xw*By-hY_hO&-eoA(& zKS+|#i!Yx$b{7hDCbgoCQ1RplP^dvis2!oMS*Sn$`CSu*dL|{i)w;4NXl { + if (YMap.prototype.has.call(this, key, snapshot)) { keys.push(key) } - } + }) } return keys } diff --git a/types/YText.js b/types/YText.js index d5f32ddc..b49f7037 100644 --- a/types/YText.js +++ b/types/YText.js @@ -572,11 +572,12 @@ export class YText extends YArray { * Returns the Delta representation of this YText type. * * @param {import('../protocols/history.js').HistorySnapshot} [snapshot] + * @param {import('../protocols/history.js').HistorySnapshot} [prevSnapshot] * @return {Delta} The Delta representation of this type. * * @public */ - toDelta (snapshot) { + toDelta (snapshot, prevSnapshot) { let ops = [] let currentAttributes = new Map() let str = '' @@ -602,9 +603,24 @@ export class YText extends YArray { } } while (n !== null) { - if (isVisible(n, snapshot)) { + if (isVisible(n, snapshot) || (prevSnapshot !== undefined && isVisible(n, prevSnapshot))) { switch (n.constructor) { case ItemString: + const cur = currentAttributes.get('ychange') + if (snapshot !== undefined && !isVisible(n, snapshot)) { + if (cur === undefined || cur.user !== n._id.user || cur.state !== 'removed') { + packStr() + currentAttributes.set('ychange', { user: n._id.user, state: 'removed' }) + } + } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { + if (cur === undefined || cur.user !== n._id.user || cur.state !== 'added') { + packStr() + currentAttributes.set('ychange', { user: n._id.user, state: 'added' }) + } + } else if (cur !== undefined) { + packStr() + currentAttributes.delete('ychange') + } str += n._content break case ItemFormat: diff --git a/types/YXmlElement.js b/types/YXmlElement.js index 72131b9f..8ede1092 100644 --- a/types/YXmlElement.js +++ b/types/YXmlElement.js @@ -326,9 +326,9 @@ export class YXmlElement extends YXmlFragment { } } } else { - for (let key in this._map) { - return YMap.prototype.get.call(this, key, snapshot) - } + YMap.prototype.keys.call(this, snapshot).forEach(key => { + obj[key] = YMap.prototype.get.call(this, key, snapshot) + }) } return obj } diff --git a/utils/DeleteStore.js b/utils/DeleteStore.js index 9b44bc2f..83fa657b 100644 --- a/utils/DeleteStore.js +++ b/utils/DeleteStore.js @@ -92,7 +92,6 @@ export class DeleteStore extends Tree { } } - /** * Stringifies a message-encoded Delete Set. * diff --git a/utils/snapshot.js b/utils/snapshot.js index 08b5c7a6..ce121c7f 100644 --- a/utils/snapshot.js +++ b/utils/snapshot.js @@ -1,8 +1,7 @@ - /** - * + * * @param {Item} item * @param {import("../protocols/history").HistorySnapshot} [snapshot] */ -export const isVisible = (item, snapshot) => snapshot === undefined ? !item._deleted : (snapshot.sm.has(item._id.user) && snapshot.sm.get(item._id.user) > item._id.clock && !snapshot.ds.isDeleted(item._id)) \ No newline at end of file +export const isVisible = (item, snapshot) => snapshot === undefined ? !item._deleted : (snapshot.sm.has(item._id.user) && snapshot.sm.get(item._id.user) > item._id.clock && !snapshot.ds.isDeleted(item._id))