yonote/yjs
23
0
Fork 0
Code Pull Requests 1 Releases Activity

reimplement persistence approach

Browse Source
This commit is contained in:
Kevin Jahns 2017-12-24 03:18:00 +01:00
parent 08f37a86e3
commit f2debc150c
27 changed files with 880 additions and 1275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/html-editor/index.html
View File

@ -1,10 +1,7 @@
<!DOCTYPE html>
<html>
</head>
<!-- jquery is not required for y-xml. It is just here for convenience, and to test batch operations. -->
<script src="https://code.jquery.com/jquery-3.2.1.min.js"></script>
<script src="../yjs-dist.js"></script>
<script src="./canvasjs.min.js"></script>
<script src="./index.js"></script>
</head>
<body contenteditable="true">

63
examples/html-editor/index.js
View File

@ -1,67 +1,10 @@
/* global Y, HTMLElement, customElements, CanvasJS */
/* global Y */
window.onload = function () {
window.yXmlType.bindToDom(document.body)
let mt = document.createElement('magic-table')
mt.innerHTML = '<table><tr><th>Amount</th></tr><tr><td>1</td></tr><tr><td>1</td></tr></table>'
document.body.append(mt)
}
class MagicTable extends HTMLElement {
constructor () {
super()
this.createShadowRoot()
}
get _yjsHook () {
return 'magic-table'
}
showTable () {
this.shadowRoot.innerHTML = ''
this.shadowRoot.append(document.createElement('content'))
}
showDiagram () {
let dataPoints = []
this.querySelectorAll('td').forEach(td => {
let number = Number(td.textContent)
dataPoints.push({
x: (dataPoints.length + 1) * 10,
y: number,
label: '<magic-table> content'
})
})
this.shadowRoot.innerHTML = ''
var chart = new CanvasJS.Chart(this.shadowRoot,
{
title: {
text: 'Bar chart'
},
data: [
{
type: 'bar',
dataPoints: dataPoints
}
]
})
chart.render()
// this.shadowRoot.innerHTML = '<p>dtrn</p>'
}
}
customElements.define('magic-table', MagicTable)
Y.XmlHook.addHook('magic-table', {
fillType: function (dom, type) {
type.set('table', new Y.XmlElement(dom.querySelector('table')))
},
createDom: function (type) {
const table = type.get('table').getDom()
const dom = document.createElement('magic-table')
dom.insertBefore(table, null)
return dom
}
})
let persistence = new Y.IndexedDBPersistence()
// initialize a shared object. This function call returns a promise!
let y = new Y({
@ -71,7 +14,7 @@ let y = new Y({
room: 'html-editor-example6'
// maxBufferLength: 100
}
})
}, persistence)
window.yXml = y
window.yXmlType = y.define('xml', Y.XmlFragment)
window.undoManager = new Y.utils.UndoManager(window.yXmlType, {

663
examples/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
examples/package.json
View File

@ -9,12 +9,15 @@
"author": "Kevin Jahns",
"license": "MIT",
"dependencies": {
"monaco-editor": "^0.8.3"
"monaco-editor": "^0.8.3",
"rollup": "^0.52.3"
},
"devDependencies": {
"standard": "^10.0.2"
},
"standard": {
"ignore": ["bower_components"]
"ignore": [
"bower_components"
]
}
}

2
examples/rollup.config.js
View File

@ -4,7 +4,7 @@ import commonjs from 'rollup-plugin-commonjs'
var pkg = require('./package.json')
export default {
input: 'yjs-dist.esm',
input: 'yjs-dist.mjs',
name: 'Y',
output: {
file: 'yjs-dist.js',

2
examples/yjs-dist.esm → examples/yjs-dist.mjs
View File

@ -1,7 +1,9 @@
import Y from '../src/Y.js'
import yWebsocketsClient from '../../y-websockets-client/src/y-websockets-client.js'
import IndexedDBPersistence from '../../y-indexeddb/src/y-indexeddb.js'
Y.extend(yWebsocketsClient)
Y.IndexedDBPersistence = IndexedDBPersistence
export default Y

1148
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
package.json
View File

@ -65,9 +65,6 @@
"tag-dist-files": "^0.1.6"
},
"dependencies": {
"debug": "^2.6.8",
"fast-diff": "^1.1.2",
"utf-8": "^1.0.0",
"utf8": "^2.1.2"
"debug": "^2.6.8"
}
}

2
rollup.test.js
View File

@ -3,7 +3,7 @@ import commonjs from 'rollup-plugin-commonjs'
import multiEntry from 'rollup-plugin-multi-entry'
export default {
input: 'test/y-xml.tests.js',
input: 'test/index.js',
name: 'y-tests',
sourcemap: true,
output: {

4
src/Binary/Decoder.js
View File

@ -1,4 +1,3 @@
import utf8 from 'utf-8'
import ID from '../Util/ID.js'
import { default as RootID, RootFakeUserID } from '../Util/RootID.js'
@ -91,7 +90,8 @@ export default class BinaryDecoder {
for (let i = 0; i < len; i++) {
bytes[i] = this.uint8arr[this.pos++]
}
return utf8.getStringFromBytes(bytes)
let encodedString = String.fromCodePoint(...bytes)
return decodeURIComponent(escape(encodedString))
}
/**
* Look ahead and read varString without incrementing position

4
src/Binary/Encoder.js
View File

@ -1,4 +1,3 @@
import utf8 from 'utf-8'
import { RootFakeUserID } from '../Util/RootID.js'
const bits7 = 0b1111111
@ -62,7 +61,8 @@ export default class BinaryEncoder {
}
writeVarString (str) {
let bytes = utf8.setBytesFromString(str)
let encodedString = unescape(encodeURIComponent(str))
let bytes = encodedString.split('').map(c => c.codePointAt())
let len = bytes.length
this.writeVarUint(len)
for (let i = 0; i < len; i++) {

2
src/Connector.js
View File

@ -268,7 +268,7 @@ export default class AbstractConnector {
if (messageType === 'sync step 2' && senderConn.auth === 'write') {
readSyncStep2(decoder, encoder, y, senderConn, sender)
} else if (messageType === 'update' && (skipAuth || senderConn.auth === 'write')) {
integrateRemoteStructs(decoder, encoder, y, senderConn, sender)
integrateRemoteStructs(y, decoder)
} else {
throw new Error('Unable to receive message')
}

16
src/MessageHandler/binaryEncode.js Normal file
View File

@ -0,0 +1,16 @@
import { writeStructs } from './syncStep1.js'
import { integrateRemoteStructs } from './integrateRemoteStructs.js'
import { readDeleteSet, writeDeleteSet } from './deleteSet.js'
import BinaryEncoder from '../Binary/Encoder.js'
export function fromBinary (y, decoder) {
integrateRemoteStructs(y, decoder)
readDeleteSet(y, decoder)
}
export function toBinary (y) {
let encoder = new BinaryEncoder()
writeStructs(y, encoder, new Map())
writeDeleteSet(y, encoder)
return encoder
}

2
src/MessageHandler/integrateRemoteStructs.js
View File

@ -65,7 +65,7 @@ export function stringifyStructs (y, decoder, strBuilder) {
}
}
export function integrateRemoteStructs (decoder, encoder, y) {
export function integrateRemoteStructs (y, decoder) {
const len = decoder.readUint32()
for (let i = 0; i < len; i++) {
let reference = decoder.readVarUint()

4
src/MessageHandler/syncStep1.js
View File

@ -30,7 +30,7 @@ export function sendSyncStep1 (connector, syncUser) {
connector.send(syncUser, encoder.createBuffer())
}
export default function writeStructs (encoder, decoder, y, ss) {
export function writeStructs (y, encoder, ss) {
const lenPos = encoder.pos
encoder.writeUint32(0)
let len = 0
@ -60,7 +60,7 @@ export function readSyncStep1 (decoder, encoder, y, senderConn, sender) {
encoder.writeVarString('sync step 2')
encoder.writeVarString(y.connector.authInfo || '')
const ss = readStateSet(decoder)
writeStructs(encoder, decoder, y, ss)
writeStructs(y, encoder, ss)
writeDeleteSet(y, encoder)
y.connector.send(senderConn.uid, encoder.createBuffer())
senderConn.receivedSyncStep2 = true

2
src/MessageHandler/syncStep2.js
View File

@ -22,7 +22,7 @@ export function stringifySyncStep2 (y, decoder, strBuilder) {
}
export function readSyncStep2 (decoder, encoder, y, senderConn, sender) {
integrateRemoteStructs(decoder, encoder, y)
integrateRemoteStructs(y, decoder)
readDeleteSet(y, decoder)
y.connector._setSyncedWith(sender)
}

110
src/Persistence.js
View File

@ -1,47 +1,81 @@
// import BinaryEncoder from './Binary/Encoder.js'
import BinaryEncoder from './Binary/Encoder.js'
import BinaryDecoder from './Binary/Decoder.js'
import { toBinary, fromBinary } from './MessageHandler/binaryEncode.js'
import { integrateRemoteStructs } from './MessageHandler/integrateRemoteStructs.js'
export default function extendPersistence (Y) {
class AbstractPersistence {
constructor (y, opts) {
this.y = y
this.opts = opts
this.saveOperationsBuffer = []
this.log = Y.debug('y:persistence')
}
function getFreshCnf () {
let buffer = new BinaryEncoder()
buffer.writeUint32(0)
return {
len: 0,
buffer
}
}
saveToMessageQueue (binary) {
this.log('Room %s: Save message to message queue', this.y.options.connector.room)
}
export default class AbstractPersistence {
constructor (opts) {
this.opts = opts
this.ys = new Map()
}
saveOperations (ops) {
ops = ops.map(function (op) {
return Y.Struct[op.struct].encode(op)
})
/*
const saveOperations = () => {
if (this.saveOperationsBuffer.length > 0) {
let encoder = new BinaryEncoder()
encoder.writeVarString(this.opts.room)
encoder.writeVarString('update')
let ops = this.saveOperationsBuffer
this.saveOperationsBuffer = []
let length = ops.length
encoder.writeUint32(length)
for (var i = 0; i < length; i++) {
let op = ops[i]
Y.Struct[op.struct].binaryEncode(encoder, op)
_init (y) {
let cnf = this.ys.get(y)
if (cnf === undefined) {
cnf = getFreshCnf()
this.ys.set(y, cnf)
this.init(y)
y.on('afterTransaction', () => {
let cnf = this.ys.get(y)
if (cnf.len > 0) {
cnf.buffer.setUint32(0, cnf.len)
this.saveUpdate(y, cnf.buffer.createBuffer())
let _cnf = getFreshCnf()
for (let key in _cnf) {
cnf[key] = _cnf[key]
}
this.saveToMessageQueue(encoder.createBuffer())
}
}
*/
if (this.saveOperationsBuffer.length === 0) {
this.saveOperationsBuffer = ops
} else {
this.saveOperationsBuffer = this.saveOperationsBuffer.concat(ops)
}
})
}
return this.retrieve(y).then(function () {
return Promise.resolve(cnf)
})
}
deinit (y) {
this.ys.delete(y)
}
/* overwrite */
saveUpdate (buffer) {
}
/**
* Save struct to update buffer.
* saveUpdate is called when transaction ends
*/
saveStruct (y, struct) {
let cnf = this.ys.get(y)
if (cnf !== undefined) {
struct._toBinary(cnf.buffer)
cnf.len++
}
}
Y.AbstractPersistence = AbstractPersistence
/* overwrite */
retrieve (y, model, updates) {
y.transact(function () {
if (model != null) {
fromBinary(y, new BinaryDecoder(new Uint8Array(model)))
}
if (updates != null) {
for (let i = 0; i < updates.length; i++) {
integrateRemoteStructs(y, new BinaryDecoder(new Uint8Array(updates[i])))
}
}
})
}
/* overwrite */
persist (y) {
return toBinary(y).createBuffer()
}
}

6
src/Struct/Delete.js
View File

@ -7,7 +7,7 @@ import { logID } from '../MessageHandler/messageToString.js'
* TODO: implement getItemCleanStartNode for better performance (only one lookup)
*/
export function deleteItemRange (y, user, clock, range) {
const createDelete = y.connector._forwardAppliedStructs
const createDelete = y.connector !== null && y.connector._forwardAppliedStructs
let item = y.os.getItemCleanStart(new ID(user, clock))
if (item !== null) {
if (!item._deleted) {
@ -70,12 +70,12 @@ export default class Delete {
// from remote
const id = this._targetID
deleteItemRange(y, id.user, id.clock, this._length)
} else {
} else if (y.connector !== null) {
// from local
y.connector.broadcastStruct(this)
}
if (y.persistence !== null) {
y.persistence.saveOperations(this)
y.persistence.saveStruct(y, this)
}
}
_logString () {

4
src/Struct/Item.js
View File

@ -216,11 +216,11 @@ export default class Item {
y.os.put(this)
transactionTypeChanged(y, parent, parentSub)
if (this._id.user !== RootFakeUserID) {
if (y.connector._forwardAppliedStructs || this._id.user === y.userID) {
if (y.connector !== null && (y.connector._forwardAppliedStructs || this._id.user === y.userID)) {
y.connector.broadcastStruct(this)
}
if (y.persistence !== null) {
y.persistence.saveOperations(this)
y.persistence.saveStruct(y, this)
}
}
}

1
src/Type/y-xml/YXmlElement.js
View File

@ -1,4 +1,3 @@
// import diff from 'fast-diff'
import { defaultDomFilter } from './utils.js'
import YMap from '../YMap.js'

18
src/Type/y-xml/YXmlFragment.js
View File

@ -7,7 +7,7 @@ import YArray from '../YArray.js'
import YXmlEvent from './YXmlEvent.js'
import { YXmlText, YXmlHook } from './y-xml'
import { logID } from '../../MessageHandler/messageToString.js'
import diff from 'fast-diff'
import diff from '../../Util/simpleDiff.js'
function domToYXml (parent, doms, _document) {
const types = []
@ -291,19 +291,9 @@ export default class YXmlFragment extends YArray {
}
switch (mutation.type) {
case 'characterData':
var diffs = diff(yxml.toString(), dom.nodeValue)
var pos = 0
for (var i = 0; i < diffs.length; i++) {
var d = diffs[i]
if (d[0] === 0) { // EQUAL
pos += d[1].length
} else if (d[0] === -1) { // DELETE
yxml.delete(pos, d[1].length)
} else { // INSERT
yxml.insert(pos, d[1])
pos += d[1].length
}
}
var change = diff(yxml.toString(), dom.nodeValue)
yxml.delete(change.pos, change.remove)
yxml.insert(change.pos, change.insert)
break
case 'attributes':
if (yxml.constructor === YXmlFragment) {

19
src/Util/simpleDiff.js Normal file
View File

@ -0,0 +1,19 @@
export default function simpleDiff (a, b) {
let left = 0 // number of same characters counting from left
let right = 0 // number of same characters counting from right
while (left < a.length && left < b.length && a[left] === b[left]) {
left++
}
if (left !== a.length || left !== b.length) {
// Only check right if a !== b
while (right + left < a.length && right + left < b.length && a[a.length - right - 1] === b[b.length - right - 1]) {
right++
}
}
return {
pos: left,
remove: a.length - left - right,
insert: b.slice(left, b.length - right)
}
}

28
src/Y.js
View File

@ -22,7 +22,7 @@ import debug from 'debug'
import Transaction from './Transaction.js'
export default class Y extends NamedEventHandler {
constructor (opts) {
constructor (opts, persistence) {
super()
this._opts = opts
this.userID = opts._userID != null ? opts._userID : generateUserID()
@ -30,17 +30,22 @@ export default class Y extends NamedEventHandler {
this.ds = new DeleteStore(this)
this.os = new OperationStore(this)
this.ss = new StateStore(this)
this.connector = new Y[opts.connector.name](this, opts.connector)
if (opts.persistence != null) {
this.persistence = new Y[opts.persistence.name](this, opts.persistence)
this.persistence.retrieveContent()
} else {
this.persistence = null
}
this.connected = true
this._missingStructs = new Map()
this._readyToIntegrate = []
this._transaction = null
this.connector = null
this.connected = false
let initConnection = () => {
this.connector = new Y[opts.connector.name](this, opts.connector)
this.connected = true
}
if (persistence !== undefined) {
this.persistence = persistence
persistence._init(this).then(initConnection)
} else {
this.persistence = null
initConnection()
}
}
_beforeChange () {}
transact (f, remote = false) {
@ -123,12 +128,17 @@ export default class Y extends NamedEventHandler {
}
}
destroy () {
super.destroy()
this.share = null
if (this.connector.destroy != null) {
this.connector.destroy()
} else {
this.connector.disconnect()
}
if (this.persistence !== null) {
this.persistence.deinit(this)
this.persistence = null
}
this.os = null
this.ds = null
this.ss = null

29
test/diff.tests.js Normal file
View File

@ -0,0 +1,29 @@
import { test } from '../node_modules/cutest/cutest.mjs'
import simpleDiff from '../src/Util/simpleDiff.js'
import Chance from 'chance'
function runDiffTest (t, a, b, expected) {
let result = simpleDiff(a, b)
t.compare(result, expected, `Compare "${a}" with "${b}"`)
}
test('diff tests', async function diff1 (t) {
runDiffTest(t, 'abc', 'axc', { pos: 1, remove: 1, insert: 'x' })
runDiffTest(t, 'bc', 'xc', { pos: 0, remove: 1, insert: 'x' })
runDiffTest(t, 'ab', 'ax', { pos: 1, remove: 1, insert: 'x' })
runDiffTest(t, 'b', 'x', { pos: 0, remove: 1, insert: 'x' })
runDiffTest(t, '', 'abc', { pos: 0, remove: 0, insert: 'abc' })
runDiffTest(t, 'abc', 'xyz', { pos: 0, remove: 3, insert: 'xyz' })
runDiffTest(t, 'axz', 'au', { pos: 1, remove: 2, insert: 'u' })
runDiffTest(t, 'ax', 'axy', { pos: 2, remove: 0, insert: 'y' })
})
test('random diff tests', async function randomDiff (t) {
const chance = new Chance(t.getSeed() * 1000000000)
let a = chance.word()
let b = chance.word()
let change = simpleDiff(a, b)
let arr = Array.from(a)
arr.splice(change.pos, change.remove, ...Array.from(change.insert))
t.assert(arr.join('') === b, 'Applying change information is correct')
})

3
test/encode-decode.js → test/encode-decode.tests.js
View File

@ -54,6 +54,9 @@ test('varString', async function varString (t) {
testEncoding(t, writeVarString, readVarString, 'test!')
testEncoding(t, writeVarString, readVarString, '☺☺☺')
testEncoding(t, writeVarString, readVarString, '1234')
testEncoding(t, writeVarString, readVarString, '쾟')
testEncoding(t, writeVarString, readVarString, '龟') // surrogate length 3
testEncoding(t, writeVarString, readVarString, '😝') // surrogate length 4
})
test('varString random', async function varStringRandom (t) {

2
test/index.html
View File

@ -3,6 +3,6 @@
<head>
</head>
<body>
<script type="module" src="./encode-decode.js"></script>
<script type="module" src="./index.js"></script>
</body>
</html>

6
test/index.js Normal file
View File

@ -0,0 +1,6 @@
import './red-black-tree.js'
import './y-array.tests.js'
import './y-map.tests.js'
import './y-xml.tests.js'
import './encode-decode.tests.js'
import './diff.tests.js'