yonote/yjs
14
0
Fork 0
Code Pull Requests 1 Releases Activity

Implement experimental new encoder 🚀

Browse Source
This commit is contained in:
Kevin Jahns
2020-07-12 18:25:45 +02:00
parent e31e968f0d
commit 6c2cf0f769
36 changed files with 1224 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
src/utils/encoding.js
View File

@@ -1,7 +1,8 @@
/**
* @module encoding
*
*/
/*
* We use the first five bits in the info flag for determining the type of the struct.
*
* 0: GC
@@ -16,8 +17,6 @@
import {
findIndexSS,
writeID,
readID,
getState,
createID,
getStateVector,
@@ -25,16 +24,36 @@ import {
writeDeleteSet,
createDeleteSetFromStructStore,
transact,
readItem,
Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line
readItemContent,
UpdateDecoderV1,
UpdateDecoderV2,
UpdateEncoderV1,
UpdateEncoderV2,
DSDecoderV2,
DSEncoderV2,
DSDecoderV1,
DSEncoderV1,
AbstractDSEncoder, AbstractDSDecoder, AbstractUpdateEncoder, AbstractUpdateDecoder, AbstractContent, Doc, Transaction, GC, Item, StructStore, ID // eslint-disable-line
} from '../internals.js'
import * as encoding from 'lib0/encoding.js'
import * as decoding from 'lib0/decoding.js'
import * as binary from 'lib0/binary.js'
export let DefaultDSEncoder = DSEncoderV1
export let DefaultDSDecoder = DSDecoderV1
export let DefaultUpdateEncoder = UpdateEncoderV1
export let DefaultUpdateDecoder = UpdateDecoderV1
export const useV2Encoding = () => {
DefaultDSEncoder = DSEncoderV2
DefaultDSDecoder = DSDecoderV2
DefaultUpdateEncoder = UpdateEncoderV2
DefaultUpdateDecoder = UpdateDecoderV2
}
/**
* @param {encoding.Encoder} encoder
* @param {AbstractUpdateEncoder} encoder
* @param {Array<GC|Item>} structs All structs by `client`
* @param {number} client
* @param {number} clock write structs starting with `ID(client,clock)`
@@ -45,8 +64,9 @@ const writeStructs = (encoder, structs, client, clock) => {
// write first id
const startNewStructs = findIndexSS(structs, clock)
// write # encoded structs
encoding.writeVarUint(encoder, structs.length - startNewStructs)
writeID(encoder, createID(client, clock))
encoding.writeVarUint(encoder.restEncoder, structs.length - startNewStructs)
encoder.writeClient(client)
encoding.writeVarUint(encoder.restEncoder, clock)
const firstStruct = structs[startNewStructs]
// write first struct with an offset
firstStruct.write(encoder, clock - firstStruct.id.clock)
@@ -56,7 +76,7 @@ const writeStructs = (encoder, structs, client, clock) => {
}
/**
* @param {encoding.Encoder} encoder
* @param {AbstractUpdateEncoder} encoder
* @param {StructStore} store
* @param {Map<number,number>} _sm
*
@@ -78,7 +98,7 @@ export const writeClientsStructs = (encoder, store, _sm) => {
}
})
// write # states that were updated
encoding.writeVarUint(encoder, sm.size)
encoding.writeVarUint(encoder.restEncoder, sm.size)
// Write items with higher client ids first
// This heavily improves the conflict algorithm.
Array.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => {
@@ -88,7 +108,7 @@ export const writeClientsStructs = (encoder, store, _sm) => {
}
/**
* @param {decoding.Decoder} decoder The decoder object to read data from.
* @param {AbstractUpdateDecoder} decoder The decoder object to read data from.
* @param {Map<number,Array<GC|Item>>} clientRefs
* @param {Doc} doc
* @return {Map<number,Array<GC|Item>>}
@@ -97,21 +117,52 @@ export const writeClientsStructs = (encoder, store, _sm) => {
* @function
*/
export const readClientsStructRefs = (decoder, clientRefs, doc) => {
const numOfStateUpdates = decoding.readVarUint(decoder)
const numOfStateUpdates = decoding.readVarUint(decoder.restDecoder)
for (let i = 0; i < numOfStateUpdates; i++) {
const numberOfStructs = decoding.readVarUint(decoder)
const numberOfStructs = decoding.readVarUint(decoder.restDecoder)
/**
* @type {Array<GC|Item>}
*/
const refs = []
let { client, clock } = readID(decoder)
let info, struct
const client = decoder.readClient()
let clock = decoding.readVarUint(decoder.restDecoder)
clientRefs.set(client, refs)
for (let i = 0; i < numberOfStructs; i++) {
info = decoding.readUint8(decoder)
struct = (binary.BITS5 & info) === 0 ? new GC(createID(client, clock), decoding.readVarUint(decoder)) : readItem(decoder, createID(client, clock), info, doc)
refs.push(struct)
clock += struct.length
const info = decoder.readInfo()
if ((binary.BITS5 & info) !== 0) {
/**
* The item that was originally to the left of this item.
* @type {ID | null}
*/
const origin = (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null
/**
* The item that was originally to the right of this item.
* @type {ID | null}
*/
const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null
const canCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
const hasParentYKey = canCopyParentInfo ? decoder.readParentInfo() : false
/**
* If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
* and we read the next string as parentYKey.
* It indicates how we store/retrieve parent from `y.share`
* @type {string|null}
*/
const parentYKey = canCopyParentInfo && hasParentYKey ? decoder.readString() : null
const struct = new Item(
createID(client, clock), null, origin, null, rightOrigin,
canCopyParentInfo && !hasParentYKey ? decoder.readLeftID() : (parentYKey ? doc.get(parentYKey) : null), // parent
canCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub
/** @type {AbstractContent} */ (readItemContent(decoder, info)) // item content
)
refs.push(struct)
clock += struct.length
} else {
const len = decoder.readLen()
refs.push(new GC(createID(client, clock), len))
clock += len
}
}
}
return clientRefs
@@ -222,7 +273,7 @@ export const tryResumePendingDeleteReaders = (transaction, store) => {
}
/**
* @param {encoding.Encoder} encoder
* @param {AbstractUpdateEncoder} encoder
* @param {Transaction} transaction
*
* @private
@@ -275,7 +326,7 @@ const cleanupPendingStructs = pendingClientsStructRefs => {
*
* This is called when data is received from a remote peer.
*
* @param {decoding.Decoder} decoder The decoder object to read data from.
* @param {AbstractUpdateDecoder} decoder The decoder object to read data from.
* @param {Transaction} transaction
* @param {StructStore} store
*
@@ -299,15 +350,46 @@ export const readStructs = (decoder, transaction, store) => {
* @param {decoding.Decoder} decoder
* @param {Doc} ydoc
* @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
* @param {AbstractUpdateDecoder} [structDecoder]
*
* @function
*/
export const readUpdate = (decoder, ydoc, transactionOrigin) =>
export const readUpdateV2 = (decoder, ydoc, transactionOrigin, structDecoder = new UpdateDecoderV2(decoder)) =>
transact(ydoc, transaction => {
readStructs(decoder, transaction, ydoc.store)
readAndApplyDeleteSet(decoder, transaction, ydoc.store)
readStructs(structDecoder, transaction, ydoc.store)
readAndApplyDeleteSet(structDecoder, transaction, ydoc.store)
}, transactionOrigin, false)
/**
* Read and apply a document update.
*
* This function has the same effect as `applyUpdate` but accepts an decoder.
*
* @param {decoding.Decoder} decoder
* @param {Doc} ydoc
* @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
*
* @function
*/
export const readUpdate = (decoder, ydoc, transactionOrigin) => readUpdateV2(decoder, ydoc, transactionOrigin, new DefaultUpdateDecoder(decoder))
/**
* Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`.
*
* This function has the same effect as `readUpdate` but accepts an Uint8Array instead of a Decoder.
*
* @param {Doc} ydoc
* @param {Uint8Array} update
* @param {any} [transactionOrigin] This will be stored on `transaction.origin` and `.on('update', (update, origin))`
* @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder]
*
* @function
*/
export const applyUpdateV2 = (ydoc, update, transactionOrigin, YDecoder = UpdateDecoderV2) => {
const decoder = decoding.createDecoder(update)
readUpdateV2(decoder, ydoc, transactionOrigin, new YDecoder(decoder))
}
/**
* Apply a document update created by, for example, `y.on('update', update => ..)` or `update = encodeStateAsUpdate()`.
*
@@ -319,14 +401,13 @@ export const readUpdate = (decoder, ydoc, transactionOrigin) =>
*
* @function
*/
export const applyUpdate = (ydoc, update, transactionOrigin) =>
readUpdate(decoding.createDecoder(update), ydoc, transactionOrigin)
export const applyUpdate = (ydoc, update, transactionOrigin) => applyUpdateV2(ydoc, update, transactionOrigin, DefaultUpdateDecoder)
/**
* Write all the document as a single update message. If you specify the state of the remote client (`targetStateVector`) it will
* only write the operations that are missing.
*
* @param {encoding.Encoder} encoder
* @param {AbstractUpdateEncoder} encoder
* @param {Doc} doc
* @param {Map<number,number>} [targetStateVector] The state of the target that receives the update. Leave empty to write all known structs
*
@@ -345,31 +426,45 @@ export const writeStateAsUpdate = (encoder, doc, targetStateVector = new Map())
*
* @param {Doc} doc
* @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs
* @param {AbstractUpdateEncoder} [encoder]
* @return {Uint8Array}
*
* @function
*/
export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => {
const encoder = encoding.createEncoder()
export const encodeStateAsUpdateV2 = (doc, encodedTargetStateVector, encoder = new UpdateEncoderV2()) => {
const targetStateVector = encodedTargetStateVector == null ? new Map() : decodeStateVector(encodedTargetStateVector)
writeStateAsUpdate(encoder, doc, targetStateVector)
return encoding.toUint8Array(encoder)
return encoder.toUint8Array()
}
/**
* Write all the document as a single update message that can be applied on the remote document. If you specify the state of the remote client (`targetState`) it will
* only write the operations that are missing.
*
* Use `writeStateAsUpdate` instead if you are working with lib0/encoding.js#Encoder
*
* @param {Doc} doc
* @param {Uint8Array} [encodedTargetStateVector] The state of the target that receives the update. Leave empty to write all known structs
* @return {Uint8Array}
*
* @function
*/
export const encodeStateAsUpdate = (doc, encodedTargetStateVector) => encodeStateAsUpdateV2(doc, encodedTargetStateVector, new DefaultUpdateEncoder())
/**
* Read state vector from Decoder and return as Map
*
* @param {decoding.Decoder} decoder
* @param {AbstractDSDecoder} decoder
* @return {Map<number,number>} Maps `client` to the number next expected `clock` from that client.
*
* @function
*/
export const readStateVector = decoder => {
const ss = new Map()
const ssLength = decoding.readVarUint(decoder)
const ssLength = decoding.readVarUint(decoder.restDecoder)
for (let i = 0; i < ssLength; i++) {
const client = decoding.readVarUint(decoder)
const clock = decoding.readVarUint(decoder)
const client = decoding.readVarUint(decoder.restDecoder)
const clock = decoding.readVarUint(decoder.restDecoder)
ss.set(client, clock)
}
return ss
@@ -383,28 +478,34 @@ export const readStateVector = decoder => {
*
* @function
*/
export const decodeStateVector = decodedState => readStateVector(decoding.createDecoder(decodedState))
export const decodeStateVectorV2 = decodedState => readStateVector(new DSDecoderV2(decoding.createDecoder(decodedState)))
/**
* Write State Vector to `lib0/encoding.js#Encoder`.
* Read decodedState and return State as Map.
*
* @param {encoding.Encoder} encoder
* @param {Uint8Array} decodedState
* @return {Map<number,number>} Maps `client` to the number next expected `clock` from that client.
*
* @function
*/
export const decodeStateVector = decodedState => readStateVector(new DefaultDSDecoder(decoding.createDecoder(decodedState)))
/**
* @param {AbstractDSEncoder} encoder
* @param {Map<number,number>} sv
* @function
*/
export const writeStateVector = (encoder, sv) => {
encoding.writeVarUint(encoder, sv.size)
encoding.writeVarUint(encoder.restEncoder, sv.size)
sv.forEach((clock, client) => {
encoding.writeVarUint(encoder, client)
encoding.writeVarUint(encoder, clock)
encoding.writeVarUint(encoder.restEncoder, client) // @todo use a special client decoder that is based on mapping
encoding.writeVarUint(encoder.restEncoder, clock)
})
return encoder
}
/**
* Write State Vector to `lib0/encoding.js#Encoder`.
*
* @param {encoding.Encoder} encoder
* @param {AbstractDSEncoder} encoder
* @param {Doc} doc
*
* @function
@@ -415,12 +516,22 @@ export const writeDocumentStateVector = (encoder, doc) => writeStateVector(encod
* Encode State as Uint8Array.
*
* @param {Doc} doc
* @param {AbstractDSEncoder} [encoder]
* @return {Uint8Array}
*
* @function
*/
export const encodeStateVector = doc => {
const encoder = encoding.createEncoder()
export const encodeStateVectorV2 = (doc, encoder = new DSEncoderV2()) => {
writeDocumentStateVector(encoder, doc)
return encoding.toUint8Array(encoder)
return encoder.toUint8Array()
}
/**
* Encode State as Uint8Array.
*
* @param {Doc} doc
* @return {Uint8Array}
*
* @function
*/
export const encodeStateVector = doc => encodeStateVectorV2(doc, new DefaultDSEncoder())