import * as binary from 'lib0/binary.js' import * as decoding from 'lib0/decoding.js' import * as encoding from 'lib0/encoding.js' import * as logging from 'lib0/logging.js' import { createID, readItemContent, readDeleteSet, writeDeleteSet, Skip, mergeDeleteSets, DSEncoderV1, DSEncoderV2, Item, GC, AbstractUpdateDecoder, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2 // eslint-disable-line } from '../internals.js' /** * @param {AbstractUpdateDecoder} decoder */ function * lazyStructReaderGenerator (decoder) { const numOfStateUpdates = decoding.readVarUint(decoder.restDecoder) for (let i = 0; i < numOfStateUpdates; i++) { const numberOfStructs = decoding.readVarUint(decoder.restDecoder) const client = decoder.readClient() let clock = decoding.readVarUint(decoder.restDecoder) for (let i = 0; i < numberOfStructs; i++) { const info = decoder.readInfo() // @todo use switch instead of ifs if (info === 10) { const len = decoder.readLen() yield new Skip(createID(client, clock), len) clock += len } else if ((binary.BITS5 & info) !== 0) { const cantCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0 // If parent = null and neither left nor right are defined, then we know that `parent` is child of `y` // and we read the next string as parentYKey. // It indicates how we store/retrieve parent from `y.share` // @type {string|null} const struct = new Item( createID(client, clock), null, // left (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null, // origin null, // right (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null, // right origin // @ts-ignore Force writing a string here. cantCopyParentInfo ? (decoder.readParentInfo() ? decoder.readString() : decoder.readLeftID()) : null, // parent cantCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub readItemContent(decoder, info) // item content ) yield struct clock += struct.length } else { const len = decoder.readLen() yield new GC(createID(client, clock), len) clock += len } } } } export class LazyStructReader { /** * @param {AbstractUpdateDecoder} decoder */ constructor (decoder) { this.gen = lazyStructReaderGenerator(decoder) /** * @type {null | Item | GC} */ this.curr = null this.done = false this.next() } /** * @return {Item | GC | null} */ next () { // ignore "Skip" structs do { this.curr = this.gen.next().value || null } while (this.curr !== null && this.curr.constructor === Skip) return this.curr } } /** * @param {Uint8Array} update * */ export const logUpdate = update => logUpdateV2(update, UpdateDecoderV1) /** * @param {Uint8Array} update * @param {typeof UpdateDecoderV2 | typeof UpdateDecoderV1} [YDecoder] * */ export const logUpdateV2 = (update, YDecoder = UpdateDecoderV2) => { const structs = [] const updateDecoder = new YDecoder(decoding.createDecoder(update)) const lazyDecoder = new LazyStructReader(updateDecoder) for (let curr = lazyDecoder.curr; curr !== null; curr = lazyDecoder.next()) { structs.push(curr) } logging.print('Structs: ', structs) const ds = readDeleteSet(updateDecoder) logging.print('DeleteSet: ', ds) } export class LazyStructWriter { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder */ constructor (encoder) { this.currClient = 0 this.startClock = 0 this.written = 0 this.encoder = encoder /** * We want to write operations lazily, but also we need to know beforehand how many operations we want to write for each client. * * This kind of meta-information (#clients, #structs-per-client-written) is written to the restEncoder. * * We fragment the restEncoder and store a slice of it per-client until we know how many clients there are. * When we flush (toUint8Array) we write the restEncoder using the fragments and the meta-information. * * @type {Array<{ written: number, restEncoder: Uint8Array }>} */ this.clientStructs = [] } } /** * @param {Array} updates * @return {Uint8Array} */ export const mergeUpdates = updates => mergeUpdatesV2(updates, UpdateDecoderV1, UpdateEncoderV1) /** * @param {Uint8Array} update * @param {typeof DSEncoderV1 | typeof DSEncoderV2} YEncoder * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} YDecoder * @return {Uint8Array} */ export const encodeStateVectorFromUpdateV2 = (update, YEncoder = DSEncoderV2, YDecoder = UpdateDecoderV2) => { const encoder = new YEncoder() const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update))) let curr = updateDecoder.curr if (curr !== null) { let size = 1 let currClient = curr.id.client let currClock = curr.id.clock for (; curr !== null; curr = updateDecoder.next()) { if (currClient !== curr.id.client) { size++ // We found a new client // write what we have to the encoder encoding.writeVarUint(encoder.restEncoder, currClient) encoding.writeVarUint(encoder.restEncoder, currClock) currClient = curr.id.client } currClock = curr.id.clock + curr.length } // write what we have encoding.writeVarUint(encoder.restEncoder, currClient) encoding.writeVarUint(encoder.restEncoder, currClock) // prepend the size of the state vector const enc = encoding.createEncoder() encoding.writeVarUint(enc, size) encoding.writeBinaryEncoder(enc, encoder.restEncoder) encoder.restEncoder = enc return encoder.toUint8Array() } else { encoding.writeVarUint(encoder.restEncoder, 0) return encoder.toUint8Array() } } /** * @param {Uint8Array} update * @return {Uint8Array} */ export const encodeStateVectorFromUpdate = update => encodeStateVectorFromUpdateV2(update, DSEncoderV1, UpdateDecoderV2) /** * This method is intended to slice any kind of struct and retrieve the right part. * It does not handle side-effects, so it should only be used by the lazy-encoder. * * @param {Item | GC | Skip} left * @param {number} diff * @return {Item | GC} */ const sliceStruct = (left, diff) => { if (left.constructor === GC) { const { client, clock } = left.id return new GC(createID(client, clock + diff), left.length - diff) } else if (left.constructor === Skip) { const { client, clock } = left.id return new Skip(createID(client, clock + diff), left.length - diff) } else { const leftItem = /** @type {Item} */ (left) const { client, clock } = leftItem.id return new Item( createID(client, clock + diff), null, createID(client, clock + diff - 1), null, leftItem.rightOrigin, leftItem.parent, leftItem.parentSub, leftItem.content.splice(diff) ) } } /** * * This function works similarly to `readUpdateV2`. * * @param {Array} updates * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} [YDecoder] * @param {typeof UpdateEncoderV1 | typeof UpdateEncoderV2} [YEncoder] * @return {Uint8Array} */ export const mergeUpdatesV2 = (updates, YDecoder = UpdateDecoderV2, YEncoder = UpdateEncoderV2) => { const updateDecoders = updates.map(update => new YDecoder(decoding.createDecoder(update))) let lazyStructDecoders = updateDecoders.map(decoder => new LazyStructReader(decoder)) /** * @todo we don't need offset because we always slice before * @type {null | { struct: Item | GC | Skip, offset: number }} */ let currWrite = null const updateEncoder = new YEncoder() // write structs lazily const lazyStructEncoder = new LazyStructWriter(updateEncoder) // Note: We need to ensure that all lazyStructDecoders are fully consumed // Note: Should merge document updates whenever possible - even from different updates // Note: Should handle that some operations cannot be applied yet () while (true) { // Write higher clients first ⇒ sort by clientID & clock and remove decoders without content lazyStructDecoders = lazyStructDecoders.filter(dec => dec.curr !== null) lazyStructDecoders.sort( /** @type {function(any,any):number} */ (dec1, dec2) => { if (dec1.curr.id.client === dec2.curr.id.client) { const clockDiff = dec1.curr.id.clock - dec2.curr.id.clock if (clockDiff === 0) { return dec1.curr.constructor === dec2.curr.constructor ? 0 : ( dec1.curr.constructor === Skip ? 1 : -1 ) } else { return clockDiff } } else { return dec2.curr.id.client - dec1.curr.id.client } } ) if (lazyStructDecoders.length === 0) { break } const currDecoder = lazyStructDecoders[0] // write from currDecoder until the next operation is from another client or if filler-struct // then we need to reorder the decoders and find the next operation to write const firstClient = /** @type {Item | GC} */ (currDecoder.curr).id.client if (currWrite !== null) { let curr = /** @type {Item | GC | null} */ (currDecoder.curr) // iterate until we find something that we haven't written already // remember: first the high client-ids are written while (curr !== null && curr.id.clock + curr.length <= currWrite.struct.id.clock + currWrite.struct.length && curr.id.client >= currWrite.struct.id.client) { curr = currDecoder.next() } if (curr === null || curr.id.client !== firstClient) { continue } if (firstClient !== currWrite.struct.id.client) { writeStructToLazyStructWriter(lazyStructEncoder, currWrite.struct, currWrite.offset) currWrite = { struct: curr, offset: 0 } currDecoder.next() } else { if (currWrite.struct.id.clock + currWrite.struct.length < curr.id.clock) { // @todo write currStruct & set currStruct = Skip(clock = currStruct.id.clock + currStruct.length, length = curr.id.clock - self.clock) if (currWrite.struct.constructor === Skip) { // extend existing skip currWrite.struct.length = curr.id.clock + curr.length - currWrite.struct.id.clock } else { writeStructToLazyStructWriter(lazyStructEncoder, currWrite.struct, currWrite.offset) const diff = curr.id.clock - currWrite.struct.id.clock - currWrite.struct.length /** * @type {Skip} */ const struct = new Skip(createID(firstClient, currWrite.struct.id.clock + currWrite.struct.length), diff) currWrite = { struct, offset: 0 } } } else { // if (currWrite.struct.id.clock + currWrite.struct.length >= curr.id.clock) { const diff = currWrite.struct.id.clock + currWrite.struct.length - curr.id.clock if (diff > 0) { if (currWrite.struct.constructor === Skip) { // prefer to slice Skip because the other struct might contain more information currWrite.struct.length -= diff } else { curr = sliceStruct(curr, diff) } } if (!currWrite.struct.mergeWith(/** @type {any} */ (curr))) { writeStructToLazyStructWriter(lazyStructEncoder, currWrite.struct, currWrite.offset) currWrite = { struct: curr, offset: 0 } currDecoder.next() } } } } else { currWrite = { struct: /** @type {Item | GC} */ (currDecoder.curr), offset: 0 } currDecoder.next() } for ( let next = currDecoder.curr; next !== null && next.id.client === firstClient && next.id.clock === currWrite.struct.id.clock + currWrite.struct.length && next.constructor !== Skip; next = currDecoder.next() ) { writeStructToLazyStructWriter(lazyStructEncoder, currWrite.struct, currWrite.offset) currWrite = { struct: next, offset: 0 } } } if (currWrite !== null) { writeStructToLazyStructWriter(lazyStructEncoder, currWrite.struct, currWrite.offset) currWrite = null } finishLazyStructWriting(lazyStructEncoder) const dss = updateDecoders.map(decoder => readDeleteSet(decoder)) const ds = mergeDeleteSets(dss) writeDeleteSet(updateEncoder, ds) return updateEncoder.toUint8Array() } /** * @param {Uint8Array} update * @param {Uint8Array} sv */ export const diffUpdate = (update, sv) => { return update } /** * @param {LazyStructWriter} lazyWriter */ const flushLazyStructWriter = lazyWriter => { if (lazyWriter.written > 0) { lazyWriter.clientStructs.push({ written: lazyWriter.written, restEncoder: encoding.toUint8Array(lazyWriter.encoder.restEncoder) }) lazyWriter.encoder.restEncoder = encoding.createEncoder() lazyWriter.written = 0 } } /** * @param {LazyStructWriter} lazyWriter * @param {Item | GC} struct * @param {number} offset */ const writeStructToLazyStructWriter = (lazyWriter, struct, offset) => { // flush curr if we start another client if (lazyWriter.written > 0 && lazyWriter.currClient !== struct.id.client) { flushLazyStructWriter(lazyWriter) } if (lazyWriter.written === 0) { lazyWriter.currClient = struct.id.client // write next client lazyWriter.encoder.writeClient(struct.id.client) // write startClock encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock) } struct.write(lazyWriter.encoder, offset) lazyWriter.written++ } /** * Call this function when we collected all parts and want to * put all the parts together. After calling this method, * you can continue using the UpdateEncoder. * * @param {LazyStructWriter} lazyWriter */ const finishLazyStructWriting = (lazyWriter) => { flushLazyStructWriter(lazyWriter) // this is a fresh encoder because we called flushCurr const restEncoder = lazyWriter.encoder.restEncoder /** * Now we put all the fragments together. * This works similarly to `writeClientsStructs` */ // write # states that were updated - i.e. the clients encoding.writeVarUint(restEncoder, lazyWriter.clientStructs.length) for (let i = 0; i < lazyWriter.clientStructs.length; i++) { const partStructs = lazyWriter.clientStructs[i] /** * Works similarly to `writeStructs` */ // write # encoded structs encoding.writeVarUint(restEncoder, partStructs.written) // write the rest of the fragment encoding.writeUint8Array(restEncoder, partStructs.restEncoder) } }