implemented leveldb persistence for websocket server
This commit is contained in:
Kevin Jahns
74
persistence/FilePersistence.js
Normal file
74
persistence/FilePersistence.js
Normal file
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import * as encoding from '../lib/encoding.js'
|
||||
import * as decoding from '../lib/decoding.js'
|
||||
import { createMutex } from '../lib/mutex.js'
|
||||
import { encodeUpdate, encodeStructsDS, decodePersisted } from './decodePersisted.js'
|
||||
|
||||
function createFilePath (persistence, roomName) {
|
||||
// TODO: filename checking!
|
||||
return path.join(persistence.dir, roomName)
|
||||
}
|
||||
|
||||
export class FilePersistence {
|
||||
constructor (dir) {
|
||||
this.dir = dir
|
||||
this._mutex = createMutex()
|
||||
}
|
||||
setRemoteUpdateCounter (roomName, remoteUpdateCounter) {
|
||||
// TODO: implement
|
||||
// nop
|
||||
}
|
||||
saveUpdate (room, y, encodedStructs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this._mutex(() => {
|
||||
const filePath = createFilePath(this, room)
|
||||
const updateMessage = encoding.createEncoder()
|
||||
encodeUpdate(y, encodedStructs, updateMessage)
|
||||
fs.appendFile(filePath, Buffer.from(encoding.toBuffer(updateMessage)), (err) => {
|
||||
if (err !== null) {
|
||||
reject(err)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
}, resolve)
|
||||
})
|
||||
}
|
||||
saveState (roomName, y) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const encoder = encoding.createEncoder()
|
||||
encodeStructsDS(y, encoder)
|
||||
const filePath = createFilePath(this, roomName)
|
||||
fs.writeFile(filePath, Buffer.from(encoding.toBuffer(encoder)), (err) => {
|
||||
if (err !== null) {
|
||||
reject(err)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
readState (roomName, y) {
|
||||
// Check if the file exists in the current directory.
|
||||
return new Promise((resolve, reject) => {
|
||||
const filePath = path.join(this.dir, roomName)
|
||||
fs.readFile(filePath, (err, data) => {
|
||||
if (err !== null) {
|
||||
resolve()
|
||||
// reject(err)
|
||||
} else {
|
||||
this._mutex(() => {
|
||||
console.info(`unpacking data (${data.length})`)
|
||||
console.time('unpacking')
|
||||
decodePersisted(y, decoding.createDecoder(data.buffer))
|
||||
console.timeEnd('unpacking')
|
||||
})
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
*/
|
||||
553
persistence/IndexedDBPersistence.js
Normal file
553
persistence/IndexedDBPersistence.js
Normal file
@@ -0,0 +1,553 @@
|
||||
/*
|
||||
import { Y } from '../utils/Y.js'
|
||||
import { createMutex } from '../lib/mutex.js'
|
||||
import { decodePersisted, encodeStructsDS, encodeUpdate, PERSIST_STRUCTS_DS, PERSIST_UPDATE } from './decodePersisted.js'
|
||||
|
||||
function rtop (request) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
request.onerror = function (event) {
|
||||
reject(new Error(event.target.error))
|
||||
}
|
||||
request.onblocked = function () {
|
||||
location.reload()
|
||||
}
|
||||
request.onsuccess = function (event) {
|
||||
resolve(event.target.result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function openDB (room) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
let request = indexedDB.open(room)
|
||||
request.onupgradeneeded = function (event) {
|
||||
const db = event.target.result
|
||||
if (db.objectStoreNames.contains('updates')) {
|
||||
db.deleteObjectStore('updates')
|
||||
}
|
||||
db.createObjectStore('updates', {autoIncrement: true})
|
||||
}
|
||||
request.onerror = function (event) {
|
||||
reject(new Error(event.target.error))
|
||||
}
|
||||
request.onblocked = function () {
|
||||
location.reload()
|
||||
}
|
||||
request.onsuccess = function (event) {
|
||||
const db = event.target.result
|
||||
db.onversionchange = function () { db.close() }
|
||||
resolve(db)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function persist (room) {
|
||||
let t = room.db.transaction(['updates'], 'readwrite')
|
||||
let updatesStore = t.objectStore('updates')
|
||||
return rtop(updatesStore.getAll())
|
||||
.then(updates => {
|
||||
// apply all previous updates before deleting them
|
||||
room.mutex(() => {
|
||||
updates.forEach(update => {
|
||||
decodePersisted(y, new BinaryDecoder(update))
|
||||
})
|
||||
})
|
||||
const encoder = new BinaryEncoder()
|
||||
encodeStructsDS(y, encoder)
|
||||
// delete all pending updates
|
||||
rtop(updatesStore.clear()).then(() => {
|
||||
// write current model
|
||||
updatesStore.put(encoder.createBuffer())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function saveUpdate (room, updateBuffer) {
|
||||
const db = room.db
|
||||
if (db !== null) {
|
||||
const t = db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
const updatePut = rtop(updatesStore.put(updateBuffer))
|
||||
rtop(updatesStore.count()).then(cnt => {
|
||||
if (cnt >= PREFERRED_TRIM_SIZE) {
|
||||
persist(room)
|
||||
}
|
||||
})
|
||||
return updatePut
|
||||
}
|
||||
}
|
||||
|
||||
function registerRoomInPersistence (documentsDB, roomName) {
|
||||
return documentsDB.then(
|
||||
db => Promise.all([
|
||||
db,
|
||||
rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
|
||||
])
|
||||
).then(
|
||||
([db, doc]) => {
|
||||
if (doc === undefined) {
|
||||
return rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: 0 }))
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
const PREFERRED_TRIM_SIZE = 400
|
||||
|
||||
export class IndexedDBPersistence {
|
||||
constructor () {
|
||||
this._rooms = new Map()
|
||||
this._documentsDB = new Promise(function (resolve, reject) {
|
||||
let request = indexedDB.open('_yjs_documents')
|
||||
request.onupgradeneeded = function (event) {
|
||||
const db = event.target.result
|
||||
if (db.objectStoreNames.contains('documents')) {
|
||||
db.deleteObjectStore('documents')
|
||||
}
|
||||
db.createObjectStore('documents', { keyPath: "roomName" })
|
||||
}
|
||||
request.onerror = function (event) {
|
||||
reject(new Error(event.target.error))
|
||||
}
|
||||
request.onblocked = function () {
|
||||
location.reload()
|
||||
}
|
||||
request.onsuccess = function (event) {
|
||||
const db = event.target.result
|
||||
db.onversionchange = function () { db.close() }
|
||||
resolve(db)
|
||||
}
|
||||
})
|
||||
addEventListener('unload', () => {
|
||||
// close everything when page unloads
|
||||
this._rooms.forEach(room => {
|
||||
if (room.db !== null) {
|
||||
room.db.close()
|
||||
} else {
|
||||
room.dbPromise.then(db => db.close())
|
||||
}
|
||||
})
|
||||
this._documentsDB.then(db => db.close())
|
||||
})
|
||||
}
|
||||
getAllDocuments () {
|
||||
return this._documentsDB.then(
|
||||
db => rtop(db.transaction(['documents'], 'readonly').objectStore('documents').getAll())
|
||||
)
|
||||
}
|
||||
setRemoteUpdateCounter (roomName, remoteUpdateCounter) {
|
||||
this._documentsDB.then(
|
||||
db => rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').put({ roomName, remoteUpdateCounter }))
|
||||
)
|
||||
}
|
||||
|
||||
_createYInstance (roomName) {
|
||||
const room = this._rooms.get(roomName)
|
||||
if (room !== undefined) {
|
||||
return room.y
|
||||
}
|
||||
const y = new Y()
|
||||
return openDB(roomName).then(
|
||||
db => rtop(db.transaction(['updates'], 'readonly').objectStore('updates').getAll())
|
||||
).then(
|
||||
updates =>
|
||||
y.transact(() => {
|
||||
updates.forEach(update => {
|
||||
decodePersisted(y, new BinaryDecoder(update))
|
||||
})
|
||||
}, true)
|
||||
).then(() => Promise.resolve(y))
|
||||
}
|
||||
|
||||
_persistStructsDS (roomName, structsDS) {
|
||||
const encoder = new BinaryEncoder()
|
||||
encoder.writeVarUint(PERSIST_STRUCTS_DS)
|
||||
encoder.writeArrayBuffer(structsDS)
|
||||
return openDB(roomName).then(db => {
|
||||
const t = db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
return rtop(updatesStore.put(encoder.createBuffer()))
|
||||
})
|
||||
}
|
||||
|
||||
_persistStructs (roomName, structs) {
|
||||
const encoder = new BinaryEncoder()
|
||||
encoder.writeVarUint(PERSIST_UPDATE)
|
||||
encoder.writeArrayBuffer(structs)
|
||||
return openDB(roomName).then(db => {
|
||||
const t = db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
return rtop(updatesStore.put(encoder.createBuffer()))
|
||||
})
|
||||
}
|
||||
|
||||
connectY (roomName, y) {
|
||||
if (this._rooms.has(roomName)) {
|
||||
throw new Error('A Y instance is already bound to this room!')
|
||||
}
|
||||
let room = {
|
||||
db: null,
|
||||
dbPromise: null,
|
||||
channel: null,
|
||||
mutex: createMutex(),
|
||||
y
|
||||
}
|
||||
if (typeof BroadcastChannel !== 'undefined') {
|
||||
room.channel = new BroadcastChannel('__yjs__' + roomName)
|
||||
room.channel.addEventListener('message', e => {
|
||||
room.mutex(function () {
|
||||
decodePersisted(y, new BinaryDecoder(e.data))
|
||||
})
|
||||
})
|
||||
}
|
||||
y.on('destroyed', () => {
|
||||
this.disconnectY(roomName, y)
|
||||
})
|
||||
y.on('afterTransaction', (y, transaction) => {
|
||||
room.mutex(() => {
|
||||
if (transaction.encodedStructsLen > 0) {
|
||||
const encoder = new BinaryEncoder()
|
||||
const update = new BinaryEncoder()
|
||||
encodeUpdate(y, transaction.encodedStructs, update)
|
||||
const updateBuffer = update.createBuffer()
|
||||
if (room.channel !== null) {
|
||||
room.channel.postMessage(updateBuffer)
|
||||
}
|
||||
if (transaction.encodedStructsLen > 0
|
||||
import { Y } from '../utils/Y.js'
|
||||
import { createMutex } from '../lib/mutex.js'
|
||||
import { decodePersisted, encodeStructsDS, encodeUpdate, PERSIST_STRUCTS_DS, PERSIST_UPDATE } from './decodePersisted.js'
|
||||
|
||||
function rtop (request) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
request.onerror = function (event) {
|
||||
reject(new Error(event.target.error))
|
||||
}
|
||||
request.onblocked = function () {
|
||||
location.reload()
|
||||
}
|
||||
request.onsuccess = function (event) {
|
||||
resolve(event.target.result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function openDB (room) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
let request = indexedDB.open(room)
|
||||
request.onupgradeneeded = function (event) {
|
||||
const db = event.target.result
|
||||
if (db.objectStoreNames.contains('updates')) {
|
||||
db.deleteObjectStore('updates')
|
||||
}
|
||||
db.createObjectStore('updates', {autoIncrement: true})
|
||||
}
|
||||
request.onerror = function (event) {
|
||||
reject(new Error(event.target.error))
|
||||
}
|
||||
request.onblocked = function () {
|
||||
location.reload()
|
||||
}
|
||||
request.onsuccess = function (event) {
|
||||
const db = event.target.result
|
||||
db.onversionchange = function () { db.close() }
|
||||
resolve(db)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function persist (room) {
|
||||
let t = room.db.transaction(['updates'], 'readwrite')
|
||||
let updatesStore = t.objectStore('updates')
|
||||
return rtop(updatesStore.getAll())
|
||||
.then(updates => {
|
||||
// apply all previous updates before deleting them
|
||||
room.mutex(() => {
|
||||
updates.forEach(update => {
|
||||
decodePersisted(y, new BinaryDecoder(update))
|
||||
})
|
||||
})
|
||||
const encoder = new BinaryEncoder()
|
||||
encodeStructsDS(y, encoder)
|
||||
// delete all pending updates
|
||||
rtop(updatesStore.clear()).then(() => {
|
||||
// write current model
|
||||
updatesStore.put(encoder.createBuffer())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function saveUpdate (room, updateBuffer) {
|
||||
const db = room.db
|
||||
if (db !== null) {
|
||||
const t = db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
const updatePut = rtop(updatesStore.put(updateBuffer))
|
||||
rtop(updatesStore.count()).then(cnt => {
|
||||
if (cnt >= PREFERRED_TRIM_SIZE) {
|
||||
persist(room)
|
||||
}
|
||||
})
|
||||
return updatePut
|
||||
}
|
||||
}
|
||||
|
||||
function registerRoomInPersistence (documentsDB, roomName) {
|
||||
return documentsDB.then(
|
||||
db => Promise.all([
|
||||
db,
|
||||
rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
|
||||
])
|
||||
).then(
|
||||
([db, doc]) => {
|
||||
if (doc === undefined) {
|
||||
return rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: 0 }))
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
const PREFERRED_TRIM_SIZE = 400
|
||||
|
||||
export class IndexedDBPersistence {
|
||||
constructor () {
|
||||
this._rooms = new Map()
|
||||
this._documentsDB = new Promise(function (resolve, reject) {
|
||||
let request = indexedDB.open('_yjs_documents')
|
||||
request.onupgradeneeded = function (event) {
|
||||
const db = event.target.result
|
||||
if (db.objectStoreNames.contains('documents')) {
|
||||
db.deleteObjectStore('documents')
|
||||
}
|
||||
db.createObjectStore('documents', { keyPath: "roomName" })
|
||||
}
|
||||
request.onerror = function (event) {
|
||||
reject(new Error(event.target.error))
|
||||
}
|
||||
request.onblocked = function () {
|
||||
location.reload()
|
||||
}
|
||||
request.onsuccess = function (event) {
|
||||
const db = event.target.result
|
||||
db.onversionchange = function () { db.close() }
|
||||
resolve(db)
|
||||
}
|
||||
})
|
||||
addEventListener('unload', () => {
|
||||
// close everything when page unloads
|
||||
this._rooms.forEach(room => {
|
||||
if (room.db !== null) {
|
||||
room.db.close()
|
||||
} else {
|
||||
room.dbPromise.then(db => db.close())
|
||||
}
|
||||
})
|
||||
this._documentsDB.then(db => db.close())
|
||||
})
|
||||
}
|
||||
getAllDocuments () {
|
||||
return this._documentsDB.then(
|
||||
db => rtop(db.transaction(['documents'], 'readonly').objectStore('documents').getAll())
|
||||
)
|
||||
}
|
||||
setRemoteUpdateCounter (roomName, remoteUpdateCounter) {
|
||||
this._documentsDB.then(
|
||||
db => rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').put({ roomName, remoteUpdateCounter }))
|
||||
)
|
||||
}
|
||||
|
||||
_createYInstance (roomName) {
|
||||
const room = this._rooms.get(roomName)
|
||||
if (room !== undefined) {
|
||||
return room.y
|
||||
}
|
||||
const y = new Y()
|
||||
return openDB(roomName).then(
|
||||
db => rtop(db.transaction(['updates'], 'readonly').objectStore('updates').getAll())
|
||||
).then(
|
||||
updates =>
|
||||
y.transact(() => {
|
||||
updates.forEach(update => {
|
||||
decodePersisted(y, new BinaryDecoder(update))
|
||||
})
|
||||
}, true)
|
||||
).then(() => Promise.resolve(y))
|
||||
}
|
||||
|
||||
_persistStructsDS (roomName, structsDS) {
|
||||
const encoder = new BinaryEncoder()
|
||||
encoder.writeVarUint(PERSIST_STRUCTS_DS)
|
||||
encoder.writeArrayBuffer(structsDS)
|
||||
return openDB(roomName).then(db => {
|
||||
const t = db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
return rtop(updatesStore.put(encoder.createBuffer()))
|
||||
})
|
||||
}
|
||||
|
||||
_persistStructs (roomName, structs) {
|
||||
const encoder = new BinaryEncoder()
|
||||
encoder.writeVarUint(PERSIST_UPDATE)
|
||||
encoder.writeArrayBuffer(structs)
|
||||
return openDB(roomName).then(db => {
|
||||
const t = db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
return rtop(updatesStore.put(encoder.createBuffer()))
|
||||
})
|
||||
}
|
||||
|
||||
connectY (roomName, y) {
|
||||
if (this._rooms.has(roomName)) {
|
||||
throw new Error('A Y instance is already bound to this room!')
|
||||
}
|
||||
let room = {
|
||||
db: null,
|
||||
dbPromise: null,
|
||||
channel: null,
|
||||
mutex: createMutex(),
|
||||
y
|
||||
}
|
||||
if (typeof BroadcastChannel !== 'undefined') {
|
||||
room.channel = new BroadcastChannel('__yjs__' + roomName)
|
||||
room.channel.addEventListener('message', e => {
|
||||
room.mutex(function () {
|
||||
decodePersisted(y, new BinaryDecoder(e.data))
|
||||
})
|
||||
})
|
||||
}
|
||||
y.on('destroyed', () => {
|
||||
this.disconnectY(roomName, y)
|
||||
})
|
||||
y.on('afterTransaction', (y, transaction) => {
|
||||
room.mutex(() => {
|
||||
if (transaction.encodedStructsLen > 0) {
|
||||
const encoder = new BinaryEncoder()
|
||||
const update = new BinaryEncoder()
|
||||
encodeUpdate(y, transaction.encodedStructs, update)
|
||||
const updateBuffer = update.createBuffer()
|
||||
if (room.channel !== null) {
|
||||
room.channel.postMessage(updateBuffer)
|
||||
}
|
||||
if (transaction.encodedStructsLen > 0) {
|
||||
if (room.db !== null) {
|
||||
saveUpdate(room, updateBuffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
// register document in documentsDB
|
||||
this._documentsDB.then(
|
||||
db =>
|
||||
rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
|
||||
.then(
|
||||
doc => doc === undefined && rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: -1 }))
|
||||
)
|
||||
)
|
||||
// open room db and read existing data
|
||||
return room.dbPromise = openDB(roomName)
|
||||
.then(db => {
|
||||
room.db = db
|
||||
const t = room.db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
// write current state as update
|
||||
const encoder = new BinaryEncoder()
|
||||
encodeStructsDS(y, encoder)
|
||||
return rtop(updatesStore.put(encoder.createBuffer())).then(() => {
|
||||
// read persisted state
|
||||
return rtop(updatesStore.getAll()).then(updates => {
|
||||
room.mutex(() => {
|
||||
y.transact(() => {
|
||||
updates.forEach(update => {
|
||||
decodePersisted(y, new BinaryDecoder(update))
|
||||
})
|
||||
}, true)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
disconnectY (roomName) {
|
||||
const {
|
||||
db, channel
|
||||
} = this._rooms.get(roomName)
|
||||
db.close()
|
||||
if (channel !== null) {
|
||||
channel.close()
|
||||
}
|
||||
this._rooms.delete(roomName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all persisted data that belongs to a room.
|
||||
* Automatically destroys all Yjs all Yjs instances that persist to
|
||||
* the room. If `destroyYjsInstances = false` the persistence functionality
|
||||
* will be removed from the Yjs instances.
|
||||
*
|
||||
removePersistedData (roomName, destroyYjsInstances = true) {
|
||||
this.disconnectY(roomName)
|
||||
return rtop(indexedDB.deleteDatabase(roomName))
|
||||
}
|
||||
}
|
||||
{
|
||||
if (room.db !== null) {
|
||||
saveUpdate(room, updateBuffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
// register document in documentsDB
|
||||
this._documentsDB.then(
|
||||
db =>
|
||||
rtop(db.transaction(['documents'], 'readonly').objectStore('documents').get(roomName))
|
||||
.then(
|
||||
doc => doc === undefined && rtop(db.transaction(['documents'], 'readwrite').objectStore('documents').add({ roomName, serverUpdateCounter: -1 }))
|
||||
)
|
||||
)
|
||||
// open room db and read existing data
|
||||
return room.dbPromise = openDB(roomName)
|
||||
.then(db => {
|
||||
room.db = db
|
||||
const t = room.db.transaction(['updates'], 'readwrite')
|
||||
const updatesStore = t.objectStore('updates')
|
||||
// write current state as update
|
||||
const encoder = new BinaryEncoder()
|
||||
encodeStructsDS(y, encoder)
|
||||
return rtop(updatesStore.put(encoder.createBuffer())).then(() => {
|
||||
// read persisted state
|
||||
return rtop(updatesStore.getAll()).then(updates => {
|
||||
room.mutex(() => {
|
||||
y.transact(() => {
|
||||
updates.forEach(update => {
|
||||
decodePersisted(y, new BinaryDecoder(update))
|
||||
})
|
||||
}, true)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
disconnectY (roomName) {
|
||||
const {
|
||||
db, channel
|
||||
} = this._rooms.get(roomName)
|
||||
db.close()
|
||||
if (channel !== null) {
|
||||
channel.close()
|
||||
}
|
||||
this._rooms.delete(roomName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all persisted data that belongs to a room.
|
||||
* Automatically destroys all Yjs all Yjs instances that persist to
|
||||
* the room. If `destroyYjsInstances = false` the persistence functionality
|
||||
* will be removed from the Yjs instances.
|
||||
*
|
||||
removePersistedData (roomName, destroyYjsInstances = true) {
|
||||
this.disconnectY(roomName)
|
||||
return rtop(indexedDB.deleteDatabase(roomName))
|
||||
}
|
||||
}
|
||||
*/
|
||||
53
persistence/decodePersisted.js
Normal file
53
persistence/decodePersisted.js
Normal file
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
import { integrateRemoteStructs } from '../MessageHandler/integrateRemoteStructs.js'
|
||||
import { writeStructs } from '../MessageHandler/syncStep1.js'
|
||||
import { writeDeleteSet, readDeleteSet } from '../MessageHandler/deleteSet.js'
|
||||
|
||||
export const PERSIST_UPDATE = 0
|
||||
/**
|
||||
* Write an update to an encoder.
|
||||
*
|
||||
* @param {Y} y A Yjs instance
|
||||
* @param {Encoder} updateEncoder I.e. transaction.encodedStructs
|
||||
*
|
||||
export const encodeUpdate = (y, updateEncoder, encoder) => {
|
||||
encoder.writeVarUint(PERSIST_UPDATE)
|
||||
encoder.writeBinaryEncoder(updateEncoder)
|
||||
}
|
||||
|
||||
export const PERSIST_STRUCTS_DS = 1
|
||||
|
||||
/**
|
||||
* Write the current Yjs data model to an encoder.
|
||||
*
|
||||
* @param {Y} y A Yjs instance
|
||||
* @param {Encoder} encoder An encoder to write to
|
||||
*
|
||||
export const encodeStructsDS = (y, encoder) => {
|
||||
encoder.writeVarUint(PERSIST_STRUCTS_DS)
|
||||
writeStructs(y, encoder, new Map())
|
||||
writeDeleteSet(y, encoder)
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed the Yjs instance with the persisted state
|
||||
* @param {Y} y A Yjs instance.
|
||||
* @param {Decoder} decoder A Decoder instance that holds the file content.
|
||||
*
|
||||
export const decodePersisted = (y, decoder) => {
|
||||
y.transact(() => {
|
||||
while (decoder.hasContent()) {
|
||||
const contentType = decoder.readVarUint()
|
||||
switch (contentType) {
|
||||
case PERSIST_UPDATE:
|
||||
integrateRemoteStructs(decoder, y)
|
||||
break
|
||||
case PERSIST_STRUCTS_DS:
|
||||
integrateRemoteStructs(decoder, y)
|
||||
readDeleteSet(y, decoder)
|
||||
break
|
||||
}
|
||||
}
|
||||
}, true)
|
||||
}
|
||||
*/
|
||||
0
persistence/indexeddb.js
Normal file
0
persistence/indexeddb.js
Normal file
125
persistence/leveldb.js
Normal file
125
persistence/leveldb.js
Normal file
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* @module persistence/leveldb
|
||||
* This module re-uses the encoding of syncProtocol to store and read updates from leveldb.
|
||||
*/
|
||||
|
||||
const level = require('level')
|
||||
const Y = require('../build/yjs.js')
|
||||
const mux = Y.createMutex()
|
||||
|
||||
/*
|
||||
* Improves the uniqueness of timestamps.
|
||||
* We gamble with the fact that users won't create more than 10000 changes on a single document
|
||||
* within one millisecond (also assuming clock works correctly).
|
||||
*/
|
||||
let timestampIterator = 0
|
||||
/**
|
||||
* @return {string} A random, time-based string starting with "${roomName}:"
|
||||
*/
|
||||
const getNextTimestamp = () => {
|
||||
timestampIterator = (timestampIterator + 1) % 10000
|
||||
return `${Date.now()}${timestampIterator.toString().padStart(4, '0')}`
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} docName
|
||||
* @return {string}
|
||||
*/
|
||||
const generateEntryKey = docName => `${docName}#${getNextTimestamp()}`
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {any} db
|
||||
* @param {string} docName
|
||||
* @param {Uint8Array | ArrayBuffer} buf
|
||||
*/
|
||||
const writeEntry = (db, docName, buf) => db.put(generateEntryKey(docName), buf)
|
||||
|
||||
/**
|
||||
* @param {Uint8Array} arr
|
||||
* @param {Y.Y} ydocument
|
||||
*/
|
||||
const readEntry = (arr, ydocument) => mux(() =>
|
||||
Y.syncProtocol.readSyncMessage(Y.decoding.createDecoder(arr), Y.encoding.createEncoder(), ydocument)
|
||||
)
|
||||
|
||||
/**
|
||||
* @param {any} db
|
||||
* @param {string} docName
|
||||
* @param {Y.Y} ydocument
|
||||
*/
|
||||
const loadFromPersistence = (db, docName, ydocument) => new Promise((resolve, reject)=>
|
||||
db.createReadStream({
|
||||
gte: `${docName}#`,
|
||||
lte: `${docName}#Z`,
|
||||
keys: false,
|
||||
values: true
|
||||
})
|
||||
.on('data', data => readEntry(data, ydocument))
|
||||
.on('error', reject)
|
||||
.on('end', resolve)
|
||||
.on('close', resolve)
|
||||
)
|
||||
|
||||
const persistState = (db, docName, ydocument) => {
|
||||
const encoder = Y.encoding.createEncoder()
|
||||
Y.syncProtocol.writeSyncStep2(encoder, ydocument, new Map())
|
||||
const entryKey = generateEntryKey(docName)
|
||||
const entryPromise = db.put(entryKey, Y.encoding.toBuffer(encoder))
|
||||
const delOps = []
|
||||
return new Promise((resolve, reject) => db.createKeyStream({
|
||||
gte: `${docName}#`,
|
||||
lt: entryKey,
|
||||
})
|
||||
.on('data', key => delOps.push({ type: 'del', key }))
|
||||
.on('error', reject)
|
||||
.on('end', resolve)
|
||||
.on('close', resolve)
|
||||
).then(() => entryPromise).then(() => db.batch(delOps))
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistence layer for Leveldb.
|
||||
*/
|
||||
exports.LevelDbPersistence = class LevelDbPersistence {
|
||||
/**
|
||||
* @param {string} fpath Path to leveldb database
|
||||
*/
|
||||
constructor (fpath) {
|
||||
this.db = level(fpath, { valueEncoding: 'binary' })
|
||||
}
|
||||
/**
|
||||
* Retrieve all data from LevelDB and automatically persist all document updates to leveldb.
|
||||
*
|
||||
* @param {string} docName
|
||||
* @param {Y.Y} ydocument
|
||||
*/
|
||||
bindState (docName, ydocument) {
|
||||
// write all updates received from other clients
|
||||
// - unless it is created by this persistence layer (e.g. loadFromPersistence, we we mux).
|
||||
ydocument.on('afterTransaction', (y, transaction) => {
|
||||
if (transaction.encodedStructsLen > 0) {
|
||||
mux(() => {
|
||||
const encoder = Y.encoding.createEncoder()
|
||||
Y.syncProtocol.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
|
||||
writeEntry(this.db, docName, Y.encoding.toBuffer(encoder))
|
||||
})
|
||||
}
|
||||
})
|
||||
// read all data from persistence
|
||||
return loadFromPersistence(this.db, docName, ydocument).then(() =>
|
||||
// write current state (just in case anything was added before state was bound)
|
||||
this.writeState(docName, ydocument)
|
||||
)
|
||||
}
|
||||
/**
|
||||
* Write current state to persistence layer. Deletes all entries that were made before.
|
||||
* Call this method at any time - the recommended time to call this method is before the ydocument is destroyed.
|
||||
*
|
||||
* @param {string} docName
|
||||
* @param {Y.Y} ydocument
|
||||
*/
|
||||
writeState (docName, ydocument) {
|
||||
return persistState(this.db, docName, ydocument)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user