diff --git a/README.v13.md b/README.v13.md index 5d345f43..ea085128 100644 --- a/README.v13.md +++ b/README.v13.md @@ -159,6 +159,16 @@ The Websocket Provider is a solid choice if you want a central source that handl PORT=1234 node ./node_modules/yjs/provider/websocket/server.js ``` +**Websocket Server with Persistence** + +Persist document updates in a LevelDB database. + +See [LevelDB Persistence](#LevelDB Persistence) for more info. + +```sh +PORT=1234 YPERSISTENCE=./dbDir node ./node_modules/yjs/provider/websocket/server.js +``` + ##### Client Code: ```js diff --git a/package-lock.json b/package-lock.json index 4622de6d..e28ca4fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -96,6 +96,15 @@ "dev": true, "optional": true }, + "abstract-leveldown": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-5.0.0.tgz", + "integrity": "sha512-5mU5P1gXtsMIXg65/rsYGsi93+MlogXZ9FA8JnwKurHQg64bfXwGYVdVdijNTVNOlAsuIiOwHdvFFD5JqCJQ7A==", + "dev": true, + "requires": { + "xtend": "~4.0.0" + } + }, "accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", @@ -226,6 +235,22 @@ "integrity": "sha1-7klza2ObTxCLbp5ibG2pkwa0FpI=", "dev": true }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "dev": true + }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "dev": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1640,6 +1665,54 @@ "integrity": "sha1-SOyNFt9Dd+rl+liEaCSAr02Vx3Q=", "dev": true }, + "bindings": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz", + "integrity": "sha512-i47mqjF9UbjxJhxGf+pZ6kSxrnI3wBLlnGI2ArWJ4r0VrvDS7ZYXkprq/pLaBWYq4GM0r4zdHY+NNRqEMU7uew==", + "dev": true + }, + "bl": { + "version": "1.2.2", + "resolved": "http://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", + "dev": true, + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + }, + "dependencies": { + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "bluebird": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", @@ -1673,6 +1746,28 @@ "repeat-element": "^1.1.2" } }, + "buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "dev": true, + "requires": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "dev": true + }, + "buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=", + "dev": true + }, "buffer-from": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz", @@ -1794,6 +1889,12 @@ "readdirp": "^2.0.0" } }, + "chownr": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", + "dev": true + }, "circular-json": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", @@ -1827,6 +1928,12 @@ "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", "dev": true }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "dev": true + }, "codemirror": { "version": "5.42.0", "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.42.0.tgz", @@ -2036,6 +2143,12 @@ } } }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "dev": true + }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -2168,18 +2281,43 @@ "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", "dev": true }, + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + } + }, "deep-equal": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", "dev": true }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, + "deferred-leveldown": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-4.0.2.tgz", + "integrity": "sha512-5fMC8ek8alH16QiV0lTCis610D1Zt1+LA4MS4d63JgS32lrCjTFDUFz2ao09/j2I4Bqb5jL4FZYwu7Jz0XO1ww==", + "dev": true, + "requires": { + "abstract-leveldown": "~5.0.0", + "inherits": "^2.0.3" + } + }, "define-properties": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", @@ -2225,6 +2363,12 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", "dev": true }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "dev": true + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -2246,6 +2390,12 @@ "repeating": "^2.0.0" } }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "dev": true + }, "doctrine": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", @@ -2336,12 +2486,43 @@ "iconv-lite": "~0.4.13" } }, + "encoding-down": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-5.0.4.tgz", + "integrity": "sha512-8CIZLDcSKxgzT+zX8ZVfgNbu8Md2wq/iqa1Y7zyVR18QBEAc0Nmzuvj/N5ykSKpfGzjM8qxbaFntLPwnVoUhZw==", + "dev": true, + "requires": { + "abstract-leveldown": "^5.0.0", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0", + "xtend": "^4.0.1" + } + }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, "entities": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=", "dev": true }, + "errno": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", + "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "dev": true, + "requires": { + "prr": "~1.0.1" + } + }, "error-ex": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz", @@ -3117,6 +3298,12 @@ "fill-range": "^2.1.0" } }, + "expand-template": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz", + "integrity": "sha512-cebqLtV8KOZfw0UI8TEFWxtczxxC1jvyUvx6H4fyp1K1FN7A4Q+uggVUlOsI1K8AGU0rwOGqP8nCapdrw8CYQg==", + "dev": true + }, "expand-tilde": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-1.2.2.tgz", @@ -3179,6 +3366,12 @@ "integrity": "sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==", "dev": true }, + "fast-future": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/fast-future/-/fast-future-1.0.2.tgz", + "integrity": "sha1-hDWpqqAteSSNF9cE52JZMB2ZKAo=", + "dev": true + }, "fast-json-stable-stringify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", @@ -3365,6 +3558,12 @@ "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=", "dev": true }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true + }, "fs-exists-sync": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz", @@ -3935,6 +4134,44 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "dev": true, + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dev": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dev": true, + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + } + } + }, "get-stdin": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz", @@ -3951,6 +4188,12 @@ "assert-plus": "^1.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", + "dev": true + }, "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", @@ -4089,6 +4332,12 @@ "is-glob": "^2.0.1" } }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "dev": true + }, "home-or-tmp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-2.0.0.tgz", @@ -4865,6 +5114,118 @@ "set-getter": "^0.1.0" } }, + "level": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/level/-/level-4.0.0.tgz", + "integrity": "sha512-4epzCOlEcJ529NOdlAYiuiakS/kZTDdiKSBNJmE1B8bsmA+zEVwcpxyH86qJSQTpOu7SODrlaD9WgPRHLkGutA==", + "dev": true, + "requires": { + "level-packager": "^3.0.0", + "leveldown": "^4.0.0", + "opencollective-postinstall": "^2.0.0" + } + }, + "level-codec": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.0.tgz", + "integrity": "sha512-OIpVvjCcZNP5SdhcNupnsI1zo5Y9Vpm+k/F1gfG5kXrtctlrwanisakweJtE0uA0OpLukRfOQae+Fg0M5Debhg==", + "dev": true + }, + "level-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.0.tgz", + "integrity": "sha512-AmY4HCp9h3OiU19uG+3YWkdELgy05OTP/r23aNHaQKWv8DO787yZgsEuGVkoph40uwN+YdUKnANlrxSsoOaaxg==", + "dev": true, + "requires": { + "errno": "~0.1.1" + } + }, + "level-iterator-stream": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-3.0.1.tgz", + "integrity": "sha512-nEIQvxEED9yRThxvOrq8Aqziy4EGzrxSZK+QzEFAVuJvQ8glfyZ96GB6BoI4sBbLfjMXm2w4vu3Tkcm9obcY0g==", + "dev": true, + "requires": { + "inherits": "^2.0.1", + "readable-stream": "^2.3.6", + "xtend": "^4.0.0" + }, + "dependencies": { + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "level-packager": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-3.1.0.tgz", + "integrity": "sha512-UxVEfK5WH0u0InR3WxTCSAroiorAGKzXWZT6i+nBjambmvINuXFUsFx2Ai3UIjUUtnyWhluv42jMlzUZCsAk9A==", + "dev": true, + "requires": { + "encoding-down": "~5.0.0", + "levelup": "^3.0.0" + } + }, + "leveldown": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-4.0.1.tgz", + "integrity": "sha512-ZlBKVSsglPIPJnz4ggB8o2R0bxDxbsMzuQohbfgoFMVApyTE118DK5LNRG0cRju6rt3OkGxe0V6UYACGlq/byg==", + "dev": true, + "requires": { + "abstract-leveldown": "~5.0.0", + "bindings": "~1.3.0", + "fast-future": "~1.0.2", + "nan": "~2.10.0", + "prebuild-install": "^4.0.0" + }, + "dependencies": { + "nan": { + "version": "2.10.0", + "resolved": "http://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==", + "dev": true + } + } + }, + "levelup": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-3.1.1.tgz", + "integrity": "sha512-9N10xRkUU4dShSRRFTBdNaBxofz+PGaIZO962ckboJZiNmLuhVT6FZ6ZKAsICKfUBO76ySaYU6fJWX/jnj3Lcg==", + "dev": true, + "requires": { + "deferred-leveldown": "~4.0.0", + "level-errors": "~2.0.0", + "level-iterator-stream": "~3.0.0", + "xtend": "~4.0.0" + } + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -5282,6 +5643,12 @@ "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", "dev": true }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -5350,6 +5717,23 @@ "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=", "dev": true }, + "node-abi": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.5.0.tgz", + "integrity": "sha512-9g2twBGSP6wIR5PW7tXvAWnEWKJDH/VskdXp168xsw9VVxpEGov8K4jsP4/VeoC7b2ZAyzckvMCuQuQlw44lXg==", + "dev": true, + "requires": { + "semver": "^5.4.1" + }, + "dependencies": { + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "dev": true + } + } + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -5360,6 +5744,12 @@ "is-stream": "^1.0.1" } }, + "noop-logger": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", + "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=", + "dev": true + }, "normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -5381,6 +5771,18 @@ "remove-trailing-separator": "^1.0.1" } }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "dev": true, + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, "nth-check": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz", @@ -5465,6 +5867,12 @@ "mimic-fn": "^1.0.0" } }, + "opencollective-postinstall": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz", + "integrity": "sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ==", + "dev": true + }, "optionator": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", @@ -5736,6 +6144,37 @@ "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==", "dev": true }, + "prebuild-install": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-4.0.0.tgz", + "integrity": "sha512-7tayxeYboJX0RbVzdnKyGl2vhQRWr6qfClEXDhOkXjuaOKCw2q8aiuFhONRYVsG/czia7KhpykIlI2S2VaPunA==", + "dev": true, + "requires": { + "detect-libc": "^1.0.3", + "expand-template": "^1.0.2", + "github-from-package": "0.0.0", + "minimist": "^1.2.0", + "mkdirp": "^0.5.1", + "node-abi": "^2.2.0", + "noop-logger": "^0.1.1", + "npmlog": "^4.0.1", + "os-homedir": "^1.0.1", + "pump": "^2.0.1", + "rc": "^1.1.6", + "simple-get": "^2.7.0", + "tar-fs": "^1.13.0", + "tunnel-agent": "^0.6.0", + "which-pm-runs": "^1.0.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + } + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -5938,6 +6377,12 @@ "prosemirror-transform": "^1.1.0" } }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", + "dev": true + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -5951,6 +6396,16 @@ "dev": true, "optional": true }, + "pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -6036,6 +6491,26 @@ "integrity": "sha1-vOeMkhsjWCuuIR9ZdGSlkf0alPk=", "dev": true }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + } + } + }, "read-pkg": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", @@ -6685,6 +7160,12 @@ } } }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true + }, "set-getter": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/set-getter/-/set-getter-0.1.0.tgz", @@ -6733,6 +7214,23 @@ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true }, + "simple-concat": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", + "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=", + "dev": true + }, + "simple-get": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz", + "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==", + "dev": true, + "requires": { + "decompress-response": "^3.3.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "slash": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", @@ -7061,6 +7559,45 @@ "integrity": "sha1-e/gQalwaSCUbPjvAoOFzJIn9Dcg=", "dev": true }, + "tar-fs": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", + "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", + "dev": true, + "requires": { + "chownr": "^1.0.1", + "mkdirp": "^0.5.1", + "pump": "^1.0.0", + "tar-stream": "^1.1.2" + }, + "dependencies": { + "pump": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", + "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, + "tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "dev": true, + "requires": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -7088,6 +7625,12 @@ "os-tmpdir": "~1.0.2" } }, + "to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", + "dev": true + }, "to-fast-properties": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", @@ -7152,7 +7695,6 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.0.1" } @@ -7382,6 +7924,21 @@ "isexe": "^2.0.0" } }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", + "dev": true + }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "dev": true, + "requires": { + "string-width": "^1.0.2 || 2" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 8d90d524..2423821f 100644 --- a/package.json +++ b/package.json @@ -96,12 +96,11 @@ "rollup-regenerator-runtime": "^6.23.1", "rollup-watch": "^3.2.2", "standard": "^11.0.1", - "tui-jsdoc-template": "^1.2.2" + "tui-jsdoc-template": "^1.2.2", + "codemirror": "^5.42.0" }, "optionalDependencies": { + "level": "^4.0.0", "ws": "^6.1.0" - }, - "dependencies": { - "codemirror": "^5.42.0" } } diff --git a/persistences/FilePersistence.js b/persistence/FilePersistence.js similarity index 100% rename from persistences/FilePersistence.js rename to persistence/FilePersistence.js diff --git a/persistences/IndexedDBPersistence.js b/persistence/IndexedDBPersistence.js similarity index 100% rename from persistences/IndexedDBPersistence.js rename to persistence/IndexedDBPersistence.js diff --git a/persistences/decodePersisted.js b/persistence/decodePersisted.js similarity index 100% rename from persistences/decodePersisted.js rename to persistence/decodePersisted.js diff --git a/persistences/indexeddb.js b/persistence/indexeddb.js similarity index 100% rename from persistences/indexeddb.js rename to persistence/indexeddb.js diff --git a/persistence/leveldb.js b/persistence/leveldb.js new file mode 100644 index 00000000..86786596 --- /dev/null +++ b/persistence/leveldb.js @@ -0,0 +1,125 @@ +/** + * @module persistence/leveldb + * This module re-uses the encoding of syncProtocol to store and read updates from leveldb. + */ + +const level = require('level') +const Y = require('../build/yjs.js') +const mux = Y.createMutex() + +/* + * Improves the uniqueness of timestamps. + * We gamble with the fact that users won't create more than 10000 changes on a single document + * within one millisecond (also assuming clock works correctly). + */ +let timestampIterator = 0 +/** + * @return {string} A random, time-based string starting with "${roomName}:" + */ +const getNextTimestamp = () => { + timestampIterator = (timestampIterator + 1) % 10000 + return `${Date.now()}${timestampIterator.toString().padStart(4, '0')}` +} + +/** + * @param {string} docName + * @return {string} + */ +const generateEntryKey = docName => `${docName}#${getNextTimestamp()}` + +/** + * + * @param {any} db + * @param {string} docName + * @param {Uint8Array | ArrayBuffer} buf + */ +const writeEntry = (db, docName, buf) => db.put(generateEntryKey(docName), buf) + +/** + * @param {Uint8Array} arr + * @param {Y.Y} ydocument + */ +const readEntry = (arr, ydocument) => mux(() => + Y.syncProtocol.readSyncMessage(Y.decoding.createDecoder(arr), Y.encoding.createEncoder(), ydocument) +) + +/** + * @param {any} db + * @param {string} docName + * @param {Y.Y} ydocument + */ +const loadFromPersistence = (db, docName, ydocument) => new Promise((resolve, reject)=> + db.createReadStream({ + gte: `${docName}#`, + lte: `${docName}#Z`, + keys: false, + values: true + }) + .on('data', data => readEntry(data, ydocument)) + .on('error', reject) + .on('end', resolve) + .on('close', resolve) +) + +const persistState = (db, docName, ydocument) => { + const encoder = Y.encoding.createEncoder() + Y.syncProtocol.writeSyncStep2(encoder, ydocument, new Map()) + const entryKey = generateEntryKey(docName) + const entryPromise = db.put(entryKey, Y.encoding.toBuffer(encoder)) + const delOps = [] + return new Promise((resolve, reject) => db.createKeyStream({ + gte: `${docName}#`, + lt: entryKey, + }) + .on('data', key => delOps.push({ type: 'del', key })) + .on('error', reject) + .on('end', resolve) + .on('close', resolve) + ).then(() => entryPromise).then(() => db.batch(delOps)) +} + +/** + * Persistence layer for Leveldb. + */ +exports.LevelDbPersistence = class LevelDbPersistence { + /** + * @param {string} fpath Path to leveldb database + */ + constructor (fpath) { + this.db = level(fpath, { valueEncoding: 'binary' }) + } + /** + * Retrieve all data from LevelDB and automatically persist all document updates to leveldb. + * + * @param {string} docName + * @param {Y.Y} ydocument + */ + bindState (docName, ydocument) { + // write all updates received from other clients + // - unless it is created by this persistence layer (e.g. loadFromPersistence, we we mux). + ydocument.on('afterTransaction', (y, transaction) => { + if (transaction.encodedStructsLen > 0) { + mux(() => { + const encoder = Y.encoding.createEncoder() + Y.syncProtocol.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs) + writeEntry(this.db, docName, Y.encoding.toBuffer(encoder)) + }) + } + }) + // read all data from persistence + return loadFromPersistence(this.db, docName, ydocument).then(() => + // write current state (just in case anything was added before state was bound) + this.writeState(docName, ydocument) + ) + } + /** + * Write current state to persistence layer. Deletes all entries that were made before. + * Call this method at any time - the recommended time to call this method is before the ydocument is destroyed. + * + * @param {string} docName + * @param {Y.Y} ydocument + */ + writeState (docName, ydocument) { + return persistState(this.db, docName, ydocument) + } +} \ No newline at end of file diff --git a/protocols/sync.js b/protocols/sync.js index 25bd70b1..42d2abe1 100644 --- a/protocols/sync.js +++ b/protocols/sync.js @@ -331,6 +331,17 @@ export const writeSyncStep1 = (encoder, y) => { writeStateSet(encoder, y) } +/** + * @param {encoding.Encoder} encoder + * @param {Y} y + * @param {Map} ss + */ +export const writeSyncStep2 = (encoder, y, ss) => { + encoding.writeVarUint(encoder, messageYjsSyncStep2) + writeStructs(encoder, y, ss) + writeDeleteSet(encoder, y) +} + /** * Read SyncStep1 message and reply with SyncStep2. * @@ -338,14 +349,8 @@ export const writeSyncStep1 = (encoder, y) => { * @param {encoding.Encoder} encoder The received message * @param {Y} y */ -export const readSyncStep1 = (decoder, encoder, y) => { - // read sync step 1 message - const ss = readStateSet(decoder) - // write sync step 2 - encoding.writeVarUint(encoder, messageYjsSyncStep2) - writeStructs(encoder, y, ss) - writeDeleteSet(encoder, y) -} +export const readSyncStep1 = (decoder, encoder, y) => + writeSyncStep2(encoder, y, readStateSet(decoder)) /** * @param {decoding.Decoder} decoder diff --git a/provider/websocket/server.js b/provider/websocket/server.js index 7d3f5209..cc601108 100644 --- a/provider/websocket/server.js +++ b/provider/websocket/server.js @@ -12,6 +12,13 @@ const http = require('http') const port = process.env.PORT || 1234 +const persistenceDir = process.env.YPERSISTENCE +let persistence = null +if (typeof persistenceDir) { + const LevelDbPersistence = require('../../persistence/leveldb.js').LevelDbPersistence + persistence = new LevelDbPersistence(persistenceDir) +} + const server = http.createServer((req, res) => { res.writeHead(200, { 'Content-Type': 'text/plain' }) res.end('okay') @@ -80,10 +87,14 @@ const messageListener = (conn, doc, message) => { const setupConnection = (conn, req) => { conn.binaryType = 'arraybuffer' // get doc, create if it does not exist yet - let doc = docs.get(req.url.slice(1)) + const docName = req.url.slice(1) + let doc = docs.get(docName) if (doc === undefined) { doc = new WSSharedDoc() - docs.set(req.url.slice(1), doc) + if (persistence !== null) { + persistence.bindState(docName, doc) + } + docs.set(docName, doc) } doc.conns.set(conn, new Set()) // listen and reply to events @@ -99,6 +110,13 @@ const setupConnection = (conn, req) => { })) const buf = Y.encoding.toBuffer(encoder) doc.conns.forEach((_, conn) => conn.send(buf)) + if (doc.conns.size === 0 && persistence !== null) { + // if persisted, we store state and destroy ydocument + persistence.writeState(docName, doc).then(() => { + doc.destroy() + }) + docs.delete(docName) + } }) // send sync step 1 const encoder = Y.encoding.createEncoder() diff --git a/tmp/000005.ldb b/tmp/000005.ldb new file mode 100644 index 00000000..f4c6f397 Binary files /dev/null and b/tmp/000005.ldb differ diff --git a/tmp/000006.log b/tmp/000006.log new file mode 100644 index 00000000..9f199c34 Binary files /dev/null and b/tmp/000006.log differ diff --git a/tmp/CURRENT b/tmp/CURRENT new file mode 100644 index 00000000..cacca757 --- /dev/null +++ b/tmp/CURRENT @@ -0,0 +1 @@ +MANIFEST-000004 diff --git a/tmp/LOCK b/tmp/LOCK new file mode 100644 index 00000000..e69de29b diff --git a/tmp/LOG b/tmp/LOG new file mode 100644 index 00000000..5e3bd35a --- /dev/null +++ b/tmp/LOG @@ -0,0 +1,5 @@ +2018/12/22-13:33:44.376775 70000c740000 Recovering log #3 +2018/12/22-13:33:44.377203 70000c740000 Level-0 table #5: started +2018/12/22-13:33:44.379580 70000c740000 Level-0 table #5: 4834 bytes OK +2018/12/22-13:33:44.380496 70000c740000 Delete type=0 #3 +2018/12/22-13:33:44.380670 70000c740000 Delete type=3 #2 diff --git a/tmp/LOG.old b/tmp/LOG.old new file mode 100644 index 00000000..19ad4ad9 --- /dev/null +++ b/tmp/LOG.old @@ -0,0 +1 @@ +2018/12/22-13:23:54.104944 700007836000 Delete type=3 #1 diff --git a/tmp/MANIFEST-000004 b/tmp/MANIFEST-000004 new file mode 100644 index 00000000..2099f35d Binary files /dev/null and b/tmp/MANIFEST-000004 differ