implemented leveldb persistence for websocket server

README.v13.md

@@ -159,6 +159,16 @@ The Websocket Provider is a solid choice if you want a central source that handl
 PORT=1234 node ./node_modules/yjs/provider/websocket/server.js
 ```
 
+**Websocket Server with Persistence**
+
+Persist document updates in a LevelDB database.
+
+See [LevelDB Persistence](#LevelDB Persistence) for more info.
+
+```sh
+PORT=1234 YPERSISTENCE=./dbDir node ./node_modules/yjs/provider/websocket/server.js
+```
+
 ##### Client Code:
 
 ```js

package-lock.json

@@ -96,6 +96,15 @@
       "dev": true,
       "optional": true
     },
+    "abstract-leveldown": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-5.0.0.tgz",
+      "integrity": "sha512-5mU5P1gXtsMIXg65/rsYGsi93+MlogXZ9FA8JnwKurHQg64bfXwGYVdVdijNTVNOlAsuIiOwHdvFFD5JqCJQ7A==",
+      "dev": true,
+      "requires": {
+        "xtend": "~4.0.0"
+      }
+    },
     "accepts": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz",
@@ -226,6 +235,22 @@
       "integrity": "sha1-7klza2ObTxCLbp5ibG2pkwa0FpI=",
       "dev": true
     },
+    "aproba": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
+      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
+      "dev": true
+    },
+    "are-we-there-yet": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
+      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
+      "dev": true,
+      "requires": {
+        "delegates": "^1.0.0",
+        "readable-stream": "^2.0.6"
+      }
+    },
     "argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -1640,6 +1665,54 @@
       "integrity": "sha1-SOyNFt9Dd+rl+liEaCSAr02Vx3Q=",
       "dev": true
     },
+    "bindings": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz",
+      "integrity": "sha512-i47mqjF9UbjxJhxGf+pZ6kSxrnI3wBLlnGI2ArWJ4r0VrvDS7ZYXkprq/pLaBWYq4GM0r4zdHY+NNRqEMU7uew==",
+      "dev": true
+    },
+    "bl": {
+      "version": "1.2.2",
+      "resolved": "http://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
+      "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
+      "dev": true,
+      "requires": {
+        "readable-stream": "^2.3.5",
+        "safe-buffer": "^5.1.1"
+      },
+      "dependencies": {
+        "process-nextick-args": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+          "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+          "dev": true
+        },
+        "readable-stream": {
+          "version": "2.3.6",
+          "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+          "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+          "dev": true,
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "dev": true,
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
     "bluebird": {
       "version": "3.5.3",
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
@@ -1673,6 +1746,28 @@
         "repeat-element": "^1.1.2"
       }
     },
+    "buffer-alloc": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
+      "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
+      "dev": true,
+      "requires": {
+        "buffer-alloc-unsafe": "^1.1.0",
+        "buffer-fill": "^1.0.0"
+      }
+    },
+    "buffer-alloc-unsafe": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
+      "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==",
+      "dev": true
+    },
+    "buffer-fill": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
+      "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=",
+      "dev": true
+    },
     "buffer-from": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz",
@@ -1794,6 +1889,12 @@
         "readdirp": "^2.0.0"
       }
     },
+    "chownr": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
+      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==",
+      "dev": true
+    },
     "circular-json": {
       "version": "0.3.3",
       "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz",
@@ -1827,6 +1928,12 @@
       "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
       "dev": true
     },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
+      "dev": true
+    },
     "codemirror": {
       "version": "5.42.0",
       "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.42.0.tgz",
@@ -2036,6 +2143,12 @@
         }
       }
     },
+    "console-control-strings": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
+      "dev": true
+    },
     "contains-path": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz",
@@ -2168,18 +2281,43 @@
       "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
       "dev": true
     },
+    "decompress-response": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
+      "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
+      "dev": true,
+      "requires": {
+        "mimic-response": "^1.0.0"
+      }
+    },
     "deep-equal": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
       "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=",
       "dev": true
     },
+    "deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+      "dev": true
+    },
     "deep-is": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
       "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
       "dev": true
     },
+    "deferred-leveldown": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-4.0.2.tgz",
+      "integrity": "sha512-5fMC8ek8alH16QiV0lTCis610D1Zt1+LA4MS4d63JgS32lrCjTFDUFz2ao09/j2I4Bqb5jL4FZYwu7Jz0XO1ww==",
+      "dev": true,
+      "requires": {
+        "abstract-leveldown": "~5.0.0",
+        "inherits": "^2.0.3"
+      }
+    },
     "define-properties": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
@@ -2225,6 +2363,12 @@
       "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
       "dev": true
     },
+    "delegates": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
+      "dev": true
+    },
     "depd": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -2246,6 +2390,12 @@
         "repeating": "^2.0.0"
       }
     },
+    "detect-libc": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
+      "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=",
+      "dev": true
+    },
     "doctrine": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
@@ -2336,12 +2486,43 @@
         "iconv-lite": "~0.4.13"
       }
     },
+    "encoding-down": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-5.0.4.tgz",
+      "integrity": "sha512-8CIZLDcSKxgzT+zX8ZVfgNbu8Md2wq/iqa1Y7zyVR18QBEAc0Nmzuvj/N5ykSKpfGzjM8qxbaFntLPwnVoUhZw==",
+      "dev": true,
+      "requires": {
+        "abstract-leveldown": "^5.0.0",
+        "inherits": "^2.0.3",
+        "level-codec": "^9.0.0",
+        "level-errors": "^2.0.0",
+        "xtend": "^4.0.1"
+      }
+    },
+    "end-of-stream": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
+      "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
+      "dev": true,
+      "requires": {
+        "once": "^1.4.0"
+      }
+    },
     "entities": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
       "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=",
       "dev": true
     },
+    "errno": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
+      "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==",
+      "dev": true,
+      "requires": {
+        "prr": "~1.0.1"
+      }
+    },
     "error-ex": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz",
@@ -3117,6 +3298,12 @@
         "fill-range": "^2.1.0"
       }
     },
+    "expand-template": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz",
+      "integrity": "sha512-cebqLtV8KOZfw0UI8TEFWxtczxxC1jvyUvx6H4fyp1K1FN7A4Q+uggVUlOsI1K8AGU0rwOGqP8nCapdrw8CYQg==",
+      "dev": true
+    },
     "expand-tilde": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-1.2.2.tgz",
@@ -3179,6 +3366,12 @@
       "integrity": "sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==",
       "dev": true
     },
+    "fast-future": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/fast-future/-/fast-future-1.0.2.tgz",
+      "integrity": "sha1-hDWpqqAteSSNF9cE52JZMB2ZKAo=",
+      "dev": true
+    },
     "fast-json-stable-stringify": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
@@ -3365,6 +3558,12 @@
       "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=",
       "dev": true
     },
+    "fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "dev": true
+    },
     "fs-exists-sync": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz",
@@ -3935,6 +4134,44 @@
       "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
       "dev": true
     },
+    "gauge": {
+      "version": "2.7.4",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
+      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
+      "dev": true,
+      "requires": {
+        "aproba": "^1.0.3",
+        "console-control-strings": "^1.0.0",
+        "has-unicode": "^2.0.0",
+        "object-assign": "^4.1.0",
+        "signal-exit": "^3.0.0",
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1",
+        "wide-align": "^1.1.0"
+      },
+      "dependencies": {
+        "is-fullwidth-code-point": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+          "dev": true,
+          "requires": {
+            "number-is-nan": "^1.0.0"
+          }
+        },
+        "string-width": {
+          "version": "1.0.2",
+          "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+          "dev": true,
+          "requires": {
+            "code-point-at": "^1.0.0",
+            "is-fullwidth-code-point": "^1.0.0",
+            "strip-ansi": "^3.0.0"
+          }
+        }
+      }
+    },
     "get-stdin": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz",
@@ -3951,6 +4188,12 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=",
+      "dev": true
+    },
     "glob": {
       "version": "7.1.2",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
@@ -4089,6 +4332,12 @@
         "is-glob": "^2.0.1"
       }
     },
+    "has-unicode": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
+      "dev": true
+    },
     "home-or-tmp": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-2.0.0.tgz",
@@ -4865,6 +5114,118 @@
         "set-getter": "^0.1.0"
       }
     },
+    "level": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/level/-/level-4.0.0.tgz",
+      "integrity": "sha512-4epzCOlEcJ529NOdlAYiuiakS/kZTDdiKSBNJmE1B8bsmA+zEVwcpxyH86qJSQTpOu7SODrlaD9WgPRHLkGutA==",
+      "dev": true,
+      "requires": {
+        "level-packager": "^3.0.0",
+        "leveldown": "^4.0.0",
+        "opencollective-postinstall": "^2.0.0"
+      }
+    },
+    "level-codec": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.0.tgz",
+      "integrity": "sha512-OIpVvjCcZNP5SdhcNupnsI1zo5Y9Vpm+k/F1gfG5kXrtctlrwanisakweJtE0uA0OpLukRfOQae+Fg0M5Debhg==",
+      "dev": true
+    },
+    "level-errors": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.0.tgz",
+      "integrity": "sha512-AmY4HCp9h3OiU19uG+3YWkdELgy05OTP/r23aNHaQKWv8DO787yZgsEuGVkoph40uwN+YdUKnANlrxSsoOaaxg==",
+      "dev": true,
+      "requires": {
+        "errno": "~0.1.1"
+      }
+    },
+    "level-iterator-stream": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-3.0.1.tgz",
+      "integrity": "sha512-nEIQvxEED9yRThxvOrq8Aqziy4EGzrxSZK+QzEFAVuJvQ8glfyZ96GB6BoI4sBbLfjMXm2w4vu3Tkcm9obcY0g==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.3.6",
+        "xtend": "^4.0.0"
+      },
+      "dependencies": {
+        "process-nextick-args": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+          "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+          "dev": true
+        },
+        "readable-stream": {
+          "version": "2.3.6",
+          "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+          "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+          "dev": true,
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "dev": true,
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
+    "level-packager": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-3.1.0.tgz",
+      "integrity": "sha512-UxVEfK5WH0u0InR3WxTCSAroiorAGKzXWZT6i+nBjambmvINuXFUsFx2Ai3UIjUUtnyWhluv42jMlzUZCsAk9A==",
+      "dev": true,
+      "requires": {
+        "encoding-down": "~5.0.0",
+        "levelup": "^3.0.0"
+      }
+    },
+    "leveldown": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-4.0.1.tgz",
+      "integrity": "sha512-ZlBKVSsglPIPJnz4ggB8o2R0bxDxbsMzuQohbfgoFMVApyTE118DK5LNRG0cRju6rt3OkGxe0V6UYACGlq/byg==",
+      "dev": true,
+      "requires": {
+        "abstract-leveldown": "~5.0.0",
+        "bindings": "~1.3.0",
+        "fast-future": "~1.0.2",
+        "nan": "~2.10.0",
+        "prebuild-install": "^4.0.0"
+      },
+      "dependencies": {
+        "nan": {
+          "version": "2.10.0",
+          "resolved": "http://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
+          "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==",
+          "dev": true
+        }
+      }
+    },
+    "levelup": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/levelup/-/levelup-3.1.1.tgz",
+      "integrity": "sha512-9N10xRkUU4dShSRRFTBdNaBxofz+PGaIZO962ckboJZiNmLuhVT6FZ6ZKAsICKfUBO76ySaYU6fJWX/jnj3Lcg==",
+      "dev": true,
+      "requires": {
+        "deferred-leveldown": "~4.0.0",
+        "level-errors": "~2.0.0",
+        "level-iterator-stream": "~3.0.0",
+        "xtend": "~4.0.0"
+      }
+    },
     "levn": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
@@ -5282,6 +5643,12 @@
       "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
       "dev": true
     },
+    "mimic-response": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
+      "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
+      "dev": true
+    },
     "minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -5350,6 +5717,23 @@
       "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=",
       "dev": true
     },
+    "node-abi": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.5.0.tgz",
+      "integrity": "sha512-9g2twBGSP6wIR5PW7tXvAWnEWKJDH/VskdXp168xsw9VVxpEGov8K4jsP4/VeoC7b2ZAyzckvMCuQuQlw44lXg==",
+      "dev": true,
+      "requires": {
+        "semver": "^5.4.1"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.6.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
+          "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==",
+          "dev": true
+        }
+      }
+    },
     "node-fetch": {
       "version": "1.7.3",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
@@ -5360,6 +5744,12 @@
         "is-stream": "^1.0.1"
       }
     },
+    "noop-logger": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
+      "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=",
+      "dev": true
+    },
     "normalize-package-data": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
@@ -5381,6 +5771,18 @@
         "remove-trailing-separator": "^1.0.1"
       }
     },
+    "npmlog": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
+      "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
+      "dev": true,
+      "requires": {
+        "are-we-there-yet": "~1.1.2",
+        "console-control-strings": "~1.1.0",
+        "gauge": "~2.7.3",
+        "set-blocking": "~2.0.0"
+      }
+    },
     "nth-check": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz",
@@ -5465,6 +5867,12 @@
         "mimic-fn": "^1.0.0"
       }
     },
+    "opencollective-postinstall": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz",
+      "integrity": "sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ==",
+      "dev": true
+    },
     "optionator": {
       "version": "0.8.2",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
@@ -5736,6 +6144,37 @@
       "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==",
       "dev": true
     },
+    "prebuild-install": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-4.0.0.tgz",
+      "integrity": "sha512-7tayxeYboJX0RbVzdnKyGl2vhQRWr6qfClEXDhOkXjuaOKCw2q8aiuFhONRYVsG/czia7KhpykIlI2S2VaPunA==",
+      "dev": true,
+      "requires": {
+        "detect-libc": "^1.0.3",
+        "expand-template": "^1.0.2",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.0",
+        "mkdirp": "^0.5.1",
+        "node-abi": "^2.2.0",
+        "noop-logger": "^0.1.1",
+        "npmlog": "^4.0.1",
+        "os-homedir": "^1.0.1",
+        "pump": "^2.0.1",
+        "rc": "^1.1.6",
+        "simple-get": "^2.7.0",
+        "tar-fs": "^1.13.0",
+        "tunnel-agent": "^0.6.0",
+        "which-pm-runs": "^1.0.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
+          "dev": true
+        }
+      }
+    },
     "prelude-ls": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
@@ -5938,6 +6377,12 @@
         "prosemirror-transform": "^1.1.0"
       }
     },
+    "prr": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
+      "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
+      "dev": true
+    },
     "pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
@@ -5951,6 +6396,16 @@
       "dev": true,
       "optional": true
     },
+    "pump": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
+      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+      "dev": true,
+      "requires": {
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.1"
+      }
+    },
     "punycode": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
@@ -6036,6 +6491,26 @@
       "integrity": "sha1-vOeMkhsjWCuuIR9ZdGSlkf0alPk=",
       "dev": true
     },
+    "rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "dev": true,
+      "requires": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
+          "dev": true
+        }
+      }
+    },
     "read-pkg": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz",
@@ -6685,6 +7160,12 @@
         }
       }
     },
+    "set-blocking": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
+      "dev": true
+    },
     "set-getter": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/set-getter/-/set-getter-0.1.0.tgz",
@@ -6733,6 +7214,23 @@
       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
       "dev": true
     },
+    "simple-concat": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz",
+      "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=",
+      "dev": true
+    },
+    "simple-get": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz",
+      "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==",
+      "dev": true,
+      "requires": {
+        "decompress-response": "^3.3.0",
+        "once": "^1.3.1",
+        "simple-concat": "^1.0.0"
+      }
+    },
     "slash": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
@@ -7061,6 +7559,45 @@
       "integrity": "sha1-e/gQalwaSCUbPjvAoOFzJIn9Dcg=",
       "dev": true
     },
+    "tar-fs": {
+      "version": "1.16.3",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz",
+      "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==",
+      "dev": true,
+      "requires": {
+        "chownr": "^1.0.1",
+        "mkdirp": "^0.5.1",
+        "pump": "^1.0.0",
+        "tar-stream": "^1.1.2"
+      },
+      "dependencies": {
+        "pump": {
+          "version": "1.0.3",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz",
+          "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==",
+          "dev": true,
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        }
+      }
+    },
+    "tar-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
+      "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
+      "dev": true,
+      "requires": {
+        "bl": "^1.0.0",
+        "buffer-alloc": "^1.2.0",
+        "end-of-stream": "^1.0.0",
+        "fs-constants": "^1.0.0",
+        "readable-stream": "^2.3.0",
+        "to-buffer": "^1.1.1",
+        "xtend": "^4.0.0"
+      }
+    },
     "text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -7088,6 +7625,12 @@
         "os-tmpdir": "~1.0.2"
       }
     },
+    "to-buffer": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
+      "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==",
+      "dev": true
+    },
     "to-fast-properties": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
@@ -7152,7 +7695,6 @@
       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
       "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
       "dev": true,
-      "optional": true,
       "requires": {
         "safe-buffer": "^5.0.1"
       }
@@ -7382,6 +7924,21 @@
         "isexe": "^2.0.0"
       }
     },
+    "which-pm-runs": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
+      "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=",
+      "dev": true
+    },
+    "wide-align": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
+      "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
+      "dev": true,
+      "requires": {
+        "string-width": "^1.0.2 || 2"
+      }
+    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",

package.json

@@ -96,12 +96,11 @@
     "rollup-regenerator-runtime": "^6.23.1",
     "rollup-watch": "^3.2.2",
     "standard": "^11.0.1",
-    "tui-jsdoc-template": "^1.2.2"
+    "tui-jsdoc-template": "^1.2.2",
+    "codemirror": "^5.42.0"
   },
   "optionalDependencies": {
+    "level": "^4.0.0",
     "ws": "^6.1.0"
-  },
-  "dependencies": {
-    "codemirror": "^5.42.0"
   }
 }

persistence/leveldb.js

@@ -0,0 +1,125 @@
+/**
+ * @module persistence/leveldb
+ * This module re-uses the encoding of syncProtocol to store and read updates from leveldb.
+ */
+
+const level = require('level')
+const Y = require('../build/yjs.js')
+const mux = Y.createMutex()
+
+/*
+ * Improves the uniqueness of timestamps.
+ * We gamble with the fact that users won't create more than 10000 changes on a single document 
+ * within one millisecond (also assuming clock works correctly).
+ */
+let timestampIterator = 0
+/**
+ * @return {string} A random, time-based string starting with "${roomName}:"
+ */
+const getNextTimestamp = () => {
+  timestampIterator = (timestampIterator + 1) % 10000
+  return `${Date.now()}${timestampIterator.toString().padStart(4, '0')}`
+}
+
+/**
+ * @param {string} docName 
+ * @return {string}
+ */
+const generateEntryKey = docName => `${docName}#${getNextTimestamp()}`
+
+/**
+ * 
+ * @param {any} db
+ * @param {string} docName
+ * @param {Uint8Array | ArrayBuffer} buf
+ */
+const writeEntry = (db, docName, buf) => db.put(generateEntryKey(docName), buf)
+
+/**
+ * @param {Uint8Array} arr
+ * @param {Y.Y} ydocument
+ */
+const readEntry = (arr, ydocument) => mux(() =>
+  Y.syncProtocol.readSyncMessage(Y.decoding.createDecoder(arr), Y.encoding.createEncoder(), ydocument)
+)
+
+/**
+ * @param {any} db 
+ * @param {string} docName 
+ * @param {Y.Y} ydocument 
+ */
+const loadFromPersistence = (db, docName, ydocument) => new Promise((resolve, reject)=>
+  db.createReadStream({
+    gte: `${docName}#`,
+    lte: `${docName}#Z`,
+    keys: false,
+    values: true
+  })
+  .on('data', data => readEntry(data, ydocument))
+  .on('error', reject)
+  .on('end', resolve)
+  .on('close', resolve)
+)
+
+const persistState = (db, docName, ydocument) => {
+  const encoder = Y.encoding.createEncoder()
+  Y.syncProtocol.writeSyncStep2(encoder, ydocument, new Map())
+  const entryKey = generateEntryKey(docName)
+  const entryPromise = db.put(entryKey, Y.encoding.toBuffer(encoder))
+  const delOps = []
+  return new Promise((resolve, reject) => db.createKeyStream({
+      gte: `${docName}#`,
+      lt: entryKey,
+    })
+    .on('data', key => delOps.push({ type: 'del', key }))
+    .on('error', reject)
+    .on('end', resolve)
+    .on('close', resolve)
+  ).then(() => entryPromise).then(() => db.batch(delOps))
+}
+
+/**
+ * Persistence layer for Leveldb.
+ */
+exports.LevelDbPersistence = class LevelDbPersistence {
+  /**
+   * @param {string} fpath Path to leveldb database
+   */
+  constructor (fpath) {
+    this.db = level(fpath, { valueEncoding: 'binary' })
+  }
+  /**
+   * Retrieve all data from LevelDB and automatically persist all document updates to leveldb.
+   *
+   * @param {string} docName
+   * @param {Y.Y} ydocument 
+   */
+  bindState (docName, ydocument) {
+    // write all updates received from other clients
+    // - unless it is created by this persistence layer (e.g. loadFromPersistence, we we mux).
+    ydocument.on('afterTransaction', (y, transaction) => {
+      if (transaction.encodedStructsLen > 0) {
+        mux(() => {
+          const encoder = Y.encoding.createEncoder()
+          Y.syncProtocol.writeUpdate(encoder, transaction.encodedStructsLen, transaction.encodedStructs)
+          writeEntry(this.db, docName, Y.encoding.toBuffer(encoder))
+        })
+      }
+    })
+    // read all data from persistence
+    return loadFromPersistence(this.db, docName, ydocument).then(() =>
+      // write current state (just in case anything was added before state was bound)
+      this.writeState(docName, ydocument)
+    )
+  }
+  /**
+   * Write current state to persistence layer. Deletes all entries that were made before.
+   * Call this method at any time - the recommended time to call this method is before the ydocument is destroyed.
+   *
+   * @param {string} docName 
+   * @param {Y.Y} ydocument 
+   */
+  writeState (docName, ydocument) {
+    return persistState(this.db, docName, ydocument)
+  }
+}

protocols/sync.js

@@ -331,6 +331,17 @@ export const writeSyncStep1 = (encoder, y) => {
   writeStateSet(encoder, y)
 }
 
+/**
+ * @param {encoding.Encoder} encoder
+ * @param {Y} y
+ * @param {Map<number, number>} ss
+ */
+export const writeSyncStep2 = (encoder, y, ss) => {
+  encoding.writeVarUint(encoder, messageYjsSyncStep2)
+  writeStructs(encoder, y, ss)
+  writeDeleteSet(encoder, y)
+}
+
 /**
  * Read SyncStep1 message and reply with SyncStep2.
  *
@@ -338,14 +349,8 @@ export const writeSyncStep1 = (encoder, y) => {
  * @param {encoding.Encoder} encoder The received message
  * @param {Y} y
  */
-export const readSyncStep1 = (decoder, encoder, y) => {
+export const readSyncStep1 = (decoder, encoder, y) =>
-  // read sync step 1 message
+  writeSyncStep2(encoder, y, readStateSet(decoder))
-  const ss = readStateSet(decoder)
-  // write sync step 2
-  encoding.writeVarUint(encoder, messageYjsSyncStep2)
-  writeStructs(encoder, y, ss)
-  writeDeleteSet(encoder, y)
-}
 
 /**
  * @param {decoding.Decoder} decoder

provider/websocket/server.js

@@ -12,6 +12,13 @@ const http = require('http')
 
 const port = process.env.PORT || 1234
 
+const persistenceDir = process.env.YPERSISTENCE
+let persistence = null
+if (typeof persistenceDir) {
+  const LevelDbPersistence = require('../../persistence/leveldb.js').LevelDbPersistence
+  persistence = new LevelDbPersistence(persistenceDir)
+}
+
 const server = http.createServer((req, res) => {
   res.writeHead(200, { 'Content-Type': 'text/plain' })
   res.end('okay')
@@ -80,10 +87,14 @@ const messageListener = (conn, doc, message) => {
 const setupConnection = (conn, req) => {
   conn.binaryType = 'arraybuffer'
   // get doc, create if it does not exist yet
-  let doc = docs.get(req.url.slice(1))
+  const docName = req.url.slice(1)
+  let doc = docs.get(docName)
   if (doc === undefined) {
     doc = new WSSharedDoc()
-    docs.set(req.url.slice(1), doc)
+    if (persistence !== null) {
+      persistence.bindState(docName, doc)
+    }
+    docs.set(docName, doc)
   }
   doc.conns.set(conn, new Set())
   // listen and reply to events
@@ -99,6 +110,13 @@ const setupConnection = (conn, req) => {
     }))
     const buf = Y.encoding.toBuffer(encoder)
     doc.conns.forEach((_, conn) => conn.send(buf))
+    if (doc.conns.size === 0 && persistence !== null) {
+      // if persisted, we store state and destroy ydocument
+      persistence.writeState(docName, doc).then(() => {
+        doc.destroy()
+      })
+      docs.delete(docName)
+    }
   })
   // send sync step 1
   const encoder = Y.encoding.createEncoder()

tmp/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000004

tmp/LOG

@@ -0,0 +1,5 @@
+2018/12/22-13:33:44.376775 70000c740000 Recovering log #3
+2018/12/22-13:33:44.377203 70000c740000 Level-0 table #5: started
+2018/12/22-13:33:44.379580 70000c740000 Level-0 table #5: 4834 bytes OK
+2018/12/22-13:33:44.380496 70000c740000 Delete type=0 #3
+2018/12/22-13:33:44.380670 70000c740000 Delete type=3 #2

tmp/LOG.old

@@ -0,0 +1 @@
+2018/12/22-13:23:54.104944 700007836000 Delete type=3 #1