diff --git a/.gitignore b/.gitignore index bc37d4a..cbb6960 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ node_modules # Private filter list private.filter # Logs -logs \ No newline at end of file +logs +# Database +/db diff --git a/README.md b/README.md index 1a6862d..79d1b19 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,44 @@ # cloud-server -A cloud data server for Scratch 3. Used by [forkphorus](https://forkphorus.github.io/) and [TurboWarp](https://turbowarp.org/). +A configurable cloud data server for Scratch 3. -It uses a protocol very similar to Scratch 3's cloud variable protocol. See doc/protocol.md for further details. +Used by: -## Restrictions - -This server does not implement long term variable storage. All data is stored only in memory (never on disk) and are removed promptly when rooms are emptied or the server restarts. - -This server also does not implement history logs. + - [TurboWarp](https://turbowarp.org/) + - [forkphorus](https://forkphorus.github.io/) ## Setup -Needs Node.js and npm. +Requires Node.js and npm. ``` git clone https://github.com/TurboWarp/cloud-server cd cloud-server -npm install +npm ci npm start ``` -By default the server is listening on ws://localhost:9080/. To change the port or enable wss://, read below. +By default, the server listens on ws://localhost:9080/. This is good enough for local testing, but if you want to deploy this on a real website, you will probably need to setup a secure wss://. -To use a local cloud variable server in forkphorus, you can use the `chost` URL parameter, for example: https://forkphorus.github.io/?chost=ws://localhost:9080/ +To test locally in TurboWarp, use the `cloud_host` URL parameter: https://turbowarp.org/?cloud_host=ws://localhost:9080/ -You can do a similar thing in TurboWarp with the `cloud_host` URL parameter: https://turbowarp.org/?cloud_host=ws://localhost:9080/ +To test locally in forkphorus, uses the `chost` URL parameter: https://forkphorus.github.io/?chost=ws://localhost:9080/ ## Configuration HTTP requests are served static files in the `public` directory. -### src/config.js - -src/config.js is the configuration file for cloud-server. - -The `port` property (or the `PORT` environment variable) configures the port to listen on. - -On unix-like systems, port can also be a path to a unix socket. By default cloud-server will set the permission of unix sockets to `777`. This can be configured with `unixSocketPermissions`. - -If you use a reverse proxy, set the `trustProxy` property (or `TRUST_PROXY` environment variable) to `true` so that logs contain the user's IP address instead of your proxy's. +Edit src/config.js for additional settings. There's a lot of stuff in there and it's all documented. We will highlight a couple important settings below: -Set `anonymizeAddresses` to `true` if you want IP addresses to be not be logged. +### Database -Set `perMessageDeflate` to an object to enable "permessage-deflate", which uses compression to reduce the bandwidth of data transfers. This can lead to poor performance and catastrophic memory fragmentation on Linux (https://github.com/nodejs/node/issues/8871). See here for options: https://github.com/websockets/ws/blob/master/doc/ws.md#new-websocketserveroptions-callback (look for `perMessageDeflate`) +By default, cloud-server does not use a database, so any variables will be lost when no one has been in a project for a short while or the server restarts. -You can configure logging with the `logging` property of src/config.js. By default cloud-server logs to stdout and to files in the `logs` folder. stdout logging can be disabled by setting `logging.console` to false. File logging is configured with `logging.rotation`, see here for options: https://github.com/winstonjs/winston-daily-rotate-file#options. Set to false to disable. +In src/config.js, you can change the `database` option to `'sqlite'` to instead persist variables in an sqlite database. Variables are saved when a room is automatically deleted for being empty and on a periodic schedule set by `autosaveInterval`. If the server dies unexpectedly, anything that wasn't yet saved will not be saved. -### Production setup +## Production setup -cloud-server is considered production ready as it has been in use in a production environment for months without issue. That said, there is no warranty. If a bug in cloud-server results in you losing millions of dollars, tough luck. (see LICENSE for more details) +For this section, we will assume you have complete access to a Linux machine as this is the ideal environment for cloud-server. You should probably be using a reverse proxy such as nginx or caddy in a production environment. diff --git a/package-lock.json b/package-lock.json index 0012877..0f055fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1094,6 +1094,38 @@ } } }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "better-sqlite3": { + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.6.2.tgz", + "integrity": "sha512-S5zIU1Hink2AH4xPsN0W43T1/AJ5jrPh7Oy07ocuW/AKYYY02GWzz9NH0nbSMn/gw6fDZ5jZ1QsHt1BXAwJ6Lg==", + "requires": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.0" + } + }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "requires": { + "file-uri-to-path": "1.0.0" + } + }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1128,6 +1160,15 @@ "node-int64": "^0.4.0" } }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -1197,6 +1238,11 @@ "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", "dev": true }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", @@ -1431,6 +1477,19 @@ "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", "dev": true }, + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "requires": { + "mimic-response": "^3.1.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1500,6 +1559,11 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, + "detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==" + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -1560,7 +1624,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -1698,6 +1761,11 @@ } } }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "expect": { "version": "26.6.2", "resolved": "https://registry.npmjs.org/expect/-/expect-26.6.2.tgz", @@ -1832,6 +1900,11 @@ "moment": "^2.29.1" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1916,6 +1989,11 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1968,6 +2046,11 @@ "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", "dev": true }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -2144,6 +2227,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "import-local": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", @@ -2175,6 +2263,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, "is-accessor-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", @@ -3094,7 +3187,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "requires": { "yallist": "^4.0.0" } @@ -3182,6 +3274,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -3194,8 +3291,7 @@ "minimist": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", - "dev": true + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "mixin-deep": { "version": "1.3.2", @@ -3218,6 +3314,11 @@ } } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "moment": { "version": "2.29.3", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz", @@ -3247,6 +3348,11 @@ "to-regex": "^3.0.1" } }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3259,6 +3365,53 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-abi": { + "version": "3.24.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.24.0.tgz", + "integrity": "sha512-YPG3Co0luSu6GwOBsmIdGW6Wx0NyNDLg/hriIyDllVsNwnI6UeqaWShxC3lbH4LtEQUgoLP3XR1ndXiDAWvmRw==", + "requires": { + "semver": "^7.3.5" + }, + "dependencies": { + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "requires": { + "lru-cache": "^6.0.0" + } + } + } + }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, "node-gyp-build": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.3.0.tgz", @@ -3413,7 +3566,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1" } @@ -3568,6 +3720,25 @@ "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", "dev": true }, + "prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -3606,7 +3777,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -3623,6 +3793,17 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, "react-is": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.1.tgz", @@ -3765,8 +3946,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safe-regex": { "version": "1.1.0", @@ -4072,6 +4252,21 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" + }, + "simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "simple-swizzle": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", @@ -4420,6 +4615,11 @@ "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -4445,6 +4645,29 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + }, "terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", @@ -4560,6 +4783,14 @@ "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -4887,8 +5118,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write-file-atomic": { "version": "3.0.3", @@ -4928,8 +5158,7 @@ "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yargs": { "version": "15.4.1", diff --git a/package.json b/package.json index c1331f0..8f4b366 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,9 @@ "author": "GarboMuffin", "license": "MIT", "dependencies": { + "better-sqlite3": "^7.6.2", "finalhandler": "^1.2.0", + "node-fetch": "^2.6.7", "serve-static": "^1.15.0", "winston": "^3.7.2", "winston-daily-rotate-file": "^4.7.1", diff --git a/public/index.html b/public/index.html index 79ced78..e53f643 100644 --- a/public/index.html +++ b/public/index.html @@ -10,9 +10,9 @@ } .label { font-size: 9.45pt; - font-weight: bold; color: #575E75; white-space: pre; + margin: 5px 0; } .var { display: inline-block; @@ -36,7 +36,18 @@

Cloud Data Server

- Source Code + +
Source Code
+ +
+ diff --git a/public/robots.txt b/public/robots.txt deleted file mode 100644 index 1f53798..0000000 --- a/public/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -User-agent: * -Disallow: / diff --git a/src/ConnectionError.js b/src/ConnectionError.js index f32553c..522ea93 100644 --- a/src/ConnectionError.js +++ b/src/ConnectionError.js @@ -14,5 +14,6 @@ class ConnectionError extends Error { ConnectionError.Error = 4000; ConnectionError.Username = 4002; ConnectionError.Overloaded = 4003; +ConnectionError.ProjectUnavailable = 4004; module.exports = ConnectionError; diff --git a/src/ConnectionManager.js b/src/ConnectionManager.js index 75de8a0..62fcf23 100644 --- a/src/ConnectionManager.js +++ b/src/ConnectionManager.js @@ -27,6 +27,11 @@ class ConnectionManager { logger.info(`Pinging ${this.clients.size} clients...`); } this.clients.forEach((client) => { + if (!client.ws) { + client.timedOut('no ws'); + return; + } + if (!client.respondedToPing) { // Clients that have not responded to the most recent ping are considered dead. client.timedOut('no pong'); diff --git a/src/Room.js b/src/Room.js index 581cdda..9ca85c2 100644 --- a/src/Room.js +++ b/src/Room.js @@ -1,3 +1,5 @@ +const config = require('./config'); + /** * @typedef {import('./Client')} Client */ @@ -41,11 +43,11 @@ class Room { /** * Maximum number of variables that can be within this room. */ - this.maxVariables = 20; + this.maxVariables = config.maxVariablesPerRoom; /** * Maximum number of clients that can be connected to this room. */ - this.maxClients = 128; + this.maxClients = config.maxClientsPerRoom; } /** @@ -93,6 +95,26 @@ class Room { return this.variables; } + /** + * @returns {boolean} true if there are any variables in the room. + */ + hasAnyVariables() { + return this.variables.size !== 0; + } + + /** + * Get an object of all variables. + * @returns {Record} + */ + getAllVariablesAsObject() { + /** @type {Record} */ + const result = {}; + for (const [name, value] of this.variables.entries()) { + result[name] = value; + } + return result; + } + /** * Create a new variable. * This method does not inform clients of the change. @@ -124,6 +146,15 @@ class Room { this.variables.set(name, value); } + /** + * Forcibly set or create a variable's value. Ignores maximum variable restriction. + * @param {string} name + * @param {Value} value + */ + forceSet(name, value) { + this.variables.set(name, value); + } + /** * Delete a variable. * @param {string} name The name of the variable diff --git a/src/RoomList.js b/src/RoomList.js index d1ef09e..c7a8850 100644 --- a/src/RoomList.js +++ b/src/RoomList.js @@ -1,13 +1,12 @@ const Room = require('./Room'); const ConnectionError = require('./ConnectionError'); const logger = require('./logger'); +const config = require('./config'); +const db = require('./db'); -/** Delay between janitor runs. */ -const JANITOR_INTERVAL = 1000 * 60; -/** Time a room must be empty for before it may be removed by the janitor. */ -const JANITOR_THRESHOLD = 1000 * 60 * 60; -/** Maximum amount of rooms that can exist at once. Empty rooms are included in this limit. */ -const MAX_ROOMS = 1024; +const JANITOR_THRESHOLD = config.emptyRoomLife * 1000; +const JANITOR_INTERVAL = JANITOR_THRESHOLD * config.emptyRoomLifeInterval; +const MAX_ROOMS = config.maxRooms; /** * @typedef {import('./Room').RoomID} RoomID @@ -29,8 +28,11 @@ class RoomList { /** Enable or disable logging of events to the console. */ this.enableLogging = false; this.janitor = this.janitor.bind(this); + this.autosave = this.autosave.bind(this); /** @private */ this.janitorInterval = null; + /** @private */ + this.autosaveInterval = null; } /** @@ -76,6 +78,14 @@ class RoomList { if (this.enableLogging) { logger.info('Created room: ' + id); } + + const initialData = db.getVariables(id); + if (initialData) { + for (const variableName of Object.keys(initialData)) { + room.forceSet(variableName, initialData[variableName]); + } + } + return room; } @@ -86,6 +96,9 @@ class RoomList { */ remove(id) { const room = this.get(id); + + db.setVariables(room.id, room.getAllVariablesAsObject()); + if (room.getClients().length > 0) { throw new Error('Clients are connected to this room'); } @@ -117,11 +130,16 @@ class RoomList { } } - /** - * Begin the janitor timer. - */ - startJanitor() { - this.janitorInterval = setInterval(this.janitor, JANITOR_INTERVAL) + autosave() { + logger.info('Autosaving'); + for (const room of this.rooms.values()) { + db.setVariables(room.id, room.getAllVariablesAsObject()); + } + } + + startIntervals() { + this.janitorInterval = setInterval(this.janitor, JANITOR_INTERVAL); + this.autosaveInterval = setInterval(this.autosave, config.autosaveInterval * 1000); } /** @@ -132,6 +150,9 @@ class RoomList { if (this.janitorInterval) { clearInterval(this.janitorInterval); } + if (this.autosaveInterval) { + clearInterval(this.autosaveInterval); + } } } diff --git a/src/config.js b/src/config.js index a4ec202..324fcc6 100644 --- a/src/config.js +++ b/src/config.js @@ -1,38 +1,98 @@ module.exports = { - // port for the server to listen on - // on unix-like platforms, this can be the path to a unix socket + // Port for the server to listen on + // On unix-like platforms, this can be the path to a unix socket + // PORT is a common environment variable used used by various hosting services. port: process.env.PORT || 9080, - // the unix permissions to use for unix sockets - // set to -1 to disable permission changing - // make sure to use an octal (`0o`) instead of just a regular number + // The unix permissions to use when port is set to a unix socket. + // Set to -1 to use default permissions. + // Recommended to use an octal number (0o) so that it looks similar to what you + // would put into chmod. unixSocketPermissions: 0o777, - // enable to read x-forwarded-for - trustProxy: process.env.TRUST_PROXY === 'true', + // The database to use. Either: + // - 'none' for no database + // - 'sqlite' to use a persistent sqlite database + database: 'none', - // removes IP addresses from logs - anonymizeAddresses: process.env.ANONYMIZE_ADDRESSES === 'true', + // Time, in seconds, between automatic database saves. Variables are also saved + // when a room is deleted for being empty for too long. + autosaveInterval: 60 * 60, - // anonymize generated usernames like "player123456" - anonymizeGeneratedUsernames: true, + // The maximum length of a variable value to allow. Values longer than this will + // be silently ignored. + maxValueLength: 100000, + + // Whether to allow variable values that aren't numbers. + allowNonNumberValues: false, + + // The maximum number of variables to allow in one room. Additional variables will + // not be allowed. + maxVariablesPerRoom: 20, - // change this to an object to enable the WebSocket per-message deflate extension + // The maximum number of people that can be connected to a room at the same time. + // If another person tries to connect, their connection will be closed. + maxClientsPerRoom: 128, + + // If this is set to true, the server will validate usernames by talking to the + // Scratch API to check that an account with that username exists. Usernames that + // do not exist will be rejected. + validateUsernames: false, + + // Configures WebSocket per-message compression. + // This can allow significant bandwidth reduction, but it can use a lot of CPU + // and may cause catastrophic memory fragmentation on Linux. + // This value is passed directly into ws's perMessageDeflate option: + // https://github.com/websockets/ws#websocket-compression perMessageDeflate: false, - // If set to a non-zero number, sends will be buffered to this many per second - // This can significantly improve performance - bufferSends: 60, + // If this is set to a non-zero number, outgoing variable updates will be sent + // in batches this many times per second instead of immediately. + // This can significantly reduce CPU and network usage in projects with many + // active clients. + bufferSends: 30, + // Enables variable renaming enableRename: false, + // Enables variable deleting enableDelete: false, + // emptyRoomLife is the maximum-ish time, in seconds, a room will be kept in memory while no + // one is connected to it before it is automatically deleted. Depending on what the + // database option is set to, the variables will either be lost forever or saved to disk. + // (real maximum will be slightly higher: see emptyRoomLifeInterval below) + emptyRoomLife: 60 * 60, + + // emptyRoomLife is enforced by a periodic timer. emptyRoomLife is multiplied by + // emptyRoomLifeInterval to determine how often this timer should run. + // For example, if emptyRoomLife is 60 minutes (3600 seconds) and this is 0.1, there will be + // 0.1 * 60 minutes = 6 minutes (360 seconds) between each check. This will allow a room + // to actually be empty for 66 minutes before being removed. + emptyRoomLifeInterval: 0.1, + + // The maximum number of cloud variable rooms that can exist at once. + // Empty rooms are included in this limit until they are removed by emptyRoomLife and + // emptyRoomLifeInterval. + maxRooms: 1024, + + // If you're behind a reverse proxy such as nginx or Cloudflare, this can be + // enabled so that the logs use the IP given by x-forwarded-for instead of the + // address of the socket. + trustProxy: false, + + // Removes IP addresses from logs + anonymizeAddresses: false, + + // Anonymize generated usernames like "player123456" to just "player" in logs. + anonymizeGeneratedUsernames: true, + logging: { + // Whether logs should be printed to the console or not. console: true, - // passed directly into winston-daily-rotate-file - // see here for options: https://github.com/winstonjs/winston-daily-rotate-file#options + // Passed directly into winston-daily-rotate-file + // For options, see: https://github.com/winstonjs/winston-daily-rotate-file#options rotation: { filename: '%DATE%.log', // LOGS_DIRECTORY is used by systemd services with the LogsDirectory= directive diff --git a/src/db/index.js b/src/db/index.js new file mode 100644 index 0000000..4f40d8f --- /dev/null +++ b/src/db/index.js @@ -0,0 +1,12 @@ +const config = require('../config'); +const logger = require('../logger'); + +if (config.database === 'none') { + logger.info('Using ephemeral database (none)'); + module.exports = require('./none'); +} else if (config.database === 'sqlite') { + logger.info('Using sqlite database'); + module.exports = require('./sqlite'); +} else { + throw new Error(`Unknown database: ${config.database}`); +} diff --git a/src/db/none.js b/src/db/none.js new file mode 100644 index 0000000..8fecdbe --- /dev/null +++ b/src/db/none.js @@ -0,0 +1,13 @@ +const getVariables = (_id) => { + // No-op + return null; +}; + +const setVariables = (_id, _variables) => { + // No-op +}; + +module.exports = { + getVariables, + setVariables +}; diff --git a/src/db/sqlite.js b/src/db/sqlite.js new file mode 100644 index 0000000..48ac968 --- /dev/null +++ b/src/db/sqlite.js @@ -0,0 +1,38 @@ +const sqlite3 = require('better-sqlite3'); +const path = require('path'); +const fs = require('fs'); + +const folder = path.join(__dirname, '..', '..', 'db'); +fs.mkdirSync(folder, { + recursive: true +}); + +const VERSION = 1; +const db = new sqlite3(path.join(folder, `cloud-server-${VERSION}.db`)) +db.pragma('journal_mode = WAL'); +db.exec(` +CREATE TABLE IF NOT EXISTS variables ( + id TEXT PRIMARY KEY NOT NULL, + data TEXT NOT NULL +); +`); + +const getStatement = db.prepare('SELECT data FROM variables WHERE id=?;'); +const setStatement = db.prepare('INSERT OR REPLACE INTO variables (id, data) VALUES (?, ?);'); + +const getVariables = (id) => { + const data = getStatement.get(id); + if (data) { + return JSON.parse(data.data); + } + return null; +}; + +const setVariables = (id, variables) => { + setStatement.run(id, JSON.stringify(variables)); +}; + +module.exports = { + getVariables, + setVariables +}; diff --git a/src/filters/impersonation.filter b/src/filters/impersonation.filter index cbbfbe2..5a8c306 100644 --- a/src/filters/impersonation.filter +++ b/src/filters/impersonation.filter @@ -2,54 +2,66 @@ # These are exact matches. # This list was generated by visiting https://scratch.mit.edu/credits and running: # copy(Array.from(document.querySelectorAll('.avatar-item')).map(i=>i.querySelector('a').href).map(i=>i.split('/')[4]).map(i=>i.replace(/-|_/g,'')).map(i=>`^${i}$`).join('\n')) +^cosmosaura$ +^themorningsun$ +^LeopardFlash$ +^RulerOfTheQueendom$ ^originalwow$ ^amylaser$ ^achouse$ +^mogibear$ +^carpeediem$ ^wheelsonfire$ -^BrycedTea$ -^designerd$ -^tarmelop$ -^Champ99$ +^digitalgig$ +^beezy333$ ^chrisg$ ^cwillisf$ ^ceebee$ +^floralsunset$ ^codubee$ +^spectrespecs$ ^noncanonical$ +^Ohsohpy$ ^Harakou$ -^dsquare$ -^SunnyDay4aBlueJay$ +^BlueWillow78$ +^deeplikethesea$ +^NoodleKen11$ ^ericr$ ^speakvisually$ +^scratchererik$ ^cheddargirl$ +^fcoCCT$ ^pixelmoth$ -^jaleesa$ -^Mos20$ -^FredDog$ -^Class12321$ -^bluecrazie$ +^GourdVibesOnly$ ^MunchtheCat$ -^kittyloaf$ -^dinopickles$ -^pondermake$ -^khanning$ -^KayOh$ -^labdalla$ -^leoburd$ -^lilyland$ +^Roxie916$ +^lashaunan$ +^lamatchalattei$ +^shinyvase275$ +^JumpingRabbits$ +^ItzLinz$ ^algorithmar$ -^mwikali$ +^TheNuttyKnitter$ +^soutoam$ ^dietbacon$ ^Paddle2See$ -^mres$ -^natalie$ -^sgcc$ -^raimondious$ +^deism902$ +^pamimus$ +^rtrvmwe$ ^binnieb$ -^scmb1$ -^pizzafordessert$ -^shruti$ -^shaanmasala$ -^ZaChary$ +^delasare$ +^RupaLax$ +^Rhyolyte$ +^Onyx45$ +^RagingAvocado$ +^sgste735$ +^LT7845$ +^cardboardbee$ +^pandatt$ +^passiflora296$ +^ninja11013$ +^starrysky7$ +^YPhilip2006$ ^Zinnea$ # Other Scratch Team accounts not mentioned in the credits ^ScratchCat$ diff --git a/src/room-filters/index.js b/src/room-filters/index.js new file mode 100644 index 0000000..bd0275e --- /dev/null +++ b/src/room-filters/index.js @@ -0,0 +1,19 @@ +const fs = require('fs'); +const path = require('path'); +const logger = require('../logger'); + +const readBlockedProjects = () => { + try { + const contents = fs.readFileSync(path.join(__dirname, 'filter.txt'), 'utf-8'); + return contents.trim().split('\n'); + } catch (e) { + return []; + } +}; + +const blockedProjects = readBlockedProjects(); +logger.info(`Blocked projects: ${blockedProjects.length}`); + +const isProjectBlocked = (id) => blockedProjects.includes(id); + +module.exports = isProjectBlocked; diff --git a/src/server.js b/src/server.js index bfa7707..c286921 100644 --- a/src/server.js +++ b/src/server.js @@ -5,6 +5,8 @@ const RoomList = require('./RoomList'); const ConnectionError = require('./ConnectionError'); const ConnectionManager = require('./ConnectionManager'); const validators = require('./validators'); +const usernameUtils = require('./username'); +const isProjectBlocked = require('./room-filters'); const logger = require('./logger'); const naughty = require('./naughty'); const config = require('./config'); @@ -18,7 +20,7 @@ const wss = new WebSocket.Server({ const rooms = new RoomList(); rooms.enableLogging = true; -rooms.startJanitor(); +rooms.startIntervals(); const connectionManager = new ConnectionManager(); connectionManager.start(); @@ -101,25 +103,45 @@ wss.on('connection', (ws, req) => { const client = new Client(ws, req); + let isHandshaking = false; + let processAfterHandshakeQueue = []; + connectionManager.handleConnect(client); - function performHandshake(roomId, username) { + async function performHandshake(roomId, username) { if (client.room) throw new ConnectionError(ConnectionError.Error, 'Already performed handshake'); + + if (isHandshaking) throw new ConnectionError(ConnectionError.Error, 'Already handshaking'); + isHandshaking = true; + if (!validators.isValidRoomID(roomId)) { const roomToLog = `${roomId}`.substr(0, 100); throw new ConnectionError(ConnectionError.Error, 'Invalid room ID: ' + roomToLog); } - if (!validators.isValidUsername(username)) { + + if (isProjectBlocked(roomId)) { + throw new ConnectionError(ConnectionError.ProjectUnavailable, 'Project blocked: ' + roomId); + } + + if ( + !usernameUtils.isValidUsername(username) || + (config.validateUsernames && !await usernameUtils.isValidScratchAccount(username)) + ) { const usernameToLog = `${username}`.substr(0, 100); throw new ConnectionError(ConnectionError.Username, 'Invalid username: ' + usernameToLog); } + if (!client.ws || client.ws.readyState !== WebSocket.OPEN) { + // The connection was closed while we were validating the username. + return; + } + client.setUsername(username); - if (rooms.has(roomId)) { - const room = rooms.get(roomId); - client.setRoom(room); + const room = rooms.has(roomId) ? rooms.get(roomId) : rooms.create(roomId); + client.setRoom(room); + if (room.hasAnyVariables()) { // Send the data of all the variables in the room to the client. // This is done in one message by separating each "set" with a newline. /** @type {string[]} */ @@ -130,12 +152,16 @@ wss.on('connection', (ws, req) => { if (messages.length > 0) { client.send(messages.join('\n')); } - } else { - client.setRoom(rooms.create(roomId)); } // @ts-expect-error client.log(`Joined room (peers: ${client.room.getClients().length})`); + + isHandshaking = false; + for (const data of processAfterHandshakeQueue) { + processWithErrorHandling(data); + } + processAfterHandshakeQueue.length = 0; } function performCreate(variable, value) { @@ -196,15 +222,21 @@ wss.on('connection', (ws, req) => { } } - function processMessage(data) { + async function processMessage(data) { const message = parseMessage(data.toString()); const method = message.method; - switch (method) { - case 'handshake': - performHandshake('' + message.project_id, message.user); - break; + if (method === 'handshake') { + await performHandshake('' + message.project_id, message.user) + return; + } + if (isHandshaking) { + processAfterHandshakeQueue.push(data); + return; + } + + switch (method) { case 'set': performSet(message.name, message.value); break; @@ -226,6 +258,19 @@ wss.on('connection', (ws, req) => { } } + async function processWithErrorHandling(data) { + try { + await processMessage(data); + } catch (error) { + client.error('Error handling connection: ' + error); + if (error instanceof ConnectionError) { + client.close(error.code); + } else { + client.close(ConnectionError.Error); + } + } + } + client.log('Connection opened'); ws.on('message', (data, isBinary) => { @@ -237,16 +282,7 @@ wss.on('connection', (ws, req) => { return; } - try { - processMessage(data); - } catch (error) { - client.error('Error handling connection: ' + error); - if (error instanceof ConnectionError) { - client.close(error.code); - } else { - client.close(ConnectionError.Error); - } - } + processWithErrorHandling(data); }); ws.on('error', (error) => { diff --git a/src/tests/Room.test.js b/src/tests/Room.test.js index 7f18ecd..cf9e584 100644 --- a/src/tests/Room.test.js +++ b/src/tests/Room.test.js @@ -100,3 +100,12 @@ test('maxClients', () => { } expect(() => room.addClient(new Client(null, null))).toThrow(); }); + +test('hasAnyVariables', () => { + const room = new Room('1234'); + expect(room.hasAnyVariables()).toBe(false); + room.create('☁ foo', '1234'); + expect(room.hasAnyVariables()).toBe(true); + room.delete('☁ foo'); + expect(room.hasAnyVariables()).toBe(false); +}); diff --git a/src/tests/username.test.js b/src/tests/username.test.js index 2935a05..8214451 100644 --- a/src/tests/username.test.js +++ b/src/tests/username.test.js @@ -1,7 +1,7 @@ const username = require('../username'); test('isGenerated', () => { - expect(username.isGenerated('player')).toBe(false); + expect(username.isGenerated('player')).toBe(true); expect(username.isGenerated('player123')).toBe(true); expect(username.isGenerated('player123456')).toBe(true); expect(username.isGenerated('Player123456')).toBe(true); @@ -10,3 +10,20 @@ test('isGenerated', () => { expect(username.isGenerated('player_123')).toBe(false); expect(username.isGenerated('eplayer123')).toBe(false); }); + +test('isValidUsername', () => { + expect(username.isValidUsername(234)).toBe(false); + expect(username.isValidUsername(null)).toBe(false); + expect(username.isValidUsername(undefined)).toBe(false); + expect(username.isValidUsername(true)).toBe(false); + expect(username.isValidUsername(false)).toBe(false); + expect(username.isValidUsername([])).toBe(false); + expect(username.isValidUsername({})).toBe(false); + expect(username.isValidUsername('')).toBe(false); + expect(username.isValidUsername('griffpatch')).toBe(true); +}) + +test('isValidScratchAccount', async () => { + expect(await username.isValidScratchAccount('griffpatch')).toBe(true); + expect(await username.isValidScratchAccount('player' + Math.round(Math.random() * 10000000))).toBe(true); +}); diff --git a/src/tests/validators.test.js b/src/tests/validators.test.js index 2622986..d2d3bbe 100644 --- a/src/tests/validators.test.js +++ b/src/tests/validators.test.js @@ -1,67 +1,12 @@ const validators = require('../validators'); -test('isValidUsername', () => { - expect(validators.isValidUsername(234)).toBe(false); - expect(validators.isValidUsername('griffpatch')).toBe(true); - expect(validators.isValidUsername('gr1fF_p4tch')).toBe(true); - expect(validators.isValidUsername('griff patch')).toBe(false); - expect(validators.isValidUsername(' griffpatch')).toBe(false); - expect(validators.isValidUsername('abcdé')).toBe(false); - expect(validators.isValidUsername('')).toBe(false); - expect(validators.isValidUsername('e')).toBe(true); - expect(validators.isValidUsername('ee')).toBe(true); - expect(validators.isValidUsername('eee')).toBe(true); - expect(validators.isValidUsername('e'.repeat(19))).toBe(true); - expect(validators.isValidUsername('e'.repeat(20))).toBe(true); - expect(validators.isValidUsername('e'.repeat(21))).toBe(false); - expect(validators.isValidUsername('ScratchCat')).toBe(false); - expect(validators.isValidUsername(null)).toBe(false); - expect(validators.isValidUsername(undefined)).toBe(false); - expect(validators.isValidUsername(true)).toBe(false); - expect(validators.isValidUsername(false)).toBe(false); - expect(validators.isValidUsername([])).toBe(false); - expect(validators.isValidUsername({})).toBe(false); - // let's test some real usernames (or slight variations) - // no particular meaning in any of these names, just what I could find in the moment - // the idea is that if we get a big enough variety of names here, any problems in isValidUsername and especially naughty word detection will be caught - expect(validators.isValidUsername('griffpatch_tutor')).toBe(true); - expect(validators.isValidUsername('-RISEN-')).toBe(true); - expect(validators.isValidUsername('Coltroc')).toBe(true); - expect(validators.isValidUsername('Blue_Science')).toBe(true); - expect(validators.isValidUsername('atomicmagicnumber')).toBe(true); - expect(validators.isValidUsername('JWhandle')).toBe(true); - expect(validators.isValidUsername('josephjustin')).toBe(true); - expect(validators.isValidUsername('UltraCoolGames')).toBe(true); - expect(validators.isValidUsername('Hobson-TV')).toBe(true); - expect(validators.isValidUsername('BoyMcBoy-')).toBe(true); - expect(validators.isValidUsername('AquaLeafStudios')).toBe(true); - expect(validators.isValidUsername('GarboMuffin')).toBe(true); - expect(validators.isValidUsername('SullyBully')).toBe(true); - expect(validators.isValidUsername('SapphireDemon-')).toBe(true); - expect(validators.isValidUsername('AnExtremelyLongName')).toBe(true); - expect(validators.isValidUsername('PotatoAnimator')).toBe(true); - expect(validators.isValidUsername('EliMendez19Test')).toBe(true); - expect(validators.isValidUsername('BESTERNOOB')).toBe(true); - expect(validators.isValidUsername('cs123123123')).toBe(true); - expect(validators.isValidUsername('Elec-citrus')).toBe(true); - expect(validators.isValidUsername('yeetthehalls')).toBe(true); - expect(validators.isValidUsername('KidArachnid')).toBe(true); - expect(validators.isValidUsername('--FlamingGames--')).toBe(true); - expect(validators.isValidUsername('Executec')).toBe(true); - expect(validators.isValidUsername('Jethrochannz')).toBe(true); - expect(validators.isValidUsername('DaBaconBossOfBosses')).toBe(true); - expect(validators.isValidUsername('or2017')).toBe(true); - expect(validators.isValidUsername('CANSLP')).toBe(true); - expect(validators.isValidUsername('hfuller2953')).toBe(true); - expect(validators.isValidUsername('Kraken265')).toBe(true); - expect(validators.isValidUsername('Loev06')).toBe(true); -}); - test('isValidRoomID', () => { expect(validators.isValidRoomID('')).toBe(false); expect(validators.isValidRoomID('123')).toBe(true); - expect(validators.isValidRoomID('123.0')).toBe(false); - expect(validators.isValidRoomID('-123')).toBe(false); + expect(validators.isValidRoomID('123.0')).toBe(true); + expect(validators.isValidRoomID('-123')).toBe(true); + expect(validators.isValidRoomID('@#$)(*@#$)(')).toBe(true); + expect(validators.isValidRoomID('@p4-Appel v1.4')).toBe(true); expect(validators.isValidRoomID(123)).toBe(false); expect(validators.isValidRoomID(null)).toBe(false); expect(validators.isValidRoomID(undefined)).toBe(false); diff --git a/src/username.js b/src/username.js index 0584c39..f4c6a7e 100644 --- a/src/username.js +++ b/src/username.js @@ -1,6 +1,27 @@ +const naughty = require('./naughty'); +const fetch = require('node-fetch'); +const logger = require('./logger'); const config = require('./config'); +const https = require('https'); -const ANONYMIZE = /^player\d{2,7}$/i; +/** Maximum length of usernames, inclusive. */ +const MAX_LENGTH = 20; + +/** Minimum length of usernames, inclusive. */ +const MIN_LENGTH = 1; + +/** Regex for usernames to match. */ +const VALID_REGEX = /^[a-z0-9_-]+$/i; + +/** URL to fetch username metadata from. */ +const API = 'https://trampoline.turbowarp.org/proxy/users/$username'; + +/** Regex of usernames to anonymize. */ +const ANONYMIZE = /^player\d{1,9}$/i; + +const MIN_ACCOUNT_AGE = 1000 * 60 * 60 * 24; + +const USER_AGENT = 'https://github.com/TurboWarp/cloud-server'; /** * Anonymize a generated username, or return it unmodified @@ -20,8 +41,72 @@ function parseUsername(username) { * @returns {boolean} true if the username was probably randomly generated. */ function isGenerated(username) { - return ANONYMIZE.test(username); + return ANONYMIZE.test(username) || username === 'player'; +} + +const agent = new https.Agent({ + keepAlive: true +}); + +/** + * @param {unknown} username + * @returns {username is string} True if the username is valid. The username can be assumed to be a string + * if it is valid. + */ +function isValidUsername(username) { + return ( + typeof username === 'string' && + username.length >= MIN_LENGTH && + username.length <= MAX_LENGTH && + VALID_REGEX.test(username) && + !naughty(username) + ); } +/** + * @param {string} username + * @returns {Promise} + */ +function isValidScratchAccount(username) { + if (isGenerated(username)) { + return Promise.resolve(true); + } + + const start = Date.now(); + return fetch(API.replace('$username', username), { + timeout: 1000 * 10, + headers: { + 'user-agent': USER_AGENT + }, + agent + }) + .then((res) => { + if (res.ok) { + return res.json() + .then((data) => { + const joined = new Date(data.history.joined); + const age = Date.now() - joined.valueOf(); + return age >= MIN_ACCOUNT_AGE; + }); + } + if (res.status === 404 || res.status === 400) { + return false; + } + throw new Error(`Unexpected status code: ${res.status}`); + }) + .catch((err) => { + logger.error(err); + return true; + }) + .then((valid) => { + const end = Date.now(); + const time = end - start; + logger.info(`${username} is ${valid ? 'valid' : 'invalid'} (${Math.round(time)}ms)`); + return valid; + }); +}; + module.exports.parseUsername = parseUsername; module.exports.isGenerated = isGenerated; +module.exports.isValidUsername = isValidUsername; +module.exports.isValidScratchAccount = isValidScratchAccount; diff --git a/src/validators.js b/src/validators.js index d51d73c..80713f5 100644 --- a/src/validators.js +++ b/src/validators.js @@ -1,34 +1,19 @@ -const naughty = require('./naughty'); +const config = require('./config'); /** List of possible prefixes that must appear at the beginning of all variable's names. */ const CLOUD_PREFIXES = ['☁ ', ':cloud: ']; /** The maximum length of a variable's name. Scratch does not seem to restrict this but we don't want overly long variable names regardless. */ const VARIABLE_NAME_MAX_LENGTH = 1024; -/** The maximum length of a variable's value. */ -const VALUE_MAX_LENGTH = 100000; - -/** Maximum length of usernames, inclusive. */ -const USERNAME_MAX_LENGTH = 20; -/** Minimum length of usernames, inclusive. */ -const USERNAME_MIN_LENGTH = 1; -/** Regex for usernames to match. Letters, numbers, -, and _ */ -const USERNAME_REGEX = /^[a-z0-9_-]+$/i; - -/** - * @param {unknown} username - * @returns {boolean} - */ -module.exports.isValidUsername = function(username) { - return typeof username === 'string' && username.length >= USERNAME_MIN_LENGTH && username.length <= USERNAME_MAX_LENGTH && USERNAME_REGEX.test(username) && !naughty(username); -}; +const VALUE_MAX_LENGTH = config.maxValueLength; +const ALLOW_NON_NUMBER_VALUES = config.allowNonNumberValues; /** * @param {unknown} id * @returns {boolean} */ module.exports.isValidRoomID = function(id) { - return typeof id === 'string' && id.length > 0 && /^\d+$/.test(id); + return typeof id === 'string' && id.length > 0; }; /** @@ -50,6 +35,14 @@ module.exports.isValidVariableName = function(name) { * @returns {boolean} */ module.exports.isValidVariableValue = function(value) { + if (ALLOW_NON_NUMBER_VALUES) { + if (typeof value !== 'number' && typeof value !== 'boolean' && typeof value !== 'string') { + return false; + } + const str = value.toString(); + return str.length <= VALUE_MAX_LENGTH; + } + if (typeof value === 'number') { // If the value is a number, we don't have to parse it because we already know it's valid. // NaN and [-]Infinity are not valid, however.