From 584997826b2544170e57a81ab8bb73fc4a4c529e Mon Sep 17 00:00:00 2001 From: moonrailgun Date: Fri, 28 Jul 2023 15:24:30 +0800 Subject: [PATCH] feat: add alioss or other s3 protocol external storage support #104 --- apps/oauth-demo/package.json | 2 +- .../com.msgbyte.bbcode/src/tags/__all__.ts | 2 +- package.json | 1 - patches/moleculer-minio@2.0.0.patch | 22 - pnpm-lock.yaml | 337 +------ server/package.json | 1 - server/packages/sdk/package.json | 4 +- server/packages/sdk/src/index.ts | 1 + .../packages/sdk/src/services/lib/settings.ts | 4 +- .../sdk/src/services/mixins/minio.mixin.ts | 822 ++++++++++++++++++ server/services/core/file.service.ts | 5 +- website/docs/deployment/environment.md | 4 +- .../current/deployment/environment.md | 4 +- 13 files changed, 868 insertions(+), 341 deletions(-) delete mode 100644 patches/moleculer-minio@2.0.0.patch create mode 100644 server/packages/sdk/src/services/mixins/minio.mixin.ts diff --git a/apps/oauth-demo/package.json b/apps/oauth-demo/package.json index 6dc443e23d6..dbf28392eeb 100644 --- a/apps/oauth-demo/package.json +++ b/apps/oauth-demo/package.json @@ -13,7 +13,7 @@ "dependencies": { "express": "^4.18.2", "fs-extra": "^11.1.0", - "tailchat-server-sdk": "workspace:^0.0.14" + "tailchat-server-sdk": "workspace:^" }, "devDependencies": { "@types/express": "^4.17.15", diff --git a/client/web/plugins/com.msgbyte.bbcode/src/tags/__all__.ts b/client/web/plugins/com.msgbyte.bbcode/src/tags/__all__.ts index b706f12bd20..c6f3b3dad5b 100644 --- a/client/web/plugins/com.msgbyte.bbcode/src/tags/__all__.ts +++ b/client/web/plugins/com.msgbyte.bbcode/src/tags/__all__.ts @@ -29,4 +29,4 @@ registerBBCodeTag('at', MentionTag); registerBBCodeTag('emoji', EmojiTag); registerBBCodeTag('markdown', MarkdownTag); registerBBCodeTag('md', MarkdownTag); // alias -registerBBCodeTag('card', CardTag); // alias +registerBBCodeTag('card', CardTag); diff --git a/package.json b/package.json index 77e8276e312..505eaf101e6 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "mongoose-findorcreate": "3.0.0" }, "patchedDependencies": { - "moleculer-minio@2.0.0": "patches/moleculer-minio@2.0.0.patch", "moleculer@0.14.23": "patches/moleculer@0.14.23.patch", "vite-express@0.8.0": "patches/vite-express@0.8.0.patch" } diff --git a/patches/moleculer-minio@2.0.0.patch b/patches/moleculer-minio@2.0.0.patch deleted file mode 100644 index da912632593..00000000000 --- a/patches/moleculer-minio@2.0.0.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/src/errors/MinioInitializationError.js b/src/errors/MinioInitializationError.js -index 2022b251c5bc9c036c7456d6937c270f8f920d3a..ae9d3808d87f8fe0ee5e82076802da09a1a7f676 100644 ---- a/src/errors/MinioInitializationError.js -+++ b/src/errors/MinioInitializationError.js -@@ -1,4 +1,5 @@ --const {MoleculerError} = require("moleculer/src/errors"); -+const {Errors} = require('moleculer'); -+const MoleculerError = Errors.MoleculerError; - - /** - * Error that should be thrown when the Minio Service can not be Initialized -diff --git a/src/errors/MinioPingError.js b/src/errors/MinioPingError.js -index f73f9423f3407fe828ba99db556b2f8367483fa3..d03e31c2a223b6182c659b0c13beb7e1f11751d4 100644 ---- a/src/errors/MinioPingError.js -+++ b/src/errors/MinioPingError.js -@@ -1,4 +1,5 @@ --const {MoleculerRetryableError} = require("moleculer/src/errors"); -+const {Errors} = require('moleculer'); -+const MoleculerRetryableError = Errors.MoleculerRetryableError; - - /** - * Error that should be thrown when the Minio Backend can not be pinged \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e131dc84fba..5e4c8686717 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,6 @@ overrides: mongoose-findorcreate: 3.0.0 patchedDependencies: - moleculer-minio@2.0.0: - hash: 77awcwzrgh47fhn6qqq4ghcfau - path: patches/moleculer-minio@2.0.0.patch moleculer@0.14.23: hash: ahhlgpfy57fntn2aftq6beaeja path: patches/moleculer@0.14.23.patch @@ -320,7 +317,7 @@ importers: specifier: ^11.1.0 version: 11.1.0 tailchat-server-sdk: - specifier: workspace:^0.0.14 + specifier: workspace:^ version: link:../../server/packages/sdk devDependencies: '@types/express': @@ -1391,9 +1388,6 @@ importers: mkdirp: specifier: ^1.0.4 version: 1.0.4 - moleculer-minio: - specifier: ^2.0.0 - version: 2.0.0(patch_hash=77awcwzrgh47fhn6qqq4ghcfau)(moleculer@0.14.23) moment: specifier: ^2.29.1 version: 2.29.4 @@ -1723,6 +1717,9 @@ importers: lodash: specifier: ^4.17.21 version: 4.17.21 + minio: + specifier: ^7.1.1 + version: 7.1.1 moleculer: specifier: 0.14.23 version: 0.14.23(patch_hash=ahhlgpfy57fntn2aftq6beaeja)(ioredis@4.28.5)(nats@1.4.12)(redlock@4.2.0) @@ -1744,6 +1741,9 @@ importers: path-to-regexp: specifier: ^6.2.1 version: 6.2.1 + ramda-adjunct: + specifier: ^4.0.0 + version: 4.0.0(ramda@0.29.0) tailchat-types: specifier: workspace:^ version: link:../../../packages/types @@ -13668,15 +13668,6 @@ packages: /asap@2.0.6: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} - /asn1.js@5.4.1: - resolution: {integrity: sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==} - dependencies: - bn.js: 4.12.0 - inherits: 2.0.4 - minimalistic-assert: 1.0.1 - safer-buffer: 2.1.2 - dev: false - /asn1@0.2.6: resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} dependencies: @@ -14265,14 +14256,6 @@ packages: resolution: {integrity: sha512-0pcSSGxC0QxT+yVkivxIqW0Y4VlO2XSDPofBAqoJ1qJxgH9eiUDLv50Rixij2cDuEfx4M6DpD9UGZpRhT5Q8qg==} dev: false - /bn.js@4.12.0: - resolution: {integrity: sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==} - dev: false - - /bn.js@5.2.1: - resolution: {integrity: sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==} - dev: false - /body-parser@1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -14422,67 +14405,14 @@ packages: dependencies: fill-range: 7.0.1 - /brorand@1.1.0: - resolution: {integrity: sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==} - dev: false - - /browser-or-node@1.3.0: - resolution: {integrity: sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg==} + /browser-or-node@2.1.1: + resolution: {integrity: sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==} dev: false /browser-process-hrtime@1.0.0: resolution: {integrity: sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==} dev: true - /browserify-aes@1.2.0: - resolution: {integrity: sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==} - dependencies: - buffer-xor: 1.0.3 - cipher-base: 1.0.4 - create-hash: 1.2.0 - evp_bytestokey: 1.0.3 - inherits: 2.0.4 - safe-buffer: 5.2.1 - dev: false - - /browserify-cipher@1.0.1: - resolution: {integrity: sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==} - dependencies: - browserify-aes: 1.2.0 - browserify-des: 1.0.2 - evp_bytestokey: 1.0.3 - dev: false - - /browserify-des@1.0.2: - resolution: {integrity: sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==} - dependencies: - cipher-base: 1.0.4 - des.js: 1.0.1 - inherits: 2.0.4 - safe-buffer: 5.2.1 - dev: false - - /browserify-rsa@4.1.0: - resolution: {integrity: sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==} - dependencies: - bn.js: 5.2.1 - randombytes: 2.1.0 - dev: false - - /browserify-sign@4.2.1: - resolution: {integrity: sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==} - dependencies: - bn.js: 5.2.1 - browserify-rsa: 4.1.0 - create-hash: 1.2.0 - create-hmac: 1.1.7 - elliptic: 6.5.4 - inherits: 2.0.4 - parse-asn1: 5.1.6 - readable-stream: 3.6.1 - safe-buffer: 5.2.1 - dev: false - /browserslist@4.21.3: resolution: {integrity: sha512-898rgRXLAyRkM1GryrrBHGkqA5hlpkV5MhtZwg9QXeiyLUYs2k00Un05aX5l2/yJIOObYKOpS2JNo8nJDE7fWQ==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -14556,10 +14486,6 @@ packages: /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - /buffer-xor@1.0.3: - resolution: {integrity: sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==} - dev: false - /buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} dependencies: @@ -15060,13 +14986,6 @@ packages: resolution: {integrity: sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==} engines: {node: '>=8'} - /cipher-base@1.0.4: - resolution: {integrity: sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==} - dependencies: - inherits: 2.0.4 - safe-buffer: 5.2.1 - dev: false - /cjs-module-lexer@1.2.2: resolution: {integrity: sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA==} dev: true @@ -16147,34 +16066,6 @@ packages: react: 18.2.0 dev: false - /create-ecdh@4.0.4: - resolution: {integrity: sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==} - dependencies: - bn.js: 4.12.0 - elliptic: 6.5.4 - dev: false - - /create-hash@1.2.0: - resolution: {integrity: sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==} - dependencies: - cipher-base: 1.0.4 - inherits: 2.0.4 - md5.js: 1.3.5 - ripemd160: 2.0.2 - sha.js: 2.4.11 - dev: false - - /create-hmac@1.1.7: - resolution: {integrity: sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==} - dependencies: - cipher-base: 1.0.4 - create-hash: 1.2.0 - inherits: 2.0.4 - ripemd160: 2.0.2 - safe-buffer: 5.2.1 - sha.js: 2.4.11 - dev: false - /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -16217,22 +16108,6 @@ packages: resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} dev: false - /crypto-browserify@3.12.0: - resolution: {integrity: sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==} - dependencies: - browserify-cipher: 1.0.1 - browserify-sign: 4.2.1 - create-ecdh: 4.0.4 - create-hash: 1.2.0 - create-hmac: 1.1.7 - diffie-hellman: 5.0.3 - inherits: 2.0.4 - pbkdf2: 3.1.2 - public-encrypt: 4.0.3 - randombytes: 2.1.0 - randomfill: 1.0.4 - dev: false - /crypto-random-string@2.0.0: resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==} engines: {node: '>=8'} @@ -17075,13 +16950,6 @@ packages: engines: {node: '>=6'} dev: false - /des.js@1.0.1: - resolution: {integrity: sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==} - dependencies: - inherits: 2.0.4 - minimalistic-assert: 1.0.1 - dev: false - /destroy@1.2.0: resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -17184,14 +17052,6 @@ packages: engines: {node: '>=0.3.1'} dev: false - /diffie-hellman@5.0.3: - resolution: {integrity: sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==} - dependencies: - bn.js: 4.12.0 - miller-rabin: 4.0.1 - randombytes: 2.1.0 - dev: false - /dir-glob@2.2.2: resolution: {integrity: sha512-f9LBi5QWzIW3I6e//uxZoLBlUt9kcp66qo0sSCxL6YZKc75R1c4MFCoe/LaZiBGmgujvQdxc5Bn3QhfyvK5Hsw==} engines: {node: '>=4'} @@ -17509,18 +17369,6 @@ packages: /electron-to-chromium@1.4.310: resolution: {integrity: sha512-/xlATgfwkm5uDDwLw5nt/MNEf7c1oazLURMZLy39vOioGYyYzLWIDT8fZMJak6qTiAJ7udFTy7JG7ziyjNutiA==} - /elliptic@6.5.4: - resolution: {integrity: sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==} - dependencies: - bn.js: 4.12.0 - brorand: 1.1.0 - hash.js: 1.1.7 - hmac-drbg: 1.0.1 - inherits: 2.0.4 - minimalistic-assert: 1.0.1 - minimalistic-crypto-utils: 1.0.1 - dev: false - /emittery@0.8.1: resolution: {integrity: sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg==} engines: {node: '>=10'} @@ -18435,13 +18283,6 @@ packages: engines: {node: '>=12.0.0'} dev: false - /evp_bytestokey@1.0.3: - resolution: {integrity: sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==} - dependencies: - md5.js: 1.3.5 - safe-buffer: 5.2.1 - dev: false - /exec-sh@0.3.6: resolution: {integrity: sha512-nQn+hI3yp+oD0huYhKwvYI32+JFeq+XkNcD1GAo3Y/MjxsfVGmrrzrnzjWiNY6f+pUCP440fThsFh5gZrRAU/w==} dev: true @@ -18748,8 +18589,8 @@ packages: punycode: 1.4.1 dev: false - /fast-xml-parser@3.21.1: - resolution: {integrity: sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg==} + /fast-xml-parser@4.2.6: + resolution: {integrity: sha512-Xo1qV++h/Y3Ng8dphjahnYe+rGHaaNdsYOBWL9Y9GCPKpNKilJtilvWkLcI9f9X2DoKTLsZsGYAls5+JL5jfLA==} hasBin: true dependencies: strnum: 1.0.5 @@ -20044,22 +19885,6 @@ packages: dependencies: function-bind: 1.1.1 - /hash-base@3.1.0: - resolution: {integrity: sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==} - engines: {node: '>=4'} - dependencies: - inherits: 2.0.4 - readable-stream: 3.6.1 - safe-buffer: 5.2.1 - dev: false - - /hash.js@1.1.7: - resolution: {integrity: sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==} - dependencies: - inherits: 2.0.4 - minimalistic-assert: 1.0.1 - dev: false - /hast-to-hyperscript@9.0.1: resolution: {integrity: sha512-zQgLKqF+O2F72S1aa4y2ivxzSlko3MAvxkwG8ehGmNiqd98BIN3JM1rAJPmplEyLmGLO2QZYJtIneOSZ2YbJuA==} dependencies: @@ -20243,14 +20068,6 @@ packages: value-equal: 1.0.1 dev: false - /hmac-drbg@1.0.1: - resolution: {integrity: sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==} - dependencies: - hash.js: 1.1.7 - minimalistic-assert: 1.0.1 - minimalistic-crypto-utils: 1.0.1 - dev: false - /hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} dependencies: @@ -23362,14 +23179,6 @@ packages: resolution: {integrity: sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==} dev: false - /md5.js@1.3.5: - resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==} - dependencies: - hash-base: 3.1.0 - inherits: 2.0.4 - safe-buffer: 5.2.1 - dev: false - /md5@2.3.0: resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} dependencies: @@ -23968,14 +23777,6 @@ packages: braces: 3.0.2 picomatch: 2.3.1 - /miller-rabin@4.0.1: - resolution: {integrity: sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==} - hasBin: true - dependencies: - bn.js: 4.12.0 - brorand: 1.1.0 - dev: false - /mime-db@1.33.0: resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} engines: {node: '>= 0.6'} @@ -24117,10 +23918,6 @@ packages: /minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} - /minimalistic-crypto-utils@1.0.1: - resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==} - dev: false - /minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: @@ -24159,27 +23956,24 @@ packages: /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - /minio@7.0.32: - resolution: {integrity: sha512-txa7Vr0N24MKzeAybP/wY1jxbLnfGHXwZYyfFXuMW55HX2+HOcKEIgH4hU6Qj/kiMgyXs/ozHjAuLIDrR8nwLg==} - engines: {node: '>8 <=18'} + /minio@7.1.1: + resolution: {integrity: sha512-HBLRFXs1CkNwAkahU+j1ilB9YS/Tmkdc6orpxVW1YN11NlEJyLjarIpBYu/inF+dj+tJIsA8PSKNnRmUNm+9qQ==} + engines: {node: ^16 || ^18 || >=20} dependencies: async: 3.2.4 block-stream2: 2.1.0 - browser-or-node: 1.3.0 + browser-or-node: 2.1.1 buffer-crc32: 0.2.13 - crypto-browserify: 3.12.0 - es6-error: 4.1.1 - fast-xml-parser: 3.21.1 + fast-xml-parser: 4.2.6 ipaddr.js: 2.0.1 json-stream: 1.0.0 lodash: 4.17.21 mime-types: 2.1.35 - mkdirp: 0.5.6 query-string: 7.1.3 - through2: 3.0.2 + through2: 4.0.2 web-encoding: 1.1.5 xml: 1.0.1 - xml2js: 0.4.23 + xml2js: 0.5.0 dev: false /minipass-collect@1.0.2: @@ -24370,19 +24164,6 @@ packages: moleculer: 0.14.23(patch_hash=ahhlgpfy57fntn2aftq6beaeja)(ioredis@4.28.5)(nats@1.4.12)(redlock@4.2.0) dev: false - /moleculer-minio@2.0.0(patch_hash=77awcwzrgh47fhn6qqq4ghcfau)(moleculer@0.14.23): - resolution: {integrity: sha512-2INKAtdgboR8VQC9Vp0W4u5yy6XVxUW5UmP8akoggMa1BuvJ8m6hUlY/Y2VpcQ6FJIrQa9hHz5B38IMmXd7ZqQ==} - engines: {node: '>= 14.x.x'} - peerDependencies: - moleculer: '>= 0.13.0' - dependencies: - minio: 7.0.32 - moleculer: 0.14.23(patch_hash=ahhlgpfy57fntn2aftq6beaeja)(ioredis@4.28.5)(nats@1.4.12)(redlock@4.2.0) - ramda: 0.27.1 - ramda-adjunct: 2.36.0(ramda@0.27.1) - dev: false - patched: true - /moleculer-repl@0.6.6: resolution: {integrity: sha512-XQuz6PdosVgm8SkqJ21gba+VlDGvDHMd2yGhZyeZaoacWEe1wovA8Smr+JuAwFCz6YEpv6Gp6NVLMyHoG5DcsQ==} engines: {node: '>= 12.x.x'} @@ -25685,16 +25466,6 @@ packages: resolution: {integrity: sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==} dev: false - /parse-asn1@5.1.6: - resolution: {integrity: sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==} - dependencies: - asn1.js: 5.4.1 - browserify-aes: 1.2.0 - evp_bytestokey: 1.0.3 - pbkdf2: 3.1.2 - safe-buffer: 5.2.1 - dev: false - /parse-entities@2.0.0: resolution: {integrity: sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==} dependencies: @@ -25940,17 +25711,6 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} - /pbkdf2@3.1.2: - resolution: {integrity: sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==} - engines: {node: '>=0.12'} - dependencies: - create-hash: 1.2.0 - create-hmac: 1.1.7 - ripemd160: 2.0.2 - safe-buffer: 5.2.1 - sha.js: 2.4.11 - dev: false - /peek-readable@4.1.0: resolution: {integrity: sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==} engines: {node: '>=8'} @@ -27375,17 +27135,6 @@ packages: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true - /public-encrypt@4.0.3: - resolution: {integrity: sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==} - dependencies: - bn.js: 4.12.0 - browserify-rsa: 4.1.0 - create-hash: 1.2.0 - parse-asn1: 5.1.6 - randombytes: 2.1.0 - safe-buffer: 5.2.1 - dev: false - /pump@2.0.1: resolution: {integrity: sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==} dependencies: @@ -27521,35 +27270,28 @@ packages: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} - /ramda-adjunct@2.36.0(ramda@0.27.1): - resolution: {integrity: sha512-8w+/Hx73oByS+vo+BfAPOG3HYL2ay6O5fjrJpR7NFxMoFWksKz6vSOtvjqdfMM6MfAimHizq9tpdI0OD4xbKog==} + /ramda-adjunct@4.0.0(ramda@0.29.0): + resolution: {integrity: sha512-W/NiJAlZdwZ/iUkWEQQgRdH5Szqqet1WoVH9cdqDVjFbVaZHuJfJRvsxqHhvq6tZse+yVbFatLDLdVa30wBlGQ==} engines: {node: '>=0.10.3'} peerDependencies: - ramda: '>= 0.19.0 <= 0.27.2' + ramda: '>= 0.29.0' dependencies: - ramda: 0.27.1 - dev: false - - /ramda@0.27.1: - resolution: {integrity: sha512-PgIdVpn5y5Yns8vqb8FzBUEYn98V3xcPgawAkkgj0YJ0qDsnHCiNmZYfOGMgOvoB0eWFLpYbhxUR3mxfDIMvpw==} + ramda: 0.29.0 dev: false /ramda@0.28.0: resolution: {integrity: sha512-9QnLuG/kPVgWvMQ4aODhsBUFKOUmnbUnsSXACv+NCQZcHbeb+v8Lodp8OVxtRULN1/xOyYLLaL6npE6dMq5QTA==} dev: true + /ramda@0.29.0: + resolution: {integrity: sha512-BBea6L67bYLtdbOqfp8f58fPMqEwx0doL+pAi8TZyp2YWz8R9G8z9x75CZI8W+ftqhFHCpEX2cRnUUXK130iKA==} + dev: false + /randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: safe-buffer: 5.2.1 - /randomfill@1.0.4: - resolution: {integrity: sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==} - dependencies: - randombytes: 2.1.0 - safe-buffer: 5.2.1 - dev: false - /range-parser@1.2.0: resolution: {integrity: sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==} engines: {node: '>= 0.6'} @@ -29708,13 +29450,6 @@ packages: dependencies: glob: 7.2.3 - /ripemd160@2.0.2: - resolution: {integrity: sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==} - dependencies: - hash-base: 3.1.0 - inherits: 2.0.4 - dev: false - /rollup-plugin-copy@3.4.0: resolution: {integrity: sha512-rGUmYYsYsceRJRqLVlE9FivJMxJ7X6jDlP79fmFkL8sJs7VVMSVyA2yfyL+PGyO/vJs4A87hwhgVfz61njI+uQ==} engines: {node: '>=8.3'} @@ -30287,14 +30022,6 @@ packages: /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - /sha.js@2.4.11: - resolution: {integrity: sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==} - hasBin: true - dependencies: - inherits: 2.0.4 - safe-buffer: 5.2.1 - dev: false - /shallow-clone@3.0.1: resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==} engines: {node: '>=8'} @@ -31970,18 +31697,10 @@ packages: xtend: 4.0.2 dev: true - /through2@3.0.2: - resolution: {integrity: sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==} - dependencies: - inherits: 2.0.4 - readable-stream: 3.6.1 - dev: false - /through2@4.0.2: resolution: {integrity: sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==} dependencies: readable-stream: 3.6.1 - dev: true /through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} @@ -34363,8 +34082,8 @@ packages: resolution: {integrity: sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==} dev: true - /xml2js@0.4.23: - resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} + /xml2js@0.5.0: + resolution: {integrity: sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==} engines: {node: '>=4.0.0'} dependencies: sax: 1.2.4 diff --git a/server/package.json b/server/package.json index 57e564c5b82..f16231a6652 100644 --- a/server/package.json +++ b/server/package.json @@ -58,7 +58,6 @@ "lodash": "^4.17.21", "mime": "^2.5.2", "mkdirp": "^1.0.4", - "moleculer-minio": "^2.0.0", "moment": "^2.29.1", "mongodb": "4.2.1", "mongoose": "6.1.1", diff --git a/server/packages/sdk/package.json b/server/packages/sdk/package.json index 72bd8b4dc60..47c35882b08 100644 --- a/server/packages/sdk/package.json +++ b/server/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "tailchat-server-sdk", - "version": "0.0.14", + "version": "0.0.15", "description": "", "main": "dist/index.js", "bin": { @@ -49,6 +49,7 @@ "isstream": "^0.1.2", "kleur": "^4.1.4", "lodash": "^4.17.21", + "minio": "^7.1.1", "moleculer": "0.14.23", "moleculer-db": "0.8.19", "moleculer-repl": "^0.7.2", @@ -56,6 +57,7 @@ "mongodb": "4.2.1", "mongoose": "6.1.1", "path-to-regexp": "^6.2.1", + "ramda-adjunct": "^4.0.0", "tailchat-types": "workspace:^" } } diff --git a/server/packages/sdk/src/index.ts b/server/packages/sdk/src/index.ts index f99896dc0a9..9ef81b27636 100644 --- a/server/packages/sdk/src/index.ts +++ b/server/packages/sdk/src/index.ts @@ -2,6 +2,7 @@ export { defaultBrokerConfig } from './runner/moleculer.config'; export { TcService } from './services/base'; export { TcBroker } from './services/broker'; export type { TcDbService } from './services/mixins/db.mixin'; +export { TcMinioService } from './services/mixins/minio.mixin'; export type { TcContext, TcPureContext, diff --git a/server/packages/sdk/src/services/lib/settings.ts b/server/packages/sdk/src/services/lib/settings.ts index 03d178bf3a5..fe56bfdc371 100644 --- a/server/packages/sdk/src/services/lib/settings.ts +++ b/server/packages/sdk/src/services/lib/settings.ts @@ -9,6 +9,7 @@ dotenv.config(); const port = process.env.PORT ? Number(process.env.PORT) : 11000; const apiUrl = process.env.API_URL || `http://127.0.0.1:${port}`; const staticHost = process.env.STATIC_HOST || '{BACKEND}'; +const staticUrl = process.env.STATIC_URL || `${staticHost}/static/`; export const config = { port, secret: process.env.SECRET || 'tailchat', @@ -25,6 +26,7 @@ export const config = { user: process.env.MINIO_USER, pass: process.env.MINIO_PASS, bucketName: process.env.MINIO_BUCKET_NAME || 'tailchat', + pathStyle: process.env.MINIO_PATH_STYLE === 'VirtualHosted' ? false : true, /** * 文件上传限制 @@ -36,7 +38,7 @@ export const config = { : 1 * 1024 * 1024, }, apiUrl, - staticUrl: `${staticHost}/static/`, + staticUrl, enableOpenapi: true, // 是否开始openapi emailVerification: checkEnvTrusty(process.env.EMAIL_VERIFY) || false, // 是否在注册后验证邮箱可用性 diff --git a/server/packages/sdk/src/services/mixins/minio.mixin.ts b/server/packages/sdk/src/services/mixins/minio.mixin.ts new file mode 100644 index 00000000000..c7805fec129 --- /dev/null +++ b/server/packages/sdk/src/services/mixins/minio.mixin.ts @@ -0,0 +1,822 @@ +import { Client as MinioClient, CopyConditions } from 'minio'; +import { isString, isUndefined } from 'ramda-adjunct'; +import { Errors } from 'moleculer'; + +class MinioInitializationError extends Errors.MoleculerError { + /** + * Creates an instance of MinioInitializationError. + * + * @param {String?} message + * @param {Number?} code + * @param {String?} type + * @param {any} data + * + * @memberof MinioInitializationError + */ + constructor( + message = 'Minio can not be initialized', + code = 500, + type = 'MINIO_INITIALIZATION_ERROR', + data = {} + ) { + super(message); + this.code = code; + this.type = type; + this.data = data; + } +} + +class MinioPingError extends Errors.MoleculerRetryableError { + /** + * Creates an instance of MinioPingError. + * + * @param {String?} message + * @param {Number?} code + * @param {String?} type + * @param {any} data + * + * @memberof MinioPingError + */ + constructor( + message = 'Minio Backend not reachable', + code = 502, + type = 'MINIO_PING_ERROR', + data = {} + ) { + super(message); + this.code = code; + this.type = type; + this.data = data; + } +} + +/** + * Service mixin for managing files in a Minio S3 backend + * + * @name moleculer-minio + * @module Service + */ + +export const TcMinioService = { + // Service name + name: 'minio', + + // Default settings + settings: { + /** @type {String} The Hostname minio is running on and available at. Hostname or IP-Address */ + endPoint: undefined, + /** @type {Number} TCP/IP port number minio is listening on. Default value set to 80 for HTTP and 443 for HTTPs.*/ + port: undefined, + /** @type {Boolean?} If set to true, https is used instead of http. Default is true.*/ + useSSL: true, + /** @type {String} The AccessKey to use when connecting to minio */ + accessKey: undefined, + /** @type {String} The SecretKey to use when connecting to minio */ + secretKey: undefined, + /** @type {String?} Set this value to override region cache*/ + region: undefined, + /** @type {String?} Set this value to pass in a custom transport. (Optional)*/ + transport: undefined, + /** @type {String?} Set this value to provide x-amz-security-token (AWS S3 specific). (Optional)*/ + sessionToken: undefined, + /** @type {Number?} This service will perform a periodic healthcheck of Minio. Use this setting to configure the inverval in which the healthcheck is performed. Set to `0` to turn healthcheks of */ + minioHealthCheckInterval: 5000, + + /** + * Path Style: ://// + * Virtual hosted style: ://./ + */ + pathStyle: true, + }, + + methods: { + /** + * Creates and returns a new Minio client + * + * @methods + * + * @returns {Client} + */ + createMinioClient() { + return new MinioClient({ + endPoint: this.settings.endPoint, + port: this.settings.port, + useSSL: this.settings.useSSL, + accessKey: this.settings.accessKey, + secretKey: this.settings.secretKey, + region: this.settings.region, + transport: this.settings.transport, + sessionToken: this.settings.sessionToken, + pathStyle: this.settings.pathStyle, + }); + }, + /** + * Pings the configured minio backend + * + * @param {number} timeout - Amount of miliseconds to wait for a ping response + * @returns {PromiseLike} + */ + ping({ timeout = 5000 } = {}) { + return this.Promise.race([ + this.client.listBuckets().then(() => true), + this.Promise.delay(timeout).then(() => { + throw new MinioPingError(); + }), + ]); + }, + }, + actions: { + /** + * Creates a new Bucket + * + * @actions + * + * @param {string} bucketName - The name of the bucket + * @param {string} region - The region to create the bucket in. Defaults to "us-east-1" + * + * @returns {PromiseLike} + */ + makeBucket: { + params: { + bucketName: { type: 'string' }, + region: { type: 'string', optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, region = '' }) => + this.client.makeBucket(bucketName, region) + ); + }, + }, + /** + * Lists all buckets. + * + * @actions + * + * @returns {PromiseLike} + */ + listBuckets: { + handler() { + return this.client + .listBuckets() + .then((buckets) => (isUndefined(buckets) ? [] : buckets)); + }, + }, + /** + * Checks if a bucket exists. + * + * @actions + * @param {string} bucketName - Name of the bucket + * + * @returns {PromiseLike} + */ + bucketExists: { + params: { + bucketName: { type: 'string' }, + }, + handler(ctx) { + return this.client.bucketExists(ctx.params.bucketName); + }, + }, + /** + * Removes a bucket. + * + * @actions + * @param {string} bucketName - Name of the bucket + * + * @returns {PromiseLike} + */ + removeBucket: { + params: { + bucketName: { type: 'string' }, + }, + handler(ctx) { + return this.client.removeBucket(ctx.params.bucketName); + }, + }, + /** + * Lists all objects in a bucket. + * + * @actions + * @param {string} bucketName - Name of the bucket + * @param {string} prefix - The prefix of the objects that should be listed (optional, default ''). + * @param {boolean} recursive - `true` indicates recursive style listing and false indicates directory style listing delimited by '/'. (optional, default `false`). + * + * @returns {PromiseLike} + */ + listObjects: { + params: { + bucketName: { type: 'string' }, + prefix: { type: 'string', optional: true }, + recursive: { type: 'boolean', optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, prefix = '', recursive = false }) => { + return new this.Promise((resolve, reject) => { + try { + const stream = this.client.listObjects( + bucketName, + prefix, + recursive + ); + const objects = []; + stream.on('data', (el) => objects.push(el)); + stream.on('end', () => resolve(objects)); + stream.on('error', reject); + } catch (e) { + reject(e); + } + }); + } + ); + }, + }, + /** + * Lists all objects in a bucket using S3 listing objects V2 API + * + * @actions + * @param {string} bucketName - Name of the bucket + * @param {string} prefix - The prefix of the objects that should be listed (optional, default ''). + * @param {boolean} recursive - `true` indicates recursive style listing and false indicates directory style listing delimited by '/'. (optional, default `false`). + * @param {string} startAfter - Specifies the object name to start after when listing objects in a bucket. (optional, default ''). + * + * @returns {PromiseLike} + */ + listObjectsV2: { + params: { + bucketName: { type: 'string' }, + prefix: { type: 'string', optional: true }, + recursive: { type: 'boolean', optional: true }, + startAfter: { type: 'string', optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, prefix = '', recursive = false, startAfter = '' }) => { + return new this.Promise((resolve, reject) => { + try { + const stream = this.client.listObjectsV2( + bucketName, + prefix, + recursive, + startAfter + ); + const objects = []; + stream.on('data', (el) => objects.push(el)); + stream.on('end', () => resolve(objects)); + stream.on('error', reject); + } catch (e) { + reject(e); + } + }); + } + ); + }, + }, + /** + * Lists partially uploaded objects in a bucket. + * + * @actions + * @param {string} bucketName - Name of the bucket + * @param {string} prefix - The prefix of the objects that should be listed (optional, default ''). + * @param {boolean} recursive - `true` indicates recursive style listing and false indicates directory style listing delimited by '/'. (optional, default `false`). + * + * @returns {PromiseLike} + */ + listIncompleteUploads: { + params: { + bucketName: { type: 'string' }, + prefix: { type: 'string', optional: true }, + recursive: { type: 'boolean', optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, prefix = '', recursive = false }) => { + return new this.Promise((resolve, reject) => { + try { + const stream = this.client.listIncompleteUploads( + bucketName, + prefix, + recursive + ); + const objects = []; + stream.on('data', (el) => objects.push(el)); + stream.on('end', () => resolve(objects)); + stream.on('error', reject); + } catch (e) { + reject(e); + } + }); + } + ); + }, + }, + /** + * Downloads an object as a stream. + * + * @actions + * @param {string} bucketName - Name of the bucket + * @param {string} objectName - Name of the object. + * + * @returns {PromiseLike} + */ + getObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + }, + handler(ctx) { + return this.client.getObject( + ctx.params.bucketName, + ctx.params.objectName + ); + }, + }, + /** + * Downloads the specified range bytes of an object as a stream. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {number} offset - `offset` of the object from where the stream will start. + * @param {number} length - `length` of the object that will be read in the stream (optional, if not specified we read the rest of the file from the offset). + * + * @returns {PromiseLike} + */ + getPartialObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + offset: { type: 'number' }, + length: { type: 'number', optional: true }, + }, + handler(ctx) { + return this.client.getPartialObject( + ctx.params.bucketName, + ctx.params.objectName, + ctx.params.offset, + ctx.params.length + ); + }, + }, + /** + * Downloads and saves the object as a file in the local filesystem. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {string} filePath - Path on the local filesystem to which the object data will be written. + * + * @returns {PromiseLike} + */ + fGetObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + filePath: { type: 'string' }, + }, + handler(ctx) { + return this.client.fGetObject( + ctx.params.bucketName, + ctx.params.objectName, + ctx.params.filePath + ); + }, + }, + /** + * Uploads an object from a stream/Buffer. + * + * @actions + * @param {ReadableStream} params - Readable stream. + * + * @meta + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {number} size - Size of the object (optional). + * @param {object} metaData - metaData of the object (optional). + * + * @returns {PromiseLike} + */ + putObject: { + handler(ctx) { + return this.Promise.resolve({ + stream: ctx.params, + meta: ctx.meta, + }).then(({ stream, meta }) => + this.client.putObject( + meta.bucketName, + meta.objectName, + stream, + meta.size, + meta.metaData + ) + ); + }, + }, + /** + * Uploads contents from a file to objectName. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {string} filePath - Path of the file to be uploaded. + * @param {object} metaData - metaData of the object (optional). + * + * @returns {PromiseLike} + */ + fPutObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + filePath: { type: 'string' }, + metaData: { type: 'object', optional: true }, + }, + handler(ctx) { + return this.client.fPutObject( + ctx.params.bucketName, + ctx.params.objectName, + ctx.params.filePath, + ctx.params.metaData + ); + }, + }, + /** + * Copy a source object into a new object in the specified bucket. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {string} sourceObject - Path of the file to be copied. + * @param {object} conditions - Conditions to be satisfied before allowing object copy. + * @param {object} metaData - metaData of the object (optional). + * + * @returns {PromiseLike<{etag: {string}, lastModified: {string}}|Error>} + */ + copyObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + sourceObject: { type: 'string' }, + conditions: { + type: 'object', + properties: { + modified: { type: 'string', optional: true }, + unmodified: { type: 'string', optional: true }, + matchETag: { type: 'string', optional: true }, + matchETagExcept: { type: 'string', optional: true }, + }, + }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, objectName, sourceObject, conditions }) => { + const _conditions = new CopyConditions(); + if (conditions.modified) { + _conditions.setModified(new Date(conditions.modified)); + } + if (conditions.unmodified) { + _conditions.setUnmodified(new Date(conditions.unmodified)); + } + if (conditions.matchETag) { + _conditions.setMatchETag(conditions.matchETag); + } + if (conditions.matchETagExcept) { + _conditions.setMatchETagExcept(conditions.matchETagExcept); + } + conditions = _conditions; + return this.client.copyObject( + bucketName, + objectName, + sourceObject, + conditions + ); + } + ); + }, + }, + /** + * Gets metadata of an object. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * + * @returns {PromiseLike<{size: {number}, metaData: {object}, lastModified: {string}, etag: {string}}|Error>} + */ + statObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + }, + handler(ctx) { + return this.client.statObject( + ctx.params.bucketName, + ctx.params.objectName + ); + }, + }, + /** + * Removes an Object + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * + * @returns {PromiseLike} + */ + removeObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + }, + handler(ctx) { + return this.client.removeObject( + ctx.params.bucketName, + ctx.params.objectName + ); + }, + }, + /** + * Removes a list of Objects + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string[]} objectNames - Names of the objects. + * + * @returns {PromiseLike} + */ + removeObjects: { + params: { + bucketName: { type: 'string' }, + objectNames: { type: 'array', items: 'string' }, + }, + handler(ctx) { + return this.client.removeObjects( + ctx.params.bucketName, + ctx.params.objectNames + ); + }, + }, + /** + * Removes a partially uploaded object. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * + * @returns {PromiseLike} + */ + removeIncompleteUpload: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, objectName }) => + this.client.removeIncompleteUpload(bucketName, objectName) + ); + }, + }, + /** + * Generates a presigned URL for the provided HTTP method, 'httpMethod'. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This + * presigned URL can have an associated expiration time in seconds after which the URL is no longer valid. The default value is 7 days. + * + * @actions + * @param {string} httpMethod - The HTTP-Method (eg. `GET`). + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {number} expires - Expiry time in seconds. Default value is 7 days. (optional) + * @param {object} reqParams - request parameters. (optional) + * @param {string} requestDate - An ISO date string, the url will be issued at. Default value is now. (optional) + * @returns {PromiseLike} + */ + presignedUrl: { + params: { + httpMethod: { type: 'string' }, + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + expires: { type: 'number', integer: true, optional: true }, + reqParams: { type: 'object', optional: true }, + requestDate: { type: 'string', optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ + httpMethod, + bucketName, + objectName, + expires, + reqParams, + requestDate, + }) => { + if (isString(requestDate)) { + requestDate = new Date(requestDate); + } + + return new this.Promise((resolve, reject) => { + this.client.presignedUrl( + httpMethod, + bucketName, + objectName, + expires, + reqParams, + requestDate, + (error, url) => { + if (error) { + reject(error); + } else { + resolve(url); + } + } + ); + }); + } + ); + }, + }, + /** + * Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an + * associated expiration time in seconds after which the URL is no longer valid. The default value is 7 days. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {number} expires - Expiry time in seconds. Default value is 7 days. (optional) + * @param {object} reqParams - request parameters. (optional) + * @param {string} requestDate - An ISO date string, the url will be issued at. Default value is now. (optional) + * @returns {PromiseLike} + */ + presignedGetObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + expires: { type: 'number', integer: true, optional: true }, + reqParams: { type: 'object', optional: true }, + requestDate: { type: 'string', optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, objectName, expires, reqParams, requestDate }) => { + if (isString(requestDate)) { + requestDate = new Date(requestDate); + } + + return new this.Promise((resolve, reject) => { + this.client.presignedGetObject( + bucketName, + objectName, + expires, + reqParams, + requestDate, + (error, url) => { + if (error) { + reject(error); + } else { + resolve(url); + } + } + ); + }); + } + ); + }, + }, + /** + * Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have + * an associated expiration time in seconds after which the URL is no longer valid. The default value is 7 days. + * + * @actions + * @param {string} bucketName - Name of the bucket. + * @param {string} objectName - Name of the object. + * @param {number} expires - Expiry time in seconds. Default value is 7 days. (optional) + * @returns {PromiseLike} + */ + presignedPutObject: { + params: { + bucketName: { type: 'string' }, + objectName: { type: 'string' }, + expires: { type: 'number', integer: true, optional: true }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then( + ({ bucketName, objectName, expires }) => { + return new this.Promise((resolve, reject) => { + this.client.presignedPutObject( + bucketName, + objectName, + expires, + (error, url) => { + if (error) { + reject(error); + } else { + resolve(url); + } + } + ); + }); + } + ); + }, + }, + /** + * Allows setting policy conditions to a presigned URL for POST operations. Policies such as bucket name to receive object uploads, key name prefixes, expiry policy may be set. + * + * @actions + * @param {object} policy - Policy object created by minioClient.newPostPolicy() + * @returns {PromiseLike<{postURL: {string}, formData: {object}}|Error>} + */ + presignedPostPolicy: { + params: { + policy: { + type: 'object', + properties: { + expires: { type: 'string', optional: true }, + key: { type: 'string', optional: true }, + keyStartsWith: { type: 'string', optional: true }, + bucket: { type: 'string', optional: true }, + contentType: { type: 'string', optional: true }, + contentLengthRangeMin: { + type: 'number', + integer: true, + optional: true, + }, + contentLengthRangeMax: { + type: 'number', + integer: true, + optional: true, + }, + }, + }, + }, + handler(ctx) { + return this.Promise.resolve(ctx.params).then(({ policy }) => { + const _policy = this.client.newPostPolicy(); + if (policy.expires) { + _policy.setExpires(new Date(policy.expires)); + } + if (policy.key) { + _policy.setKey(policy.key); + } + if (policy.keyStartsWith) { + _policy.setKeyStartsWith(policy.keyStartsWith); + } + if (policy.bucket) { + _policy.setBucket(policy.bucket); + } + if (policy.contentType) { + _policy.setContentType(policy.contentType); + } + if (policy.contentLengthRangeMin && policy.contentLengthRangeMax) { + _policy.setContentLengthRange( + policy.contentLengthRangeMin, + policy.contentLengthRangeMax + ); + } + return this.client.presignedPostPolicy(_policy); + }); + }, + }, + }, + + /** + * Service created lifecycle event handler. + * Constructs a new minio client entity + */ + created() { + this.client = this.createMinioClient(); + }, + /** + * Service started lifecycle event handler. Resolves when: + * * ping of S3 backend has been successful + * * a healthCheck has been registered, given minioHealthCheckInterval > 0 + * @returns {PromiseLike} + */ + started() { + /* istanbul ignore next */ + return this.Promise.resolve() + .then(() => this.ping()) + .then(() => { + this.settings.minioHealthCheckInterval + ? (this.healthCheckInterval = setInterval( + () => + this.ping().catch((e) => + this.logger.error('Minio backend can not be reached', e) + ), + this.settings.minioHealthCheckInterval + )) + : undefined; + return undefined; + }) + .catch((e) => { + throw new MinioInitializationError(e.message); + }); + }, + /** + * Service stopped lifecycle event handler. + * Removes the healthCheckInterval + */ + stopped() { + this.healthCheckInterval && clearInterval(this.healthCheckInterval); + }, +}; diff --git a/server/services/core/file.service.ts b/server/services/core/file.service.ts index 334b93d053a..ccb2feb333e 100644 --- a/server/services/core/file.service.ts +++ b/server/services/core/file.service.ts @@ -6,8 +6,8 @@ import { config, TcDbService, NoPermissionError, + TcMinioService, } from 'tailchat-server-sdk'; -import MinioService from 'moleculer-minio'; import _ from 'lodash'; import mime from 'mime'; import type { Client as MinioClient } from 'minio'; @@ -34,7 +34,7 @@ class FileService extends TcService { onInit(): void { this.registerLocalDb(require('../../models/file').default); - this.registerMixin(MinioService); + this.registerMixin(TcMinioService); const minioUrl = config.storage.minioUrl; const [endPoint, port] = minioUrl.split(':'); @@ -44,6 +44,7 @@ class FileService extends TcService { this.registerSetting('useSSL', false); this.registerSetting('accessKey', config.storage.user); this.registerSetting('secretKey', config.storage.pass); + this.registerSetting('pathStyle', config.storage.pathStyle); this.registerAction('save', this.save); this.registerAction('saveFileWithUrl', this.saveFileWithUrl, { diff --git a/website/docs/deployment/environment.md b/website/docs/deployment/environment.md index 8f3f15137c1..38338629432 100644 --- a/website/docs/deployment/environment.md +++ b/website/docs/deployment/environment.md @@ -9,7 +9,8 @@ title: Environment Variable | ----- | ------ | --- | | PORT | 11000 | Gateway service port number | | SECRET | tailchat | encryption key, used for JWT | -| STATIC_HOST | "{BACKEND}" | Externally accessible static service address, used for file service access, the default is the dynamic server address inferred from the front-end request, if it is expected to be stored in a third-party OSS, it needs to be modified | +| STATIC_HOST | "{BACKEND}" | Externally accessible static service host, used for file service access, the default is the dynamic server address inferred from the front-end request, if it is expected to be stored in a third-party OSS, it needs to be modified | +| STATIC_URL | "{BACKEND}/static/" | Externally accessible static service complete address prefix, used for file service access, the default is the dynamic server address inferred from the front-end request, if it is expected to be stored in a third-party OSS Modify, if this variable is set, the above `STATIC_HOST` value is invalid | | API_URL | http://127.0.0.1:11000 | Externally accessible url address, used for issuer issuance on open platforms or as a fallback for file services | | MONGO_URL | - | Database service address | | REDIS_URL | - | Redis service address | @@ -17,6 +18,7 @@ title: Environment Variable | MINIO_USER | - | File service username | | MINIO_PASS | - | File service password | | MINIO_BUCKET_NAME | tailchat | file service bucket name | +| MINIO_PATH_STYLE | false | Whether to use path-style s3 communication format, `true` is `Path Style`, `false` is `Virtual hosted style` | | SMTP_SENDER | - | Mail service sender (example: `"Tailchat" example@163.com`) | | SMTP_URI | - | mail service connection address (example: `smtp://username:password@smtp.example.com/?pool=true`) | | FILE_LIMIT | 1048576 | File/image upload size limit, the default is 1m, please enter a number(unit: byte) | diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/deployment/environment.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/deployment/environment.md index 45cc5f002cc..29dcfde85e4 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/deployment/environment.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/deployment/environment.md @@ -9,7 +9,8 @@ title: 环境变量 | ----- | ------ | --- | | PORT | 11000 | 网关服务端口号 | | SECRET | tailchat | 加密秘钥, 用于JWT | -| STATIC_HOST | "{BACKEND}" | 对外可访问的静态服务地址，用于文件服务访问, 默认为动态根据前端请求推断出的服务端地址，如果期望存储在第三方OSS中需要进行修改 | +| STATIC_HOST | "{BACKEND}" | 对外可访问的静态服务主机，用于文件服务访问, 默认为动态根据前端请求推断出的服务端地址，如果期望存储在第三方OSS中需要进行修改 | +| STATIC_URL | "{BACKEND}/static/" | 对外可访问的静态服务完整地址前缀，用于文件服务访问, 默认为动态根据前端请求推断出的服务端地址，如果期望存储在第三方OSS中需要进行修改, 如果设置了本变量则上面的 `STATIC_HOST` 值无效 | | API_URL | http://127.0.0.1:11000 | 对外可访问的url地址，用于开放平台的issuer签发或者作为文件服务的fallback | | MONGO_URL | - | 数据库服务地址 | | REDIS_URL | - | Redis服务地址 | @@ -17,6 +18,7 @@ title: 环境变量 | MINIO_USER | - | 文件服务用户名 | | MINIO_PASS | - | 文件服务密码 | | MINIO_BUCKET_NAME | tailchat | 文件服务存储桶名 | +| MINIO_PATH_STYLE | "Path" | 是否使用路径形式的s3通信格式, `Path` 为 `Path Style`, `VirtualHosted` 为 `Virtual hosted style` | | SMTP_SENDER | - | 邮件服务发件人(示例: `"Tailchat" example@163.com`) | | SMTP_URI | - | 邮件服务连接地址(示例: `smtp://username:password@smtp.example.com/?pool=true`) | | FILE_LIMIT | 1048576 | 文件/图片上传的大小限制，默认为1m，请输入数字，（单位: 字节） |