From db99227f3227638b4f224c3647132fa6b8c00d55 Mon Sep 17 00:00:00 2001 From: Maksim Eltyshev Date: Fri, 30 Jan 2026 21:45:18 +0100 Subject: [PATCH] feat: Re-stream static files from S3, introduce protected static files --- server/api/helpers/attachments/present-one.js | 3 +- .../helpers/background-images/present-one.js | 6 +- server/api/helpers/users/present-one.js | 6 +- .../hooks/file-manager/LocalFileManager.js | 40 +++++-- .../api/hooks/file-manager/S3FileManager.js | 22 +++- server/config/routes.js | 109 +++++++++++------- server/package-lock.json | 89 +++++++++++--- server/package.json | 1 + 8 files changed, 196 insertions(+), 80 deletions(-) diff --git a/server/api/helpers/attachments/present-one.js b/server/api/helpers/attachments/present-one.js index db94b9c2..69e54b8c 100644 --- a/server/api/helpers/attachments/present-one.js +++ b/server/api/helpers/attachments/present-one.js @@ -38,8 +38,7 @@ module.exports = { if (sails.helpers.utils.isPreloadedFaviconExists(inputs.record.data.hostname)) { faviconUrl = `${sails.config.custom.baseUrl}/preloaded-favicons/${faviconFilename}`; } else { - const fileManager = sails.hooks['file-manager'].getInstance(); - faviconUrl = `${fileManager.buildUrl(`${sails.config.custom.faviconsPathSegment}/${faviconFilename}`)}`; + faviconUrl = `${sails.config.custom.baseUrl}/favicons/${faviconFilename}`; } data = { diff --git a/server/api/helpers/background-images/present-one.js b/server/api/helpers/background-images/present-one.js index 8d81ce67..ba89ec51 100644 --- a/server/api/helpers/background-images/present-one.js +++ b/server/api/helpers/background-images/present-one.js @@ -14,13 +14,11 @@ module.exports = { }, fn(inputs) { - const fileManager = sails.hooks['file-manager'].getInstance(); - return { ..._.omit(inputs.record, ['uploadedFileId', 'extension']), - url: `${fileManager.buildUrl(`${sails.config.custom.backgroundImagesPathSegment}/${inputs.record.uploadedFileId}/original.${inputs.record.extension}`)}`, + url: `${sails.config.custom.baseUrl}/background-images/${inputs.record.uploadedFileId}/original.${inputs.record.extension}`, thumbnailUrls: { - outside360: `${fileManager.buildUrl(`${sails.config.custom.backgroundImagesPathSegment}/${inputs.record.uploadedFileId}/outside-360.${inputs.record.extension}`)}`, + outside360: `${sails.config.custom.baseUrl}/background-images/${inputs.record.uploadedFileId}/outside-360.${inputs.record.extension}`, }, }; }, diff --git a/server/api/helpers/users/present-one.js b/server/api/helpers/users/present-one.js index 53f00535..2fbb5c91 100644 --- a/server/api/helpers/users/present-one.js +++ b/server/api/helpers/users/present-one.js @@ -17,8 +17,6 @@ module.exports = { }, fn(inputs) { - const fileManager = sails.hooks['file-manager'].getInstance(); - const data = { ..._.omit(inputs.record, [ 'password', @@ -30,9 +28,9 @@ module.exports = { 'termsAcceptedAt', ]), avatar: inputs.record.avatar && { - url: `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.uploadedFileId}/original.${inputs.record.avatar.extension}`)}`, + url: `${sails.config.custom.baseUrl}/user-avatars/${inputs.record.avatar.uploadedFileId}/original.${inputs.record.avatar.extension}`, thumbnailUrls: { - cover180: `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.uploadedFileId}/cover-180.${inputs.record.avatar.extension}`)}`, + cover180: `${sails.config.custom.baseUrl}/user-avatars/${inputs.record.avatar.uploadedFileId}/cover-180.${inputs.record.avatar.extension}`, }, }, language: inputs.record.language || sails.config.i18n.defaultLocale, diff --git a/server/api/hooks/file-manager/LocalFileManager.js b/server/api/hooks/file-manager/LocalFileManager.js index ee55643f..c6447d14 100644 --- a/server/api/hooks/file-manager/LocalFileManager.js +++ b/server/api/hooks/file-manager/LocalFileManager.js @@ -7,9 +7,10 @@ const fs = require('fs'); const fse = require('fs-extra'); const path = require('path'); const { pipeline } = require('stream/promises'); +const mime = require('mime-types'); const { rimraf } = require('rimraf'); -const PATH_SEGMENT_TO_URL_REPLACE_REGEX = /(public|private)\//; +// const PATH_SEGMENT_TO_URL_REPLACE_REGEX = /(public|private)\//; const buildPath = (pathSegment) => path.join(sails.config.custom.uploadsBasePath, pathSegment); @@ -37,27 +38,46 @@ class LocalFileManager { } // eslint-disable-next-line class-methods-use-this - async read(filePathSegment) { + async read(filePathSegment, { withHeaders = false } = {}) { const filePath = buildPath(filePathSegment); - const isFileExists = await fse.pathExists(filePath); - if (!isFileExists) { + let stat; + try { + stat = await fs.promises.stat(filePath); + } catch (error) { throw new Error('File does not exist'); } - return fs.createReadStream(filePath); + const readStream = fs.createReadStream(filePath); + + if (withHeaders) { + const weakEtag = `W/"${stat.size.toString(16)}-${stat.mtime.getTime().toString(16)}"`; + + return [ + readStream, + { + 'Content-Type': mime.lookup(filePathSegment) || 'application/octet-stream', + 'Content-Length': stat.size, + 'Last-Modified': stat.mtime.toUTCString(), + ETag: weakEtag, + 'Accept-Ranges': 'bytes', + }, + ]; + } + + return readStream; } // eslint-disable-next-line class-methods-use-this async getSize(filePathSegment) { - let result; + let stat; try { - result = await fs.promises.stat(buildPath(filePathSegment)); + stat = await fs.promises.stat(buildPath(filePathSegment)); } catch (error) { return null; } - return result.size; + return stat.size; } // eslint-disable-next-line class-methods-use-this @@ -111,10 +131,10 @@ class LocalFileManager { return fse.pathExists(buildPath(pathSegment)); } - // eslint-disable-next-line class-methods-use-this + /* // eslint-disable-next-line class-methods-use-this buildUrl(filePathSegment) { return `${sails.config.custom.baseUrl}/${filePathSegment.replace(PATH_SEGMENT_TO_URL_REPLACE_REGEX, '')}`; - } + } */ } module.exports = LocalFileManager; diff --git a/server/api/hooks/file-manager/S3FileManager.js b/server/api/hooks/file-manager/S3FileManager.js index 740fa25d..ba15cff5 100644 --- a/server/api/hooks/file-manager/S3FileManager.js +++ b/server/api/hooks/file-manager/S3FileManager.js @@ -4,6 +4,7 @@ */ const fs = require('fs'); +const mime = require('mime-types'); const { CopyObjectCommand, DeleteObjectsCommand, @@ -46,13 +47,28 @@ class S3FileManager { await upload.done(); } - async read(filePathSegment) { + async read(filePathSegment, { withHeaders = false } = {}) { const command = new GetObjectCommand({ Bucket: sails.config.custom.s3Bucket, Key: filePathSegment, }); const result = await this.client.send(command); + + if (withHeaders) { + return [ + result.Body, + { + 'Content-Type': + result.ContentType || mime.lookup(filePathSegment) || 'application/octet-stream', + 'Content-Length': result.ContentLength, + 'Last-Modified': result.LastModified && result.LastModified.toUTCString(), + ETag: result.ETag, + 'Accept-Ranges': result.AcceptRanges || 'bytes', + }, + ]; + } + return result.Body; } @@ -196,10 +212,10 @@ class S3FileManager { return !!result.Contents && result.Contents.length === 1; } - // eslint-disable-next-line class-methods-use-this + /* // eslint-disable-next-line class-methods-use-this buildUrl(filePathSegment) { return `${sails.hooks.s3.getBaseUrl()}/${filePathSegment}`; - } + } */ } module.exports = S3FileManager; diff --git a/server/config/routes.js b/server/config/routes.js index e65ec4b3..83c9ccf3 100644 --- a/server/config/routes.js +++ b/server/config/routes.js @@ -9,7 +9,6 @@ */ const path = require('path'); -const serveStatic = require('serve-static'); const sails = require('sails'); // Remove prefix from urlPath, assuming completely matches a subpath of @@ -20,9 +19,10 @@ const sails = require('sails'); // '/foo', '/foo' -> '/' // '/foo', '/foo?baz=bux' -> '/?baz=bux' // '/foo', '/foobar' -> '/foobar' -function removeRoutePrefix(prefix, urlPath) { +const removeRoutePrefix = (prefix, urlPath) => { if (urlPath.startsWith(prefix)) { const subpath = urlPath.substring(prefix.length); + if (subpath.startsWith('/')) { // Prefix matched a complete set of path segments, with a valid path // remaining. @@ -40,30 +40,67 @@ function removeRoutePrefix(prefix, urlPath) { // (e.g. we don't want to treat '/foo' as a prefix of '/foobar'). Leave the // path as-is. return urlPath; -} +}; -function staticDirServer(prefix, dirFn) { - return function handleReq(req, res, next) { - // Custom config properties are not available when the routes config is - // loaded, so resolve the target value just before serving the request. - const dir = dirFn(); - const staticServer = serveStatic(dir, { - index: false, - maxAge: sails.config.http.cache, - immutable: true, +const serveStatic = async (prefix, getPathSegment, req, res) => { + // Custom config properties are not available when the routes config is + // loaded, so resolve the target value just before serving the request. + const pathSegment = getPathSegment(); + // Remove the leading route prefix, since it's already included in path + // segment. + const normalizedUrlPath = removeRoutePrefix(prefix, req.url); + + const fileManager = sails.hooks['file-manager'].getInstance(); + const filePathSegment = path.join(pathSegment, normalizedUrlPath); + + let readStream; + let headers; + + try { + [readStream, headers] = await fileManager.read(filePathSegment, { + withHeaders: true, }); + } catch (err) { + return res.sendStatus(404); + } - const reqPath = req.url; - if (reqPath.startsWith(prefix)) { - // serve-static treats the request url as a sub-path of - // static root; remove the leading route prefix so the static root - // doesn't have to include the prefix as a subdirectory. - req.url = removeRoutePrefix(prefix, req.url); - return staticServer(req, res, next); + res.set({ + ...headers, + 'Cache-Control': `private, max-age=${sails.config.http.cache}, immutable`, + }); + + readStream.on('error', () => { + if (res.headersSent) { + res.destroy(); + } else { + res.sendStatus(404); } + }); + + return readStream.pipe(res); +}; + +const publicStaticDirServer = (prefix, getPathSegment) => (req, res, next) => { + if (!req.url.startsWith(prefix)) { return next(); - }; -} + } + + return serveStatic(prefix, getPathSegment, req, res); +}; + +const protectedStaticDirServer = (prefix, getPathSegment) => (req, res, next) => { + if (!req.url.startsWith(prefix)) { + return next(); + } + + try { + sails.helpers.utils.verifyJwtToken(req.cookies.accessToken); + } catch (error) { + return res.sendStatus(401); + } + + return serveStatic(prefix, getPathSegment, req, res); +}; module.exports.routes = { 'GET /api/bootstrap': 'bootstrap/show', @@ -199,41 +236,27 @@ module.exports.routes = { 'PATCH /api/_internal/config': '_internal/update-config', 'GET /preloaded-favicons/*': { - fn: staticDirServer('/preloaded-favicons', () => - path.join( - path.resolve(sails.config.custom.uploadsBasePath), - sails.config.custom.preloadedFaviconsPathSegment, - ), + fn: publicStaticDirServer( + '/preloaded-favicons', + () => sails.config.custom.preloadedFaviconsPathSegment, ), skipAssets: false, }, 'GET /favicons/*': { - fn: staticDirServer('/favicons', () => - path.join( - path.resolve(sails.config.custom.uploadsBasePath), - sails.config.custom.faviconsPathSegment, - ), - ), + fn: protectedStaticDirServer('/favicons', () => sails.config.custom.faviconsPathSegment), skipAssets: false, }, 'GET /user-avatars/*': { - fn: staticDirServer('/user-avatars', () => - path.join( - path.resolve(sails.config.custom.uploadsBasePath), - sails.config.custom.userAvatarsPathSegment, - ), - ), + fn: protectedStaticDirServer('/user-avatars', () => sails.config.custom.userAvatarsPathSegment), skipAssets: false, }, 'GET /background-images/*': { - fn: staticDirServer('/background-images', () => - path.join( - path.resolve(sails.config.custom.uploadsBasePath), - sails.config.custom.backgroundImagesPathSegment, - ), + fn: protectedStaticDirServer( + '/background-images', + () => sails.config.custom.backgroundImagesPathSegment, ), skipAssets: false, }, diff --git a/server/package-lock.json b/server/package-lock.json index d9e93526..1b024fe0 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -25,6 +25,7 @@ "knex": "^3.1.0", "lodash": "^4.17.23", "mime": "^3.0.0", + "mime-types": "^3.0.2", "moment": "^2.30.1", "nodemailer": "^7.0.12", "openid-client": "^5.7.1", @@ -2737,6 +2738,27 @@ "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/accepts/node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -5424,6 +5446,29 @@ "node": ">= 6" } }, + "node_modules/form-data/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/form-data/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/formidable": { "version": "3.5.4", "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", @@ -7388,24 +7433,19 @@ } }, "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", "dependencies": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/minimatch": { @@ -10938,6 +10978,27 @@ "node": ">= 0.6" } }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", diff --git a/server/package.json b/server/package.json index dd03ea7d..61263eb4 100644 --- a/server/package.json +++ b/server/package.json @@ -79,6 +79,7 @@ "knex": "^3.1.0", "lodash": "^4.17.23", "mime": "^3.0.0", + "mime-types": "^3.0.2", "moment": "^2.30.1", "nodemailer": "^7.0.12", "openid-client": "^5.7.1",