From a5d74e521cddd9b43b6caf8473966cbe6d003442 Mon Sep 17 00:00:00 2001 From: Rongrong Date: Wed, 4 May 2022 03:29:23 +0800 Subject: [PATCH] feat(core/middleware/anti-hotlink): route matching (#9681) * feat(core/middleware/anti-hotlink): route matching Signed-off-by: Rongrong * fix(test): coverage Signed-off-by: Rongrong * docs(install): fix mistaken config key Signed-off-by: Rongrong --- docs/en/install/README.md | 20 ++++- docs/install/README.md | 20 ++++- lib/config.js | 2 + lib/middleware/anti-hotlink.js | 44 +++++++--- lib/v2/test/index.js | 1 + test/middleware/anti-hotlink.js | 150 ++++++++++++++++++++++---------- 6 files changed, 176 insertions(+), 61 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index d70b3f79c8..307a2b3079 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -521,14 +521,30 @@ See the relation between access key/code and white/blacklisting. `SENTRY_ROUTE_TIMEOUT`: Report Sentry if route execution takes more than this milliseconds, default to `3000` +### Image Processing + +`HOTLINK_TEMPLATE`: replace image URL in the description to avoid anti-hotlink protection, leave it blank to disable this function. Usage reference [#2769](https://github.com/DIYgod/RSSHub/issues/2769). You may use any property listed in [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties), format of JS template literal. e.g. `${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}` + +`HOTLINK_INCLUDE_PATHS`: limit the routes to be processed, only matched routes will be processed. Set multiple values with comma `,` as delimiter. If not set, all routes will be processed + +`HOTLINK_EXCLUDE_PATHS`: exclude routes that do not need to be processed, all matched routes will be ignored. Set multiple values with comma `,` as delimiter. Can be used alone, or to exclude routes that are already included by `HOTLINK_INCLUDE_PATHS`. If not set, no routes will be ignored + +::: tip Route matching pattern + +`HOTLINK_INCLUDE_PATHS` and `HOTLINK_EXCLUDE_PATHS` match the root path and all recursive sub-paths of the route, but not substrings. Note that the path must start with `/` and end without `/`. + +e.g. `/example`, `/example/sub` and `/example/anthoer/sub/route` will be matched by `/example`, but `/example_route` will not be matched. + +It is also valid to contain route parameters, e.g. `/weibo/user/2612249974`. + +::: + ### Other Application Configurations `DISALLOW_ROBOT`: prevent indexing by search engine, default to enable, set false or 0 to disable `ENABLE_CLUSTER`: enable cluster mode, default to `false` -`HOTLINK_TEMPLATE`: replace image link in the description to avoid anti-hotlink protection, leave blank to disable this function. Usage reference [#2769](https://github.com/DIYgod/RSSHub/issues/2769). You may use any property listed in [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties), format of JS template literal. e.g. `${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}` - `NODE_ENV`: display error message on pages for authentication failing, default to `production` (i.e. no display) `NODE_NAME`: node name, used for load balancing, identify the current node diff --git a/docs/install/README.md b/docs/install/README.md index 4d21343af5..f5ff73e073 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -527,14 +527,30 @@ RSSHub 支持使用访问密钥 / 码，白名单和黑名单三种方式进行 `SENTRY_ROUTE_TIMEOUT`: 路由耗时超过此毫秒值上报 Sentry，默认 `3000` +### 图片处理 + +`HOTLINK_TEMPLATE`: 用于处理描述中图片的 URL，绕过防盗链等限制，留空不生效。用法参考 [#2769](https://github.com/DIYgod/RSSHub/issues/2769)。可以使用 [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties) 的所有属性，格式为 JS 变量模板。例子：`${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}` + +`HOTLINK_INCLUDE_PATHS`: 限制需要处理的路由，只有匹配成功的路由会被处理，设置多项时用英文逗号 `,` 隔开。若不设置，则所有路由都将被处理 + +`HOTLINK_EXCLUDE_PATHS`: 排除不需处理的路由，所有匹配成功的路由都不被处理，设置多项时用英文逗号 `,` 隔开。可单独使用，也可用于排除已被前者包含的路由。若不设置，则没有任何路由会被过滤 + +::: tip 路由匹配模式 + +`HOTLINK_INCLUDE_PATHS` 和 `HOTLINK_EXCLUDE_PATHS` 均匹配路由根路径及其所有递归子路径，但并非子字符串匹配。注意必须以 `/` 开头，且结尾不需要 `/`。 + +例：`/example`, `/example/sub` 和 `/example/anthoer/sub/route` 均可被 `/example` 匹配，但 `/example_route` 不会被匹配。 + +也可带有路由参数，如 `/weibo/user/2612249974` 也是合法的。 + +::: + ### 其他应用配置 `DISALLOW_ROBOT`: 阻止搜索引擎收录，默认开启，设置 false 或 0 关闭 `ENABLE_CLUSTER`: 是否开启集群模式，默认 `false` -`HOTLINK_TEMPLATE`: 用于处理描述中图片的链接，绕过防盗链等限制，留空不生效。用法参考 [#2769](https://github.com/DIYgod/RSSHub/issues/2769)。可以使用 [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties) 的所有属性，格式为 JS 变量模板。例子：`${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}` - `NODE_ENV`: 是否显示错误输出，默认 `production` （即关闭输出） `NODE_NAME`: 节点名，用于负载均衡，识别当前节点 diff --git a/lib/config.js b/lib/config.js index f61b34df01..42bcde8b22 100644 --- a/lib/config.js +++ b/lib/config.js @@ -89,6 +89,8 @@ const calculateValue = () => { // feed config hotlink: { template: envs.HOTLINK_TEMPLATE, + includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','), + excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','), }, suffix: envs.SUFFIX, titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150, diff --git a/lib/middleware/anti-hotlink.js b/lib/middleware/anti-hotlink.js index 945f4b1eec..098396a06d 100644 --- a/lib/middleware/anti-hotlink.js +++ b/lib/middleware/anti-hotlink.js @@ -2,6 +2,23 @@ const config = require('@/config').value; const cheerio = require('cheerio'); const logger = require('@/utils/logger'); +// match path or sub-path +const matchPath = (path, paths) => { + for (const p of paths) { + if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) { + return true; + } + } + return false; +}; + +// return ture if the path needs to be processed +const filterPath = (path) => { + const include = config.hotlink.includePaths; + const exclude = config.hotlink.excludePaths; + return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude)); +}; + const interpolate = (str, obj) => str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]); // I don't want to keep another regex and // URL will be the standard way to parse URL @@ -39,22 +56,25 @@ module.exports = async (ctx, next) => { await next(); const template = config.hotlink.template; + + if (!template || !filterPath(ctx.request.path)) { + return; + } + // Assume that only description include image link // and here we will only check them in description. // Use Cheerio to load the description as html and filter all // image link - if (template) { - if (ctx.state.data) { - if (ctx.state.data.description) { - ctx.state.data.description = replaceUrls(ctx.state.data.description, template); - } - - ctx.state.data.item && - ctx.state.data.item.forEach((item) => { - if (item.description) { - item.description = replaceUrls(item.description, template); - } - }); + if (ctx.state.data) { + if (ctx.state.data.description) { + ctx.state.data.description = replaceUrls(ctx.state.data.description, template); } + + ctx.state.data.item && + ctx.state.data.item.forEach((item) => { + if (item.description) { + item.description = replaceUrls(item.description, template); + } + }); } }; diff --git a/lib/v2/test/index.js b/lib/v2/test/index.js index dc3396bcd4..73b4df65c6 100644 --- a/lib/v2/test/index.js +++ b/lib/v2/test/index.js @@ -181,5 +181,6 @@ module.exports = async (ctx) => { link: 'https://github.com/DIYgod/RSSHub', item, allowEmpty: ctx.params.id === 'allow_empty', + description: ctx.params.id === 'complicated' ? '' : 'A test route for RSSHub', }; }; diff --git a/test/middleware/anti-hotlink.js b/test/middleware/anti-hotlink.js index 6fd54f2e4b..6b0b9d8b3b 100644 --- a/test/middleware/anti-hotlink.js +++ b/test/middleware/anti-hotlink.js @@ -6,14 +6,66 @@ let server; afterAll(() => { delete process.env.HOTLINK_TEMPLATE; + delete process.env.HOTLINK_INCLUDE_PATHS; + delete process.env.HOTLINK_EXCLUDE_PATHS; }); afterEach(() => { delete process.env.HOTLINK_TEMPLATE; + delete process.env.HOTLINK_INCLUDE_PATHS; + delete process.env.HOTLINK_EXCLUDE_PATHS; jest.resetModules(); server.close(); }); +const origin1 = ` + + + + + + + + +`; + +const processed1 = ` + + + + + + + + +`; + +const origin2 = ` +`; + +const processed2 = ` +`; + +const oriRouteDesc = ' - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)'; + +const proRouteDesc = ' - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)'; + +const testAntiHotlink = async (expect1, expect2, expectRouteDesc) => { + server = require('../../lib/index'); + const request = supertest(server); + + const response = await request.get('/test/complicated'); + const parsed = await parser.parseString(response.text); + expect(parsed.items[0].content).toBe(expect1); + expect(parsed.items[1].content).toBe(expect2); + expect(parsed.description).toBe(expectRouteDesc); + return parsed; +}; + +const expectOrigin = async () => await testAntiHotlink(origin1, origin2, oriRouteDesc); + +const expectProcessed = async () => await testAntiHotlink(processed1, processed2, proRouteDesc); + describe('anti-hotlink', () => { // First-time require is really, really slow. // If someone merely runs this test unit instead of the whole suite and this stage does not exist, @@ -24,58 +76,66 @@ describe('anti-hotlink', () => { it('template', async () => { process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; - server = require('../../lib/index'); - const request = supertest(server); - - const response = await request.get('/test/complicated'); - const parsed = await parser.parseString(response.text); - expect(parsed.items[0].content).toBe( - ` - - - - - - - - -` - ); - expect(parsed.items[1].content).toBe(` -`); + await expectProcessed(); }); - const origin1 = ` - - - - - - - - -`; - - const origin2 = ` -`; - - const testOrigin = async () => { - server = require('../../lib/index'); - const request = supertest(server); - - const response = await request.get('/test/complicated'); - const parsed = await parser.parseString(response.text); - expect(parsed.items[0].content).toBe(origin1); - expect(parsed.items[1].content).toBe(origin2); - }; - it('url', async () => { process.env.HOTLINK_TEMPLATE = '${protocol}//${host}${pathname}'; - await testOrigin(); + await expectOrigin(); }); it('no-template', async () => { process.env.HOTLINK_TEMPLATE = ''; - await testOrigin(); + await expectOrigin(); + }); + + it('include-paths-partial-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test'; + await expectProcessed(); + }); + + it('include-paths-fully-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test/complicated'; + await expectProcessed(); + }); + + it('include-paths-unmatched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/t'; + await expectOrigin(); + }); + + it('exclude-paths-partial-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test'; + await expectOrigin(); + }); + + it('exclude-paths-fully-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test/complicated'; + await expectOrigin(); + }); + + it('exclude-paths-unmatched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_EXCLUDE_PATHS = '/t'; + await expectProcessed(); + }); + + it('include-exclude-paths-mixed-filtered-out', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test/complicated'; + await expectOrigin(); + }); + + it('include-exclude-paths-mixed-unfiltered-out', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test/c'; + await expectProcessed(); }); });