mirror of
https://github.com/DIYgod/RSSHub.git
synced 2026-09-24 23:12:34 +08:00
feat(core/middleware/anti-hotlink): route matching (#9681)
* feat(core/middleware/anti-hotlink): route matching Signed-off-by: Rongrong <i@rong.moe> * fix(test): coverage Signed-off-by: Rongrong <i@rong.moe> * docs(install): fix mistaken config key Signed-off-by: Rongrong <i@rong.moe>
This commit is contained in:
Rongrong
@@ -521,14 +521,30 @@ See the relation between access key/code and white/blacklisting.
|
||||
|
||||
`SENTRY_ROUTE_TIMEOUT`: Report Sentry if route execution takes more than this milliseconds, default to `3000`
|
||||
|
||||
### Image Processing
|
||||
|
||||
`HOTLINK_TEMPLATE`: replace image URL in the description to avoid anti-hotlink protection, leave it blank to disable this function. Usage reference [#2769](https://github.com/DIYgod/RSSHub/issues/2769). You may use any property listed in [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties), format of JS template literal. e.g. `${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}`
|
||||
|
||||
`HOTLINK_INCLUDE_PATHS`: limit the routes to be processed, only matched routes will be processed. Set multiple values with comma `,` as delimiter. If not set, all routes will be processed
|
||||
|
||||
`HOTLINK_EXCLUDE_PATHS`: exclude routes that do not need to be processed, all matched routes will be ignored. Set multiple values with comma `,` as delimiter. Can be used alone, or to exclude routes that are already included by `HOTLINK_INCLUDE_PATHS`. If not set, no routes will be ignored
|
||||
|
||||
::: tip Route matching pattern
|
||||
|
||||
`HOTLINK_INCLUDE_PATHS` and `HOTLINK_EXCLUDE_PATHS` match the root path and all recursive sub-paths of the route, but not substrings. Note that the path must start with `/` and end without `/`.
|
||||
|
||||
e.g. `/example`, `/example/sub` and `/example/anthoer/sub/route` will be matched by `/example`, but `/example_route` will not be matched.
|
||||
|
||||
It is also valid to contain route parameters, e.g. `/weibo/user/2612249974`.
|
||||
|
||||
:::
|
||||
|
||||
### Other Application Configurations
|
||||
|
||||
`DISALLOW_ROBOT`: prevent indexing by search engine, default to enable, set false or 0 to disable
|
||||
|
||||
`ENABLE_CLUSTER`: enable cluster mode, default to `false`
|
||||
|
||||
`HOTLINK_TEMPLATE`: replace image link in the description to avoid anti-hotlink protection, leave blank to disable this function. Usage reference [#2769](https://github.com/DIYgod/RSSHub/issues/2769). You may use any property listed in [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties), format of JS template literal. e.g. `${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}`
|
||||
|
||||
`NODE_ENV`: display error message on pages for authentication failing, default to `production` (i.e. no display)
|
||||
|
||||
`NODE_NAME`: node name, used for load balancing, identify the current node
|
||||
|
||||
+18
-2
@@ -527,14 +527,30 @@ RSSHub 支持使用访问密钥 / 码，白名单和黑名单三种方式进行
|
||||
|
||||
`SENTRY_ROUTE_TIMEOUT`: 路由耗时超过此毫秒值上报 Sentry，默认 `3000`
|
||||
|
||||
### 图片处理
|
||||
|
||||
`HOTLINK_TEMPLATE`: 用于处理描述中图片的 URL，绕过防盗链等限制，留空不生效。用法参考 [#2769](https://github.com/DIYgod/RSSHub/issues/2769)。可以使用 [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties) 的所有属性，格式为 JS 变量模板。例子：`${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}`
|
||||
|
||||
`HOTLINK_INCLUDE_PATHS`: 限制需要处理的路由，只有匹配成功的路由会被处理，设置多项时用英文逗号 `,` 隔开。若不设置，则所有路由都将被处理
|
||||
|
||||
`HOTLINK_EXCLUDE_PATHS`: 排除不需处理的路由，所有匹配成功的路由都不被处理，设置多项时用英文逗号 `,` 隔开。可单独使用，也可用于排除已被前者包含的路由。若不设置，则没有任何路由会被过滤
|
||||
|
||||
::: tip 路由匹配模式
|
||||
|
||||
`HOTLINK_INCLUDE_PATHS` 和 `HOTLINK_EXCLUDE_PATHS` 均匹配路由根路径及其所有递归子路径，但并非子字符串匹配。注意必须以 `/` 开头，且结尾不需要 `/`。
|
||||
|
||||
例：`/example`, `/example/sub` 和 `/example/anthoer/sub/route` 均可被 `/example` 匹配，但 `/example_route` 不会被匹配。
|
||||
|
||||
也可带有路由参数，如 `/weibo/user/2612249974` 也是合法的。
|
||||
|
||||
:::
|
||||
|
||||
### 其他应用配置
|
||||
|
||||
`DISALLOW_ROBOT`: 阻止搜索引擎收录，默认开启，设置 false 或 0 关闭
|
||||
|
||||
`ENABLE_CLUSTER`: 是否开启集群模式，默认 `false`
|
||||
|
||||
`HOTLINK_TEMPLATE`: 用于处理描述中图片的链接，绕过防盗链等限制，留空不生效。用法参考 [#2769](https://github.com/DIYgod/RSSHub/issues/2769)。可以使用 [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties) 的所有属性，格式为 JS 变量模板。例子：`${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}`
|
||||
|
||||
`NODE_ENV`: 是否显示错误输出，默认 `production` （即关闭输出）
|
||||
|
||||
`NODE_NAME`: 节点名，用于负载均衡，识别当前节点
|
||||
|
||||
@@ -89,6 +89,8 @@ const calculateValue = () => {
|
||||
// feed config
|
||||
hotlink: {
|
||||
template: envs.HOTLINK_TEMPLATE,
|
||||
includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','),
|
||||
excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','),
|
||||
},
|
||||
suffix: envs.SUFFIX,
|
||||
titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
|
||||
|
||||
@@ -2,6 +2,23 @@ const config = require('@/config').value;
|
||||
const cheerio = require('cheerio');
|
||||
const logger = require('@/utils/logger');
|
||||
|
||||
// match path or sub-path
|
||||
const matchPath = (path, paths) => {
|
||||
for (const p of paths) {
|
||||
if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// return ture if the path needs to be processed
|
||||
const filterPath = (path) => {
|
||||
const include = config.hotlink.includePaths;
|
||||
const exclude = config.hotlink.excludePaths;
|
||||
return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude));
|
||||
};
|
||||
|
||||
const interpolate = (str, obj) => str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]);
|
||||
// I don't want to keep another regex and
|
||||
// URL will be the standard way to parse URL
|
||||
@@ -39,22 +56,25 @@ module.exports = async (ctx, next) => {
|
||||
await next();
|
||||
|
||||
const template = config.hotlink.template;
|
||||
|
||||
if (!template || !filterPath(ctx.request.path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Assume that only description include image link
|
||||
// and here we will only check them in description.
|
||||
// Use Cheerio to load the description as html and filter all
|
||||
// image link
|
||||
if (template) {
|
||||
if (ctx.state.data) {
|
||||
if (ctx.state.data.description) {
|
||||
ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
|
||||
}
|
||||
|
||||
ctx.state.data.item &&
|
||||
ctx.state.data.item.forEach((item) => {
|
||||
if (item.description) {
|
||||
item.description = replaceUrls(item.description, template);
|
||||
}
|
||||
});
|
||||
if (ctx.state.data) {
|
||||
if (ctx.state.data.description) {
|
||||
ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
|
||||
}
|
||||
|
||||
ctx.state.data.item &&
|
||||
ctx.state.data.item.forEach((item) => {
|
||||
if (item.description) {
|
||||
item.description = replaceUrls(item.description, template);
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -181,5 +181,6 @@ module.exports = async (ctx) => {
|
||||
link: 'https://github.com/DIYgod/RSSHub',
|
||||
item,
|
||||
allowEmpty: ctx.params.id === 'allow_empty',
|
||||
description: ctx.params.id === 'complicated' ? '<img src="http://mock.com/DIYgod/DIYgod/RSSHub">' : 'A test route for RSSHub',
|
||||
};
|
||||
};
|
||||
|
||||
+105
-45
@@ -6,14 +6,66 @@ let server;
|
||||
|
||||
afterAll(() => {
|
||||
delete process.env.HOTLINK_TEMPLATE;
|
||||
delete process.env.HOTLINK_INCLUDE_PATHS;
|
||||
delete process.env.HOTLINK_EXCLUDE_PATHS;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.HOTLINK_TEMPLATE;
|
||||
delete process.env.HOTLINK_INCLUDE_PATHS;
|
||||
delete process.env.HOTLINK_EXCLUDE_PATHS;
|
||||
jest.resetModules();
|
||||
server.close();
|
||||
});
|
||||
|
||||
const origin1 = `<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
|
||||
<a href="http://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
|
||||
<img data-src="/DIYgod/RSSHub.jpg" src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
<img data-mock="/DIYgod/RSSHub.png" src="https://mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
|
||||
<img mock="/DIYgod/RSSHub.gif" src="https://mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
|
||||
<img src="http://mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`;
|
||||
|
||||
const processed1 = `<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
|
||||
<a href="http://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
|
||||
<img data-src="/DIYgod/RSSHub.jpg" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
<img data-mock="/DIYgod/RSSHub.png" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
|
||||
<img mock="/DIYgod/RSSHub.gif" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`;
|
||||
|
||||
const origin2 = `<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`;
|
||||
|
||||
const processed2 = `<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`;
|
||||
|
||||
const oriRouteDesc = '<img src="http://mock.com/DIYgod/DIYgod/RSSHub"> - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)';
|
||||
|
||||
const proRouteDesc = '<img src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub"> - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)';
|
||||
|
||||
const testAntiHotlink = async (expect1, expect2, expectRouteDesc) => {
|
||||
server = require('../../lib/index');
|
||||
const request = supertest(server);
|
||||
|
||||
const response = await request.get('/test/complicated');
|
||||
const parsed = await parser.parseString(response.text);
|
||||
expect(parsed.items[0].content).toBe(expect1);
|
||||
expect(parsed.items[1].content).toBe(expect2);
|
||||
expect(parsed.description).toBe(expectRouteDesc);
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const expectOrigin = async () => await testAntiHotlink(origin1, origin2, oriRouteDesc);
|
||||
|
||||
const expectProcessed = async () => await testAntiHotlink(processed1, processed2, proRouteDesc);
|
||||
|
||||
describe('anti-hotlink', () => {
|
||||
// First-time require is really, really slow.
|
||||
// If someone merely runs this test unit instead of the whole suite and this stage does not exist,
|
||||
@@ -24,58 +76,66 @@ describe('anti-hotlink', () => {
|
||||
|
||||
it('template', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
server = require('../../lib/index');
|
||||
const request = supertest(server);
|
||||
|
||||
const response = await request.get('/test/complicated');
|
||||
const parsed = await parser.parseString(response.text);
|
||||
expect(parsed.items[0].content).toBe(
|
||||
`<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
|
||||
<a href="http://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
|
||||
<img data-src="/DIYgod/RSSHub.jpg" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
<img data-mock="/DIYgod/RSSHub.png" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
|
||||
<img mock="/DIYgod/RSSHub.gif" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`
|
||||
);
|
||||
expect(parsed.items[1].content).toBe(`<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`);
|
||||
await expectProcessed();
|
||||
});
|
||||
|
||||
const origin1 = `<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
|
||||
<a href="http://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
|
||||
<img data-src="/DIYgod/RSSHub.jpg" src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
|
||||
<img data-mock="/DIYgod/RSSHub.png" src="https://mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
|
||||
<img mock="/DIYgod/RSSHub.gif" src="https://mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
|
||||
<img src="http://mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`;
|
||||
|
||||
const origin2 = `<a href="https://mock.com/DIYgod/RSSHub"></a>
|
||||
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`;
|
||||
|
||||
const testOrigin = async () => {
|
||||
server = require('../../lib/index');
|
||||
const request = supertest(server);
|
||||
|
||||
const response = await request.get('/test/complicated');
|
||||
const parsed = await parser.parseString(response.text);
|
||||
expect(parsed.items[0].content).toBe(origin1);
|
||||
expect(parsed.items[1].content).toBe(origin2);
|
||||
};
|
||||
|
||||
it('url', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = '${protocol}//${host}${pathname}';
|
||||
await testOrigin();
|
||||
await expectOrigin();
|
||||
});
|
||||
|
||||
it('no-template', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = '';
|
||||
await testOrigin();
|
||||
await expectOrigin();
|
||||
});
|
||||
|
||||
it('include-paths-partial-matched', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_INCLUDE_PATHS = '/test';
|
||||
await expectProcessed();
|
||||
});
|
||||
|
||||
it('include-paths-fully-matched', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_INCLUDE_PATHS = '/test/complicated';
|
||||
await expectProcessed();
|
||||
});
|
||||
|
||||
it('include-paths-unmatched', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_INCLUDE_PATHS = '/t';
|
||||
await expectOrigin();
|
||||
});
|
||||
|
||||
it('exclude-paths-partial-matched', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_EXCLUDE_PATHS = '/test';
|
||||
await expectOrigin();
|
||||
});
|
||||
|
||||
it('exclude-paths-fully-matched', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_EXCLUDE_PATHS = '/test/complicated';
|
||||
await expectOrigin();
|
||||
});
|
||||
|
||||
it('exclude-paths-unmatched', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_EXCLUDE_PATHS = '/t';
|
||||
await expectProcessed();
|
||||
});
|
||||
|
||||
it('include-exclude-paths-mixed-filtered-out', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_INCLUDE_PATHS = '/test';
|
||||
process.env.HOTLINK_EXCLUDE_PATHS = '/test/complicated';
|
||||
await expectOrigin();
|
||||
});
|
||||
|
||||
it('include-exclude-paths-mixed-unfiltered-out', async () => {
|
||||
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
|
||||
process.env.HOTLINK_INCLUDE_PATHS = '/test';
|
||||
process.env.HOTLINK_EXCLUDE_PATHS = '/test/c';
|
||||
await expectProcessed();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user