From cbbd82918eebd6c8405f2d1b2c5b1bc96529cea0 Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 29 Feb 2024 08:38:15 +0800 Subject: [PATCH] feat: internal media proxy (#14593) * feat: image proxy * feat: use Map * docs: add docs * fix: change config name * Add lightnovel.us as a media source * fix: break hostname extraction * feat: add psl for more accurate domain extraction * feat: split referer map * perf(deps): replace `psl` with `tldts` * fix: add indienova.com to referer map --- lib/config.js | 1 + lib/v2/rsshub/maintainer.js | 1 + lib/v2/rsshub/media.js | 51 +++++++++++++++++++ lib/v2/rsshub/referer-map.js | 16 ++++++ lib/v2/rsshub/router.js | 1 + package.json | 1 + pnpm-lock.yaml | 14 +++++ website/docs/install/config.md | 4 +- website/docs/routes/other.mdx | 14 +++-- .../current/install/config.md | 4 +- 10 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 lib/v2/rsshub/media.js create mode 100644 lib/v2/rsshub/referer-map.js diff --git a/lib/config.js b/lib/config.js index 00c978a8a6..fda15849d5 100644 --- a/lib/config.js +++ b/lib/config.js @@ -108,6 +108,7 @@ const calculateValue = () => { allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true', filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true', + mediaProxyKey: envs.MEDIA_PROXY_KEY, }, suffix: envs.SUFFIX, titleLengthLimit: Number.parseInt(envs.TITLE_LENGTH_LIMIT) || 150, diff --git a/lib/v2/rsshub/maintainer.js b/lib/v2/rsshub/maintainer.js index 158998b451..1fef94f9c6 100644 --- a/lib/v2/rsshub/maintainer.js +++ b/lib/v2/rsshub/maintainer.js @@ -1,4 +1,5 @@ module.exports = { + '/m/:key/:url': ['TonyRL'], '/routes/:lang?': ['DIYgod'], '/transform/html/:url/:routeParams': ['ttttmr'], '/transform/json/:url/:routeParams': ['ttttmr'], diff --git a/lib/v2/rsshub/media.js b/lib/v2/rsshub/media.js new file mode 100644 index 0000000000..99168d1585 --- /dev/null +++ b/lib/v2/rsshub/media.js @@ -0,0 +1,51 @@ +const got = require('@/utils/got'); +const config = require('@/config').value; +const { getDomain } = require('tldts'); +const { refererMap } = require('./referer-map'); + +module.exports = async (ctx) => { + if (!config.feature.mediaProxyKey) { + ctx.throw(403, 'Internal media proxy is disabled.'); + } + + const { key } = ctx.params; + if (key !== config.feature.mediaProxyKey) { + ctx.throw(401, 'Invalid media proxy key.'); + } + + const url = decodeURIComponent(ctx.params.url); + const requestUrl = new URL(url); + const { hostname, origin } = requestUrl; + + const domain = getDomain(hostname); + + let referer = refererMap.get(domain); + referer ||= origin; + + const { headers } = await got.head(url, { + headers: { + referer, + }, + }); + + const cacheControl = headers['cache-control']; + const contentType = headers['content-type']; + const contentLength = headers['content-length']; + + if (!contentType.startsWith('image/') || headers.server === 'RSSHub') { + return ctx.redirect(url); + } + + ctx.set({ + 'cache-control': cacheControl || `public, max-age=${config.cache.contentExpire}`, + 'content-length': contentLength, + 'content-type': contentType, + server: 'RSSHub', + }); + + ctx.body = await got.stream(url, { + headers: { + referer, + }, + }); +}; diff --git a/lib/v2/rsshub/referer-map.js b/lib/v2/rsshub/referer-map.js new file mode 100644 index 0000000000..b3406068d6 --- /dev/null +++ b/lib/v2/rsshub/referer-map.js @@ -0,0 +1,16 @@ +const refererMap = new Map([ + ['fbcdn.net', 'https://www.facebook.com/'], + ['cdninstagram.com', 'https://www.instagram.com/'], + ['moyu.im', 'https://jandan.net/'], + ['lightnovel.us', 'https://www.lightnovel.us/'], + ['indienova.com', 'https://indienova.com/'], + ['pximg.net', 'https://www.pixiv.net/'], + ['me8gs.app', 'https://www.sehuatang.net/'], + ['rxn30.app', 'https://www.sehuatang.net/'], + ['sex.com', 'https://www.sex.com/'], + ['sinaimg.cn', 'https://weibo.com/'], +]); + +module.exports = { + refererMap, +}; diff --git a/lib/v2/rsshub/router.js b/lib/v2/rsshub/router.js index 7e580dcc58..f74d4aa767 100644 --- a/lib/v2/rsshub/router.js +++ b/lib/v2/rsshub/router.js @@ -1,4 +1,5 @@ module.exports = (router) => { + router.get('/m/:key/:url', require('./media')); router.get('/routes/:lang?', require('./routes')); router.get('/transform/html/:url/:routeParams', require('./transform/html')); router.get('/transform/json/:url/:routeParams', require('./transform/json')); diff --git a/package.json b/package.json index e8843829b5..05dac9781c 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "source-map": "0.7.4", "tiny-async-pool": "2.1.0", "title": "3.5.3", + "tldts": "6.1.1", "tough-cookie": "4.1.3", "twitter-api-v2": "1.16.0", "uuid": "9.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fbffccbf8b..2548e1a3bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -200,6 +200,9 @@ dependencies: title: specifier: 3.5.3 version: 3.5.3 + tldts: + specifier: 6.1.1 + version: 6.1.1 tough-cookie: specifier: 4.1.3 version: 4.1.3 @@ -8995,6 +8998,17 @@ packages: os-tmpdir: 1.0.2 dev: true + /tldts-core@6.1.1: + resolution: {integrity: sha512-xBHFfOO2YmEwogupGTKR0IBXe1IJe1/GleNeXpO294Fk90aQSvrop41BKA66CkfNLVOuomJDZ304KxwovT04Vw==} + dev: false + + /tldts@6.1.1: + resolution: {integrity: sha512-uV5xEtjR8VdMZZU0my9zLqWJNAkG0PZ5l5t9F1dBYwOyeFeWTFd0emxEs12Y1U39XKD+dF2ElzNU59Qq0Z4PGQ==} + hasBin: true + dependencies: + tldts-core: 6.1.1 + dev: false + /tmpl@1.0.5: resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==} dev: true diff --git a/website/docs/install/config.md b/website/docs/install/config.md index 38e056648e..0997b3a9b3 100644 --- a/website/docs/install/config.md +++ b/website/docs/install/config.md @@ -209,7 +209,7 @@ It is also valid to contain route parameters, e.g. `/weibo/user/2612249974`. ## Features -:::tip Experimental features +:::tip[Experimental features] Configs in this sections are in beta stage, and **are turn off by default**. Please read corresponded description and turn on if necessary. @@ -221,6 +221,8 @@ Configs in this sections are in beta stage, and **are turn off by default**. Ple `ALLOW_USER_SUPPLY_UNSAFE_DOMAIN`: allow users to provide a domain as a parameter to routes that are not in their allow list, respectively. Public instances are suggested to leave this value default, as it may lead to [Server-Side Request Forgery (SSRF)](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery) +`MEDIA_PROXY_KEY`: the access key for internal media proxy. + ## Other Application Configurations `DISALLOW_ROBOT`: prevent indexing by search engine, default to enable, set false or 0 to disable diff --git a/website/docs/routes/other.mdx b/website/docs/routes/other.mdx index 53d4a990d0..5c0fcad4e9 100644 --- a/website/docs/routes/other.mdx +++ b/website/docs/routes/other.mdx @@ -440,12 +440,16 @@ It is recommended to use with clipping tools such as Notion Web Clipper. | all | development | design | operation | product | other | marketing | sales | -## Transformation {#transformation} +## RSSHub {#rsshub} + +### Internal Media Proxy {#rsshub-internal-media-proxy} + + + +### Transformation - HTML {#rsshub-transformation-html} Pass URL and transformation rules to convert HTML/JSON into RSS. -### HTML {#transformation-html} - Specify options (in the format of query string) in parameter `routeParams` parameter to extract data from HTML. | Key | Meaning | Accepted Values | Default | @@ -476,7 +480,7 @@ Specify options (in the format of query string) in parameter `routeParams` param | `item` | `div[class='post-content'] p a` | -### JSON {#transformation-json} +### Transformation - JSON {#rsshub-transformation-json} Specify options (in the format of query string) in parameter `routeParams` parameter to extract data from JSON. @@ -511,7 +515,7 @@ JSON Path only supports format like `a.b.c`. if you need to access arrays, like | `itemDesc` | `body` | -### Sitemap {#transformation-sitemap} +### Transformation - Sitemap {#rsshub-transformation-sitemap} Specify options (in the format of query string) in parameter `routeParams` parameter to extract data from Sitemap. (Follows Sitemap Protocol 0.9) diff --git a/website/i18n/zh/docusaurus-plugin-content-docs/current/install/config.md b/website/i18n/zh/docusaurus-plugin-content-docs/current/install/config.md index 945520ad71..ee7405503b 100644 --- a/website/i18n/zh/docusaurus-plugin-content-docs/current/install/config.md +++ b/website/i18n/zh/docusaurus-plugin-content-docs/current/install/config.md @@ -209,7 +209,7 @@ RSSHub 支持使用访问密钥 / 码，允许清单和拒绝清单三种方式 ## 功能特性 -:::tip 测试特性 +:::tip[测试特性] 这个板块控制的是一些新特性的选项，他们都是**默认关闭**的。如果有需要请阅读对应说明后按需开启 @@ -221,6 +221,8 @@ RSSHub 支持使用访问密钥 / 码，允许清单和拒绝清单三种方式 `ALLOW_USER_SUPPLY_UNSAFE_DOMAIN`: 允许用户为路由提供域名作为参数。建议公共实例不要调整此选项，开启后可能会导致 [服务端请求伪造（SSRF）](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery) +`MEDIA_PROXY_KEY`: 内置多媒体代理的访问密钥 + ## 其他应用配置 `DISALLOW_ROBOT`: 阻止搜索引擎收录，默认开启，设置 false 或 0 关闭