feat: internal media proxy (#14593)

* feat: image proxy

* feat: use Map

* docs: add docs

* fix: change config name

* Add lightnovel.us as a media source

* fix: break hostname extraction

* feat: add psl for more accurate domain extraction

* feat: split referer map

* perf(deps): replace `psl` with `tldts`

* fix: add indienova.com to referer map

lib/config.js

@@ -108,6 +108,7 @@ const calculateValue = () => {
             allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
             filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
             allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true',
+            mediaProxyKey: envs.MEDIA_PROXY_KEY,
         },
         suffix: envs.SUFFIX,
         titleLengthLimit: Number.parseInt(envs.TITLE_LENGTH_LIMIT) || 150,

lib/v2/rsshub/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/m/:key/:url': ['TonyRL'],
     '/routes/:lang?': ['DIYgod'],
     '/transform/html/:url/:routeParams': ['ttttmr'],
     '/transform/json/:url/:routeParams': ['ttttmr'],

lib/v2/rsshub/media.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+const { getDomain } = require('tldts');
+const { refererMap } = require('./referer-map');
+
+module.exports = async (ctx) => {
+    if (!config.feature.mediaProxyKey) {
+        ctx.throw(403, 'Internal media proxy is disabled.');
+    }
+
+    const { key } = ctx.params;
+    if (key !== config.feature.mediaProxyKey) {
+        ctx.throw(401, 'Invalid media proxy key.');
+    }
+
+    const url = decodeURIComponent(ctx.params.url);
+    const requestUrl = new URL(url);
+    const { hostname, origin } = requestUrl;
+
+    const domain = getDomain(hostname);
+
+    let referer = refererMap.get(domain);
+    referer ||= origin;
+
+    const { headers } = await got.head(url, {
+        headers: {
+            referer,
+        },
+    });
+
+    const cacheControl = headers['cache-control'];
+    const contentType = headers['content-type'];
+    const contentLength = headers['content-length'];
+
+    if (!contentType.startsWith('image/') || headers.server === 'RSSHub') {
+        return ctx.redirect(url);
+    }
+
+    ctx.set({
+        'cache-control': cacheControl || `public, max-age=${config.cache.contentExpire}`,
+        'content-length': contentLength,
+        'content-type': contentType,
+        server: 'RSSHub',
+    });
+
+    ctx.body = await got.stream(url, {
+        headers: {
+            referer,
+        },
+    });
+};

lib/v2/rsshub/referer-map.js

@@ -0,0 +1,16 @@
+const refererMap = new Map([
+    ['fbcdn.net', 'https://www.facebook.com/'],
+    ['cdninstagram.com', 'https://www.instagram.com/'],
+    ['moyu.im', 'https://jandan.net/'],
+    ['lightnovel.us', 'https://www.lightnovel.us/'],
+    ['indienova.com', 'https://indienova.com/'],
+    ['pximg.net', 'https://www.pixiv.net/'],
+    ['me8gs.app', 'https://www.sehuatang.net/'],
+    ['rxn30.app', 'https://www.sehuatang.net/'],
+    ['sex.com', 'https://www.sex.com/'],
+    ['sinaimg.cn', 'https://weibo.com/'],
+]);
+
+module.exports = {
+    refererMap,
+};

lib/v2/rsshub/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/m/:key/:url', require('./media'));
     router.get('/routes/:lang?', require('./routes'));
     router.get('/transform/html/:url/:routeParams', require('./transform/html'));
     router.get('/transform/json/:url/:routeParams', require('./transform/json'));