From c86782b91b28eb802e1f4442159565c5f2b493a5 Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 23 Jan 2022 03:02:27 +0700 Subject: [PATCH] fix(route): Financial Times HTTPS support (#8765) --- docs/traditional-media.md | 1 - lib/router.js | 6 ++--- lib/routes/ft/myft.js | 46 -------------------------------- lib/{routes => v2}/ft/channel.js | 1 + lib/v2/ft/maintainer.js | 4 +++ lib/v2/ft/myft.js | 44 ++++++++++++++++++++++++++++++ lib/v2/ft/radar.js | 24 +++++++++++++++++ lib/v2/ft/router.js | 4 +++ lib/{routes => v2}/ft/utils.js | 38 +++++++++++++------------- 9 files changed, 99 insertions(+), 69 deletions(-) delete mode 100644 lib/routes/ft/myft.js rename lib/{routes => v2}/ft/channel.js (94%) create mode 100644 lib/v2/ft/maintainer.js create mode 100644 lib/v2/ft/myft.js create mode 100644 lib/v2/ft/radar.js create mode 100644 lib/v2/ft/router.js rename lib/{routes => v2}/ft/utils.js (65%) diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 428bd7e51a..a67bbbe00e 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -157,7 +157,6 @@ pageClass: routes ::: tip 提示 - 不支持付费文章。 -- 由于未知原因 FT 中文网的 SSL 证书不被信任 （参见 [SSL Labs 报告](https://www.ssllabs.com/ssltest/analyze.html?d=www.ftchinese.com&latest)), 所有文章通过 http 协议获取。 ::: diff --git a/lib/router.js b/lib/router.js index 49724e7bd1..25b935f614 100644 --- a/lib/router.js +++ b/lib/router.js @@ -988,9 +988,9 @@ router.get('/nhk/news_web_easy', lazyloadRouteHandler('./routes/nhk/news_web_eas // BBC router.get('/bbc/:site?/:channel?', lazyloadRouteHandler('./routes/bbc/index')); -// Financial Times -router.get('/ft/myft/:key', lazyloadRouteHandler('./routes/ft/myft')); -router.get('/ft/:language/:channel?', lazyloadRouteHandler('./routes/ft/channel')); +// Financial Times migrated to v2 +// router.get('/ft/myft/:key', lazyloadRouteHandler('./routes/ft/myft')); +// router.get('/ft/:language/:channel?', lazyloadRouteHandler('./routes/ft/channel')); // The Verge router.get('/verge', lazyloadRouteHandler('./routes/verge/index')); diff --git a/lib/routes/ft/myft.js b/lib/routes/ft/myft.js deleted file mode 100644 index 8b82025a12..0000000000 --- a/lib/routes/ft/myft.js +++ /dev/null @@ -1,46 +0,0 @@ -const got = require('@/utils/got'); -const parser = require('@/utils/rss-parser'); -const cheerio = require('cheerio'); - -module.exports = async (ctx) => { - const ProcessFeed = (content) => { - // clean up the article - content.find('div.o-share, aside, div.o-ads').remove(); - - return content.html(); - }; - - const link = `https://www.ft.com/myft/following/${ctx.params.key}.rss`; - - const feed = await parser.parseURL(link); - - const items = await Promise.all( - feed.items.map(async (item) => { - const response = await got({ - method: 'get', - url: item.link, - headers: { - Referer: 'https://www.facebook.com', - }, - }); - - const $ = cheerio.load(response.data); - - const single = { - title: item.title, - description: ProcessFeed($('div.article__content-body')), - author: $('a.n-content-tag--author').text(), - pubDate: item.pubDate, - link: item.link, - }; - return Promise.resolve(single); - }) - ); - - ctx.state.data = { - title: `FT.com - myFT`, - link, - description: `FT.com - myFT`, - item: items, - }; -}; diff --git a/lib/routes/ft/channel.js b/lib/v2/ft/channel.js similarity index 94% rename from lib/routes/ft/channel.js rename to lib/v2/ft/channel.js index dc1c6e57ce..f0c823233c 100644 --- a/lib/routes/ft/channel.js +++ b/lib/v2/ft/channel.js @@ -4,5 +4,6 @@ module.exports = async (ctx) => { ctx.state.data = await utils.getData({ site: ctx.params.language === 'chinese' ? 'www' : 'big5', channel: ctx.params.channel, + ctx, }); }; diff --git a/lib/v2/ft/maintainer.js b/lib/v2/ft/maintainer.js new file mode 100644 index 0000000000..cdbeeec836 --- /dev/null +++ b/lib/v2/ft/maintainer.js @@ -0,0 +1,4 @@ +module.exports = { + '/ft/myft/:key': ['HenryQW'], + '/ft/:language/:channel?': ['HenryQW', 'xyqfer'], +}; diff --git a/lib/v2/ft/myft.js b/lib/v2/ft/myft.js new file mode 100644 index 0000000000..c5d7d0b90c --- /dev/null +++ b/lib/v2/ft/myft.js @@ -0,0 +1,44 @@ +const got = require('@/utils/got'); +const parser = require('@/utils/rss-parser'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const ProcessFeed = (content) => { + // clean up the article + content.find('div.o-share, aside, div.o-ads').remove(); + + return content.html(); + }; + + const link = `https://www.ft.com/myft/following/${ctx.params.key}.rss`; + + const feed = await parser.parseURL(link); + + const items = await Promise.all( + feed.items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const response = await got({ + method: 'get', + url: item.link, + headers: { + Referer: 'https://www.facebook.com', + }, + }); + + const $ = cheerio.load(response.data); + + item.description = ProcessFeed($('div.article__content-body')); + item.author = $('a.n-content-tag--author').text(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: `FT.com - myFT`, + link, + description: `FT.com - myFT`, + item: items, + }; +}; diff --git a/lib/v2/ft/radar.js b/lib/v2/ft/radar.js new file mode 100644 index 0000000000..35238ff056 --- /dev/null +++ b/lib/v2/ft/radar.js @@ -0,0 +1,24 @@ +module.exports = { + 'ftchinese.com': { + _name: 'Financial Times', + '.': [ + { + title: 'FT 中文网', + docs: 'https://docs.rsshub.app/traditional-media.html#financial-times', + }, + { + title: 'myFT 个人 RSS', + docs: 'https://docs.rsshub.app/traditional-media.html#financial-times', + }, + ], + }, + 'ft.com': { + _name: 'Financial Times', + '.': [ + { + title: 'myFT personal RSS', + docs: 'https://docs.rsshub.app/en/traditional-media.html#financial-times', + }, + ], + }, +}; diff --git a/lib/v2/ft/router.js b/lib/v2/ft/router.js new file mode 100644 index 0000000000..11cf5c86a6 --- /dev/null +++ b/lib/v2/ft/router.js @@ -0,0 +1,4 @@ +module.exports = function (router) { + router.get('/myft/:key', require('./myft')); + router.get('/:language/:channel?', require('./channel')); +}; diff --git a/lib/routes/ft/utils.js b/lib/v2/ft/utils.js similarity index 65% rename from lib/routes/ft/utils.js rename to lib/v2/ft/utils.js index 794058248b..1e6a15378f 100644 --- a/lib/routes/ft/utils.js +++ b/lib/v2/ft/utils.js @@ -7,21 +7,21 @@ const ProcessFeed = ($, link) => { let content = $('div.story-container'); // 处理封面图片 - content.find('div.story-image > figure').each((i, e) => { + content.find('div.story-image > figure').each((_, e) => { const src = `https://thumbor.ftacademy.cn/unsafe/1340x754/${e.attribs['data-url']}`; $(``).insertAfter(content.find('div.story-lead')[0]); }); // 付费文章跳转 - content.find('div#subscribe-now-container').each((i, e) => { + content.find('div#subscribe-now-container').each((_, e) => { $(`

此文章为付费文章，会员请访问网站阅读。

`).insertAfter(content.find('div.story-body')[0]); $(e).remove(); }); // 获取作者 let author = ''; - content.find('span.story-author > a').each((i, e) => { + content.find('span.story-author > a').each((_, e) => { author += `${$(e).text()} `; }); author = author.trim(); @@ -31,7 +31,7 @@ const ProcessFeed = ($, link) => { .find( 'div.story-theme, h1.story-headline, div.story-byline, div.mpu-container-instory,script, div#story-action-placeholder, div.copyrightstatement-container, div.clearfloat, div.o-ads, h2.list-title, div.allcomments, div.logincomment, div.nologincomment' ) - .each((i, e) => { + .each((_, e) => { $(e).remove(); }); content = content.html(); @@ -39,7 +39,7 @@ const ProcessFeed = ($, link) => { return { content, author, title }; }; -const getData = async ({ site = 'www', channel }) => { +const getData = async ({ site = 'www', channel, ctx }) => { let feed; if (channel) { @@ -47,7 +47,7 @@ const getData = async ({ site = 'www', channel }) => { channel = channel.split('-').join('/'); try { - feed = await parser.parseURL(`http://${site}.ftchinese.com/rss/${channel}`); + feed = await parser.parseURL(`https://${site}.ftchinese.com/rss/${channel}`); } catch (error) { return { title: `FT 中文网 ${channel} 不存在`, @@ -55,23 +55,23 @@ const getData = async ({ site = 'www', channel }) => { }; } } else { - feed = await parser.parseURL(`http://${site}.ftchinese.com/rss/feed`); + feed = await parser.parseURL(`https://${site}.ftchinese.com/rss/feed`); } const items = await Promise.all( - feed.items.splice(0, 10).map(async (item) => { - const response = await got.get(`${item.link}?full=y&archive`); + feed.items.splice(0, 10).map((item) => { + item.link = item.link.replace('http://', 'https://'); + return ctx.cache.tryGet(item.link, async () => { + const response = await got.get(`${item.link}?full=y&archive`); - const $ = cheerio.load(response.data); - const result = ProcessFeed($, item.link); - const single = { - title: result.title, - description: result.content, - author: result.author, - pubDate: item.pubDate, - link: item.link, - }; - return Promise.resolve(single); + const $ = cheerio.load(response.data); + const result = ProcessFeed($, item.link); + + item.title = result.title; + item.description = result.content; + item.author = result.author; + return item; + }); }) );