From 6477e6b8e41d5b3f43a290db189c35bd7576ff76 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Sat, 19 Mar 2022 21:01:03 +0800 Subject: [PATCH] =?UTF-8?q?feat(route):=20=E8=A7=82=E5=AF=9F=E8=80=85?= =?UTF-8?q?=E7=BD=91=E8=BE=93=E5=87=BA=E5=85=A8=E6=96=87=20(#9338)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature(route): 观察者网输出全文 * fix: add radar * fix typo Co-authored-by: Tony * fix: add timezone Co-authored-by: Tony --- docs/new-media.md | 4 +- lib/router.js | 16 ++--- lib/{routes => v2}/guancha/headline.js | 19 ++--- lib/{routes => v2}/guancha/index.js | 83 ++++++++++++++-------- lib/v2/guancha/maintainer.js | 7 ++ lib/{routes => v2}/guancha/member.js | 15 ++-- lib/{routes => v2}/guancha/personalpage.js | 0 lib/v2/guancha/radar.js | 49 +++++++++++++ lib/v2/guancha/router.js | 7 ++ lib/{routes => v2}/guancha/topic.js | 27 +++---- 10 files changed, 161 insertions(+), 66 deletions(-) rename lib/{routes => v2}/guancha/headline.js (68%) rename lib/{routes => v2}/guancha/index.js (62%) create mode 100644 lib/v2/guancha/maintainer.js rename lib/{routes => v2}/guancha/member.js (93%) rename lib/{routes => v2}/guancha/personalpage.js (100%) create mode 100644 lib/v2/guancha/radar.js create mode 100644 lib/v2/guancha/router.js rename lib/{routes => v2}/guancha/topic.js (67%) diff --git a/docs/new-media.md b/docs/new-media.md index db7a39bc8a..1eba0ecb24 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -1869,7 +1869,7 @@ area 分区选项 ### 首页 - + | 全部 | 评论 & 研究 | 要闻 | 风闻 | 热点新闻 | 滚动新闻 | | --- | ------- | ----- | ------- | ------ | ------- | @@ -1889,7 +1889,7 @@ others = 热点新闻 + 滚动新闻 ### 观学院 - + | 精选 | 观书堂 | 在线课 | 观学院 | | --------- | ----- | ------- | -------- | diff --git a/lib/router.js b/lib/router.js index 97daf3bfcf..b90cf4c844 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1624,15 +1624,15 @@ router.get('/zfrontier/postlist/:type', lazyloadRouteHandler('./routes/zfrontier router.get('/zfrontier/board/:boardId', lazyloadRouteHandler('./routes/zfrontier/board_postlist')); // 观察者网 -router.get('/guancha/headline', lazyloadRouteHandler('./routes/guancha/headline')); -router.get('/guancha/topic/:id/:order?', lazyloadRouteHandler('./routes/guancha/topic')); -router.get('/guancha/member/:caty?', lazyloadRouteHandler('./routes/guancha/member')); -router.get('/guancha/personalpage/:uid', lazyloadRouteHandler('./routes/guancha/personalpage')); -router.get('/guancha/:caty?', lazyloadRouteHandler('./routes/guancha/index')); +// router.get('/guancha/headline', lazyloadRouteHandler('./routes/guancha/headline')); +// router.get('/guancha/topic/:id/:order?', lazyloadRouteHandler('./routes/guancha/topic')); +// router.get('/guancha/member/:caty?', lazyloadRouteHandler('./routes/guancha/member')); +// router.get('/guancha/personalpage/:uid', lazyloadRouteHandler('./routes/guancha/personalpage')); +// router.get('/guancha/:caty?', lazyloadRouteHandler('./routes/guancha/index')); -router.get('/guanchazhe/topic/:id/:order?', lazyloadRouteHandler('./routes/guancha/topic')); -router.get('/guanchazhe/personalpage/:uid', lazyloadRouteHandler('./routes/guancha/personalpage')); -router.get('/guanchazhe/index/:caty?', lazyloadRouteHandler('./routes/guancha/index')); +// router.get('/guanchazhe/topic/:id/:order?', lazyloadRouteHandler('./routes/guancha/topic')); +// router.get('/guanchazhe/personalpage/:uid', lazyloadRouteHandler('./routes/guancha/personalpage')); +// router.get('/guanchazhe/index/:caty?', lazyloadRouteHandler('./routes/guancha/index')); // Hpoi 手办维基 router.get('/hpoi/info/:type?', lazyloadRouteHandler('./routes/hpoi/info')); diff --git a/lib/routes/guancha/headline.js b/lib/v2/guancha/headline.js similarity index 68% rename from lib/routes/guancha/headline.js rename to lib/v2/guancha/headline.js index f1018413ea..002f7f313c 100644 --- a/lib/routes/guancha/headline.js +++ b/lib/v2/guancha/headline.js @@ -1,9 +1,12 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const timezone = require('@/utils/timezone'); +const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { const rootUrl = 'https://www.guancha.cn'; const currentUrl = `${rootUrl}/GuanChaZheTouTiao/list_1.shtml`; + const response = await got({ method: 'get', url: currentUrl, @@ -11,20 +14,20 @@ module.exports = async (ctx) => { const $ = cheerio.load(response.data); - const list = $('.headline-list li .content-headline h3 a') - .map((_, item) => { + let items = $('.headline-list li .content-headline h3 a') + .toArray() + .map((item) => { item = $(item); return { title: item.text(), - link: `${rootUrl}${item.attr('href')}`, description: item.parent().next().html(), - pubDate: new Date(item.parents('div').eq(0).find('span').text()).toUTCString(), + link: `${rootUrl}${item.attr('href').replace(/\.shtml$/, '_s.shtml')}`, + pubDate: timezone(parseDate(item.parents('div').first().find('span').text()), +8), }; - }) - .get(); + }); - const items = await Promise.all( - list.map((item) => + items = await Promise.all( + items.map((item) => ctx.cache.tryGet(item.link, async () => { const detailResponse = await got({ method: 'get', diff --git a/lib/routes/guancha/index.js b/lib/v2/guancha/index.js similarity index 62% rename from lib/routes/guancha/index.js rename to lib/v2/guancha/index.js index f9f755a83a..6286e39f4e 100644 --- a/lib/routes/guancha/index.js +++ b/lib/v2/guancha/index.js @@ -1,21 +1,42 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const date = require('@/utils/date'); +const timezone = require('@/utils/timezone'); +const { parseDate } = require('@/utils/parse-date'); +const { parseRelativeDate } = require('@/utils/parse-date'); const config = { - review: { title: '评论 & 研究', query: '.module-news-main' }, - story: { title: '要闻', query: '.img-List' }, - fengwen: { title: '风闻', query: '.fengwen-list' }, - redian: { title: '热点' }, - gundong: { title: '滚动' }, - all: { title: '全部' }, - home: { title: '首页' }, - others: { title: '热点 & 滚动' }, + review: { + title: '评论 & 研究', + query: '.module-news-main', + }, + story: { + title: '要闻', + query: '.img-List', + }, + fengwen: { + title: '风闻', + query: '.fengwen-list', + }, + redian: { + title: '热点', + }, + gundong: { + title: '滚动', + }, + all: { + title: '全部', + }, + home: { + title: '首页', + }, + others: { + title: '热点 & 滚动', + }, }; module.exports = async (ctx) => { const total = 10; - const caty = ctx.params.caty || 'all'; + const category = ctx.params.category ?? 'all'; const rootUrl = 'https://www.guancha.cn'; let newsList = [], @@ -24,7 +45,7 @@ module.exports = async (ctx) => { // 'review', 'story' and 'fengwen' come from homepage. - if (caty === 'review' || caty === 'story' || caty === 'fengwen' || caty === 'all' || caty === 'home') { + if (category === 'review' || category === 'story' || category === 'fengwen' || category === 'all' || category === 'home') { const response = await got({ method: 'get', url: rootUrl, @@ -35,54 +56,59 @@ module.exports = async (ctx) => { const fetchPost = (slice) => slice .find('h4.module-title a') + .toArray() // Filter some blank links which lead to no contents but 'https://user.guancha.cn'. - .filter((_, item) => $(item).attr('href') !== 'https://user.guancha.cn') - .map((_, item) => { + .filter((item) => $(item).attr('href') !== 'https://user.guancha.cn') + .map((item) => { item = $(item); + const link = item.attr('href'); + return { title: item.text(), - link: (link.indexOf('http') === 0 ? '' : rootUrl) + link, + link: `${link.indexOf('http') === 0 ? '' : rootUrl}${link.replace(/\.shtml/, '_s.shtml')}`, }; - }) - .get(); + }); - if (caty === 'all' || caty === 'home') { + if (category === 'all' || category === 'home') { newsList = fetchPost($(config.review.query)) .slice(0, total / 3) .concat(fetchPost($(config.story.query)).slice(0, total / 3), fetchPost($(config.fengwen.query)).slice(0, total / 3)); } else { - newsList = fetchPost($(config[caty].query)).slice(0, total); + newsList = fetchPost($(config[category].query)).slice(0, total); } } // 'redian' and 'gundong' come from api. - if (caty === 'redian' || caty === 'all' || caty === 'others') { + if (category === 'redian' || category === 'all' || category === 'others') { const response = await got({ method: 'get', url: `${rootUrl}/api/redian.htm`, }); + redianList = response.data.items .map((item) => ({ title: item.TITLE, - link: `${rootUrl}${item.HTTP_URL}`, + link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`, })) - .slice(0, caty === 'all' ? total / 3 : total); + .slice(0, category === 'all' ? total / 3 : total); } - if (caty === 'gundong' || caty === 'all' || caty === 'others') { + + if (category === 'gundong' || category === 'all' || category === 'others') { const response = await got({ method: 'get', url: `${rootUrl}/api/gundong.htm`, }); + gundongList = response.data.items .map((item) => ({ title: item.TITLE, - link: `${rootUrl}${item.HTTP_URL}`, + link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`, })) - .slice(0, caty === 'all' ? total / 3 : total); + .slice(0, category === 'all' ? total / 3 : total); } const items = await Promise.all( @@ -98,8 +124,9 @@ module.exports = async (ctx) => { // => https://user.guancha.cn/main/content?id=399176 const jumpMatch = detailResponse.data.match(/user.guancha.cn\/main\/content\?id=(.*)";/); + if (jumpMatch !== null) { - item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}`; + item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}&page=0`; detailResponse = await got({ method: 'get', url: item.link, @@ -112,9 +139,9 @@ module.exports = async (ctx) => { if (dateMatch === null) { // PubDates of posts in 'fengwen' are in an informal format. - item.pubDate = date(content('.time1').text(), 8); + item.pubDate = parseRelativeDate(content('.time1').text()); } else { - item.pubDate = new Date(dateMatch[1]).toUTCString(); + item.pubDate = timezone(parseDate(dateMatch[1]), +8); } item.description = content('.all-txt').html() || content('.article-txt-content').html(); @@ -125,7 +152,7 @@ module.exports = async (ctx) => { ); ctx.state.data = { - title: `观察者网 - ${config[caty].title}`, + title: `观察者网 - ${config[category].title}`, link: rootUrl, item: items, }; diff --git a/lib/v2/guancha/maintainer.js b/lib/v2/guancha/maintainer.js new file mode 100644 index 0000000000..a87c733dc9 --- /dev/null +++ b/lib/v2/guancha/maintainer.js @@ -0,0 +1,7 @@ +module.exports = { + '/:category?': ['nczitzk', 'Jeason0228'], + '/headline': ['nczitzk'], + '/member/:category?': ['nczitzk'], + '/personalpage/:uid': ['Jeason0228'], + '/topic/:id/:order?': ['occupy5', 'nczitzk'], +}; diff --git a/lib/routes/guancha/member.js b/lib/v2/guancha/member.js similarity index 93% rename from lib/routes/guancha/member.js rename to lib/v2/guancha/member.js index 716f47967d..d0bd48509f 100644 --- a/lib/routes/guancha/member.js +++ b/lib/v2/guancha/member.js @@ -1,8 +1,8 @@ const got = require('@/utils/got'); -const { parseDate } = require('@/utils/parse-date'); const timezone = require('@/utils/timezone'); +const { parseDate } = require('@/utils/parse-date'); -const title = { +const titles = { recommend: '精选', books: '观书堂', courses: '在线课', @@ -10,10 +10,11 @@ const title = { }; module.exports = async (ctx) => { - const caty = ctx.params.caty || 'recommend'; + const category = ctx.params.category ?? 'recommend'; const rootUrl = 'https://member.guancha.cn'; const apiUrl = `${rootUrl}/zaixianke/home`; + const response = await got({ method: 'get', url: apiUrl, @@ -21,11 +22,11 @@ module.exports = async (ctx) => { let items; - switch (caty) { + switch (category) { case 'books': items = response.data.data.books.map((item) => ({ title: item.title, - link: `${rootUrl}/guanshutang/summary.html?id=${item.id}`, + link: `${rootUrl}/guanshutang/summary.html?id=${item.id}&page=0`, description: `

[${item.audio_time}] ${item.desc_short}

`, pubDate: new Date(parseInt(item.cover.split('/').pop().substr(0, 10)) * 1000).toUTCString(), })); @@ -53,7 +54,7 @@ module.exports = async (ctx) => { break; default: - items = response.data.data[caty].map((item) => { + items = response.data.data[category].map((item) => { let timeArray = item.media_time && item.media_time.trim().split(/\D+/, 3); timeArray = timeArray && timeArray.filter((item) => item !== ''); let itunes_duration; @@ -78,7 +79,7 @@ module.exports = async (ctx) => { } ctx.state.data = { - title: `观学院 - ${title[caty]}`, + title: `观学院 - ${titles[category]}`, link: `${rootUrl}/index.html`, item: items, }; diff --git a/lib/routes/guancha/personalpage.js b/lib/v2/guancha/personalpage.js similarity index 100% rename from lib/routes/guancha/personalpage.js rename to lib/v2/guancha/personalpage.js diff --git a/lib/v2/guancha/radar.js b/lib/v2/guancha/radar.js new file mode 100644 index 0000000000..daebd69f18 --- /dev/null +++ b/lib/v2/guancha/radar.js @@ -0,0 +1,49 @@ +module.exports = { + 'guancha.cn': { + _name: '观察者网', + '.': [ + { + title: '头条', + docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-tou-tiao', + source: ['/GuanChaZheTouTiao', '/'], + target: '/guancha/headline', + }, + { + title: '首页', + docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-shou-ye', + source: ['/'], + target: '/guancha/:category?', + }, + { + title: '观学院', + docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-guan-xue-yuan', + source: ['/'], + target: '/guancha/:category?', + }, + ], + app: [ + { + title: '个人主页文章', + docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-ge-ren-zhu-ye-wen-zhang', + source: ['/user/get-published-list', '/'], + target: (params, url) => `/guancha/personalpage/${new URL(url).searchParams.get('uid')}`, + }, + ], + member: [ + { + title: '观学院', + docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-guan-xue-yuan', + source: ['/'], + target: '/guancha/member/recommend', + }, + ], + user: [ + { + title: '风闻话题', + docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-feng-wen-hua-ti', + source: ['/topic/post-list', '/'], + target: (params, url) => `/guancha/topic/${new URL(url).searchParams.get('topic_id')}/${new URL(url).searchParams.get('order')}`, + }, + ], + }, +}; diff --git a/lib/v2/guancha/router.js b/lib/v2/guancha/router.js new file mode 100644 index 0000000000..23888d2cf0 --- /dev/null +++ b/lib/v2/guancha/router.js @@ -0,0 +1,7 @@ +module.exports = function (router) { + router.get('/headline', require('./headline')); + router.get('/member/:category?', require('./member')); + router.get('/personalpage/:uid', require('./personalpage')); + router.get('/topic/:id/:order?', require('./topic')); + router.get('/:category?', require('./index')); +}; diff --git a/lib/routes/guancha/topic.js b/lib/v2/guancha/topic.js similarity index 67% rename from lib/routes/guancha/topic.js rename to lib/v2/guancha/topic.js index 1ace1c5236..c1abe7c374 100644 --- a/lib/routes/guancha/topic.js +++ b/lib/v2/guancha/topic.js @@ -1,10 +1,10 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const date = require('@/utils/date'); +const { parseRelativeDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { - const id = ctx.params.id || '0'; - const order = ctx.params.order || '1'; + const id = ctx.params.id ?? '0'; + const order = ctx.params.order ?? '1'; const rootUrl = 'https://user.guancha.cn'; const currentUrl = `${rootUrl}/${id === '0' ? 'main/index-list.json?' : 'topic/post-list?topic_id=' + id}&page=1&order=${order}`; @@ -16,28 +16,29 @@ module.exports = async (ctx) => { const $ = cheerio.load(response.data); - const list = $('.list-item h4 a, ul.home li .list-item h4 a') - .slice(0, 10) - .map((_, item) => { + let items = $('.list-item h4 a, ul.home li .list-item h4 a') + .toArray() + .map((item) => { item = $(item); + return { title: item.text(), - link: `${rootUrl}${item.attr('href')}`, + link: `${rootUrl}${item.attr('href')}&page=0`, }; - }) - .get(); + }); - const items = await Promise.all( - list.map((item) => + items = await Promise.all( + items.map((item) => ctx.cache.tryGet(item.link, async () => { const detailResponse = await got({ method: 'get', url: item.link, }); + const content = cheerio.load(detailResponse.data); - item.pubDate = date(content('.time1').text(), 8); - item.author = content('.user-main h4 a').eq(0).text(); + item.pubDate = parseRelativeDate(content('.time1').text()); + item.author = content('.user-main h4 a').first().text(); item.description = content('.article-txt-content').html(); return item;