From ba89e298bb924ac62d3c28fa5426a0808279e6b3 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 20 Jan 2023 14:30:13 -0300 Subject: [PATCH] fix(route): google news (#11662) --- docs/en/new-media.md | 2 +- docs/new-media.md | 2 +- lib/router.js | 2 +- lib/v2/google/maintainer.js | 1 + lib/v2/google/news.js | 114 +++++++++++++------------------ lib/v2/google/router.js | 1 + lib/v2/google/templates/news.art | 5 ++ 7 files changed, 58 insertions(+), 69 deletions(-) create mode 100644 lib/v2/google/templates/news.art diff --git a/docs/en/new-media.md b/docs/en/new-media.md index e2b4dea48d..c641184306 100644 --- a/docs/en/new-media.md +++ b/docs/en/new-media.md @@ -340,7 +340,7 @@ Country ### News - + ## Grub Street diff --git a/docs/new-media.md b/docs/new-media.md index ce66dc72ea..21571c2cb5 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -2260,7 +2260,7 @@ Type 栏目： ### 新闻 - + ## 观察者网 diff --git a/lib/router.js b/lib/router.js index d2f4bb6b64..369e33dc34 100644 --- a/lib/router.js +++ b/lib/router.js @@ -2455,7 +2455,7 @@ router.get('/cowlevel/element/:id', lazyloadRouteHandler('./routes/cowlevel/elem // router.get('/2048/bbs/:fid', lazyloadRouteHandler('./routes/2048/bbs')); // Google News -router.get('/google/news/:category/:locale', lazyloadRouteHandler('./routes/google/news')); +// router.get('/google/news/:category/:locale', lazyloadRouteHandler('./routes/google/news')); // 虛詞 router.get('/p-articles/section/:section', lazyloadRouteHandler('./routes/p-articles/section')); diff --git a/lib/v2/google/maintainer.js b/lib/v2/google/maintainer.js index 661875a2bd..ac1143c479 100644 --- a/lib/v2/google/maintainer.js +++ b/lib/v2/google/maintainer.js @@ -3,6 +3,7 @@ module.exports = { '/citations/:id': ['KellyHwong'], '/doodles/:language?': ['xyqfer'], '/fonts/:sort?': ['Fatpandac'], + '/news/:category/:locale': ['zoenglinghou'], '/scholar/:query': ['HenryQW'], '/sites/:id': ['hoilc'], '/sites/recentChanges/:id': ['nczitzk'], diff --git a/lib/v2/google/news.js b/lib/v2/google/news.js index 04e7068fda..a0f0525999 100644 --- a/lib/v2/google/news.js +++ b/lib/v2/google/news.js @@ -1,84 +1,66 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const url = require('url'); +const { parseDate } = require('@/utils/parse-date'); +const { art } = require('@/utils/render'); +const path = require('path'); + +const baseUrl = 'https://news.google.com'; module.exports = async (ctx) => { const category = ctx.params.category; const locale = ctx.params.locale; - const category_url = await ctx.cache.tryGet([locale, category], async () => { - const front_page = await got({ - method: 'get', - url: `https://news.google.com/?${locale}`, - }); - - const front_data = front_page.data; + const categoryUrls = await ctx.cache.tryGet(`google:news:${locale}`, async () => { + const { data: front_data } = await got(`${baseUrl}/?${locale}`); const $ = cheerio.load(front_data); - const category_list = $('a.wmzpFf.yETrXb'); - - const category_text = []; - category_list.each((index, item) => { - category_text.push($(item).text()); - }); - - return url.resolve('https://news.google.com', category_list.eq(category_text.indexOf(category)).attr('href')); + return [ + ...$('a.brSCsc') + .toArray() + .slice(3) // skip Home, For you and Following + .map((item) => { + item = $(item); + return { + category: item.text(), + url: new URL(item.attr('href'), baseUrl).href, + }; + }), + ...$('a.aqvwYd') // Home + .toArray() + .map((item) => { + item = $(item); + return { + category: item.text(), + url: new URL(item.attr('href'), baseUrl).href, + }; + }), + ]; }); + const categoryUrl = categoryUrls.find((item) => item.category === category).url; - const response = await got({ - method: 'get', - url: category_url, - }); - - const data = response.data; + const { data } = await got(categoryUrl); const $ = cheerio.load(data); - const list = $('div.xrnccd'); - let itemPicUrl; - let meta; - let description; - let link; + const list = [...$('.UwIKyb'), ...$('.IBr9hb'), ...$('.IFHyqb')]; // 3 rows of news, 3-rows-wide news, single row news + + const items = list.map((item) => { + item = $(item); + const title = item.find('h4').text(); + return { + title, + description: art(path.join(__dirname, 'templates/news.art'), { + img: item.find('img.Quavad').attr('src'), + brief: title, + }), + pubDate: parseDate(item.find('time').attr('datetime')), + author: item.find('.oovtQ').text(), + link: new URL(item.find('a.WwrzSb').first().attr('href'), baseUrl).href, + }; + }); ctx.state.data = { title: $('title').text(), - link: category_url, - item: - list && - list - .map((index, item) => { - item = $(item); - itemPicUrl = item.find('img').attr('src'); - - meta = item.find('div.SVJrMe').first(); - - description = item - .find('h4.ipQwMb') - .removeAttr('class') - .each((index, item) => { - $(item) - .children() - .removeAttr('class') - .attr('href', url.resolve('https://news.google.com', $(item).children().attr('href'))); - }) - .toString(); - - link = url.resolve('https://news.google.com', item.find('a.VDXfz').first().attr('href')); - - if (itemPicUrl !== undefined) { - description = `${description}`; - } - if (description === null) { - description = link; - } - - return { - title: item.find('h3').children().text(), - description, - pubDate: meta.find('time').first().attr('datetime'), - author: meta.find('a.wEwyrc').first().text(), - link, - }; - }) - .get(), + link: categoryUrl, + item: items, }; }; diff --git a/lib/v2/google/router.js b/lib/v2/google/router.js index 4b3d3a96be..7e9a4cfad0 100644 --- a/lib/v2/google/router.js +++ b/lib/v2/google/router.js @@ -3,6 +3,7 @@ module.exports = function (router) { router.get('/citations/:id', require('./citations')); router.get('/doodles/:language?', require('./doodles')); router.get('/fonts/:sort?', require('./fonts')); + router.get('/news/:category/:locale', require('./news')); router.get('/scholar/:query', require('./scholar')); router.get('/sites/recentChanges/:id', require('./sitesRecentChanges')); router.get('/sites/:id', require('./sites')); diff --git a/lib/v2/google/templates/news.art b/lib/v2/google/templates/news.art new file mode 100644 index 0000000000..15c7fe3815 --- /dev/null +++ b/lib/v2/google/templates/news.art @@ -0,0 +1,5 @@ +{{ if img }} + +
+{{ /if }} +{{ brief }}