opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-09-24 23:12:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(route): /yomiuri/news fetching error due to frame changed (#7809)

Browse Source
This commit is contained in:
Arracc
2021-07-17 02:12:45 -07:00
committed by GitHub
parent b008df1244
commit ec40eaab74
2 changed files with 87 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/traditional-media.md
+4 -4
View File
@@ -531,11 +531,11 @@ IT・科学 tech_science
<Route author="Arracc" example="/yomiuri/news" path="/yomiuri/:category" :paramsDesc="['板块']">
无料全文，综合页文章标题添加板块标签。
无料全文，综合页 (新着・速報) 文章标题补充板块标签。
| 総合 | 社会 | 政治 | 経済 | スポーツ | 国際 | 科学・ＩＴ | 選挙・世論調査 | エンタメ・文化 | 囲碁・将棋 | ライフ | 地域 | 社説 |
| ---- | -------- | -------- | ------- | -------- | ----- | ---------- | -------------- | -------------- | ---------- | ------ | ----- | --------- |
| news | national | politics | economy | sports | world | science | election | culture | igoshougi | life | local | editorial |
| 新着・速報 | 　　社会 | 政治 | 経済 | スポーツ | 国際 | 科学・ＩＴ | 選挙・世論調査 | エンタメ・文化 | 囲碁・将棋 | ライフ | 地域 | 社説 |
| ---------- | -------- | -------- | ------- | -------- | ----- | ---------- | -------------- | -------------- | ---------- | ------ | ----- | --------- |
| 　news 　 | national | politics | economy | sports | world | science | election | culture | igoshougi | life | local | editorial |
</Route>
lib/routes/yomiuri/news.js
+83 -10
View File
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const timezone = require('@/utils/timezone');
module.exports = async (ctx) => {
const category = ctx.params.category;
@@ -12,30 +13,99 @@ module.exports = async (ctx) => {
const data = response.data;
const $ = cheerio.load(data);
let category_name = '';
let categoryName;
let list = null;
if (category === 'news') {
category_name = '総合';
list = $('ul.news-top-upper-content-topics-content-list.p-list').children('li').not('.p-member-only');
categoryName = '新着・速報';
const urlPrefix = 'https://www.yomiuri.co.jp/y_ajax/';
// get list1
const param1 = $('input#latest_list_news_other_params').val();
const response1 = await got({
method: 'get',
url: urlPrefix + 'latest_list_news/' + param1 + '/?action=latest_list_news&others=' + param1,
});
const data1 = response1.data;
const firstArticleList = cheerio.load(data1);
// get list2
const param2 = $('input#latest_list_news_other_params2').val();
const response2 = await got({
method: 'get',
url: urlPrefix + 'latest_list_news2/' + param2 + '/?action=latest_list_news2&others=' + param2,
});
const data2 = response2.data;
const secondArticleList = cheerio.load(data2);
list = secondArticleList('article').add(firstArticleList('article'));
list = list.map((index, item) => {
item = $(item);
// remove matome title
const matomeTitle = item.find('div.c-matome-title');
if (matomeTitle.length !== 0) {
matomeTitle.remove();
}
// remove member only
const lockedIcon = item.find('svg[class=icon-locked]');
if (lockedIcon.length === 0) {
return item;
} else {
return null;
}
});
} else {
category_name = $('h1.p-header-category-current-title').text();
list = $('div.p-category-organization,.p-category-time-series').find('li').not('.p-member-only').not('.p-ad-list-item');
let parentSelector = 'div.p-category-organization,.p-category-time-series';
// igoshougi channel sepcial div selector
if (category === 'igoshougi') {
parentSelector = 'div.uni-news-igoshougi-top';
}
categoryName = $('h1.p-header-category-current-title,h1.p-header-category-content-title').text();
// p-header-category-content-title
list = $(parentSelector).find('li').not('.p-member-only').not('.p-ad-list-item');
list = list.map((index, item) => {
item = $(item);
// remove matome title
const matomeTitle = item.find('div.c-matome-title');
if (matomeTitle.length !== 0) {
matomeTitle.remove();
}
return item;
});
}
// format category name
if (categoryName !== '') {
categoryName = ' - ' + categoryName.trim();
}
const items = await Promise.all(
list
.map(async (index, item) => {
item = $(item);
const link = item.find('a').attr('href');
let link = '';
if (categoryName === '新着・速報') {
link = item.find('h3').children('a').attr('href');
} else {
link = item.find('a').attr('href');
}
const description = await ctx.cache.tryGet(link, async () => {
const response = await got.get(link);
const $ = cheerio.load(response.data);
return $('div.p-main-contents').html();
const mainContent = $('div.p-main-contents');
// remove articles recomended
const recommendNode = mainContent.find('aside.ev-article-manual-related__article-inline');
if (recommendNode.length !== 0) {
recommendNode.remove();
}
return mainContent.html();
});
let tag = '';
if (category_name === '総合') {
if (categoryName === '新着・速報') {
const tag_table = {
news: '総合',
news: '新着・速報',
national: '社会',
politics: '政治',
economy: '経済',
@@ -49,14 +119,17 @@ module.exports = async (ctx) => {
local: '地域',
editorial: '社説',
medical: '医療・健康',
olympic: 'オリンピック',
};
const tag_value = tag_table[link.split('/')[3]];
tag = tag_value ? tag_value : 'その他';
tag = '[' + tag + '] ';
}
return {
title: tag + item.find('a').text(),
description,
pubDate: timezone(new Date(item.find('time').prop('datetime')), +9),
link,
};
})
@@ -64,7 +137,7 @@ module.exports = async (ctx) => {
);
ctx.state.data = {
title: '読売新聞-' + category_name,
title: '読売新聞' + categoryName,
link: url,
item: items,
};