opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-10-24 12:15:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 虎嗅网文章全文提取 (#1803)

Browse Source 
逻辑复用，我爱全文。
This commit is contained in:
Henry Wang
2019-03-22 17:11:59 +08:00
committed by DIYgod
parent 7180ca6f2f
commit c464ef5967
5 changed files with 75 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/README.md
+2 -2
View File
@@ -3073,9 +3073,9 @@ board 和 build 可在[这里](http://api.ineal.me/tss/status)查看
### 虎嗅
<route name="标签" author="xyqfer" example="/huxiu/tag/291" path="/huxiu/tag/:id" :paramsDesc="['标签 id']" />
<route name="标签" author="xyqfer HenryQW" example="/huxiu/tag/291" path="/huxiu/tag/:id" :paramsDesc="['标签 id']" />
<route name="搜索" author="xyqfer" example="/huxiu/search/%E8%99%8E%E5%97%85%E6%97%A9%E6%8A%A5" path="/huxiu/search/:keyword" :paramsDesc="['关键字']" />
<route name="搜索" author="xyqfer HenryQW" example="/huxiu/search/%E8%99%8E%E5%97%85%E6%97%A9%E6%8A%A5" path="/huxiu/search/:keyword" :paramsDesc="['关键字']" />
<route name="作者" author="HenryQW" example="/huxiu/author/29318" path="/huxiu/author/:id" :paramsDesc="['用户 id']" />
lib/routes/huxiu/author.js
+2 -28
View File
@@ -1,12 +1,10 @@
const axios = require('../../utils/axios');
const cheerio = require('cheerio');
const url = require('url');
const date = require('../../utils/date');
const utils = require('./utils');
module.exports = async (ctx) => {
const { id } = ctx.params;
const link = `https://www.huxiu.com/member/${id}.html`;
const host = 'https://www.huxiu.com';
const response = await axios({
method: 'get',
url: link,
@@ -25,32 +23,8 @@ module.exports = async (ctx) => {
.get()
.map((e) => $(e).attr('href'));
const items = await Promise.all(
list.map(async (e) => {
const itemUrl = url.resolve(host, e);
const items = await utils.ProcessFeed(list, ctx.cache);
const single = await ctx.cache.tryGet(
itemUrl,
async () => {
const response = await axios.get(itemUrl);
const $ = cheerio.load(response.data);
$('.neirong-shouquan, .neirong-shouquan-public').remove();
return {
title: $('.t-h1')
.text()
.trim(),
link: itemUrl,
description: $('.article-img-box').html() + $('.article-content-wrap').html(),
pubDate: date($('.article-time').text(), 8),
author,
};
},
2 * 24 * 60 * 60
);
return Promise.resolve(single);
})
);
const authorInfo = `虎嗅网 - ${author}`;
ctx.state.data = {
title: authorInfo,
lib/routes/huxiu/search.js
+15 -23
View File
@@ -1,41 +1,33 @@
const axios = require('../../utils/axios');
const cheerio = require('cheerio');
const utils = require('./utils');
module.exports = async (ctx) => {
const { keyword } = ctx.params;
const url = `https://www.huxiu.com/search.html?s=${encodeURIComponent(keyword)}&sort=dateline:desc`;
const link = `https://www.huxiu.com/search.html?s=${encodeURIComponent(keyword)}&sort=dateline:desc`;
const response = await axios({
method: 'get',
url,
url: link,
headers: {
Referer: url,
Referer: link,
},
});
const data = response.data;
const $ = cheerio.load(data);
const list = $('.search-wrap-list-ul > li')
.map((index, elem) => {
const $elem = $(elem);
const $link = $elem.find('a').eq(0);
const title = $link.text();
const time = $elem.find('.time').text();
const link = $link.attr('href');
const description = $elem.find('.mob-summay').text();
return {
title,
link: `https://www.huxiu.com${link}`,
description,
pubDate: new Date(time).toUTCString(),
};
})
.get();
const list = $('.search-wrap-list-ul > li > h2 > a')
.get()
.map((e) => $(e).attr('href'));
const items = await utils.ProcessFeed(list, ctx.cache);
const info = `虎嗅网 - ${keyword}`;
ctx.state.data = {
title: `虎嗅-${keyword}`,
link: url,
description: $('meta[name="description"]').attr('content'),
item: list,
title: info,
link,
description: info,
item: items,
};
};
lib/routes/huxiu/tag.js
+13 -22
View File
@@ -1,40 +1,31 @@
const axios = require('../../utils/axios');
const cheerio = require('cheerio');
const utils = require('./utils');
module.exports = async (ctx) => {
const { id } = ctx.params;
const url = `https://www.huxiu.com/tags/${id}.html`;
const link = `https://www.huxiu.com/tags/${id}.html`;
const response = await axios({
method: 'get',
url,
url: link,
headers: {
Referer: url,
Referer: link,
},
});
const data = response.data;
const $ = cheerio.load(data);
const list = $('.related-article li')
.map((index, elem) => {
const $elem = $(elem);
const $link = $elem.find('a');
const title = $link.text();
const time = $elem.find('.time').text();
const link = $link.attr('href');
const list = $('.related-article li a')
.get()
.map((e) => $(e).attr('href'));
return {
title,
link: `https://www.huxiu.com${link}`,
description: title,
pubDate: new Date(time).toUTCString(),
};
})
.get();
const items = await utils.ProcessFeed(list, ctx.cache);
const info = `虎嗅 - ${$('.tag-title').text()}`;
ctx.state.data = {
title: `虎嗅-${$('.tag-title').text()}`,
link: url,
description: $('meta[name="description"]').attr('content'),
item: list,
title: info,
link,
description: info,
item: items,
};
};
lib/routes/huxiu/utils.js
+43
View File
@@ -0,0 +1,43 @@
const cheerio = require('cheerio');
const date = require('../../utils/date');
const axios = require('../../utils/axios');
const url = require('url');
const ProcessFeed = async (list, cache) => {
const host = 'https://www.huxiu.com';
const items = await Promise.all(
list.map(async (e) => {
const link = url.resolve(host, e);
const single = await cache.tryGet(
link,
async () => {
const response = await axios.get(link);
const $ = cheerio.load(response.data);
$('.neirong-shouquan, .neirong-shouquan-public').remove();
return {
title: $('.t-h1')
.text()
.trim(),
description: $('.article-img-box').html() + $('.article-content-wrap').html(),
pubDate: date($('.article-time').text(), 8),
author: $('.article-author .author-name').text(),
link,
};
},
2 * 24 * 60 * 60
);
return Promise.resolve(single);
})
);
return items;
};
module.exports = {
ProcessFeed,
};