opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-09-01 14:57:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: avoid promise overhead (#8028)

Browse Source
This commit is contained in:
Sukka
2021-08-17 02:45:53 +08:00
committed by GitHub
parent 8053b33b03
commit 6e3b58ed1d
543 changed files with 5295 additions and 5694 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.eslintrc
+3 -1
View File
@@ -54,6 +54,8 @@
"no-await-in-loop": 2,
"require-atomic-updates": 0,
"no-prototype-builtins": 0,
"no-new-func": 2
"no-new-func": 2,
"require-await": 2,
"no-return-await": 2
}
}
lib/middleware/access-control.js
+4 -4
View File
@@ -29,24 +29,24 @@ module.exports = async (ctx, next) => {
await next();
} else {
if (!isControlled || allowLocalhost) {
return await grant();
return grant();
}
if (config.accessKey) {
if (config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey)) {
return await grant();
return grant();
}
}
if (config.whitelist) {
if (config.whitelist.find((white) => ip.includes(white) || requestPath.includes(white) || requestUA.includes(white))) {
return await grant();
return grant();
}
}
if (config.blacklist) {
if (!config.blacklist.find((black) => ip.includes(black) || requestPath.includes(black) || requestUA.includes(black))) {
return await grant();
return grant();
}
}
lib/middleware/cache.js
+1 -1
View File
@@ -145,7 +145,7 @@ module.exports = function (app) {
const key = 'koa-redis-cache:' + md5(ctx.request.path);
if (!available) {
return await next();
return next();
}
try {
lib/middleware/parameter.js
+1 -1
View File
@@ -30,7 +30,7 @@ module.exports = async (ctx, next) => {
// sort items
ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
const handleItem = async (item) => {
const handleItem = (item) => {
item.title && (item.title = entities.decodeXML(item.title + ''));
// handle pubDate
lib/router.js
+1 -1
View File
@@ -35,7 +35,7 @@ const lazyloadRouteHandler = (routeHandlerPath) => (ctx) => {
// index
router.get('/', lazyloadRouteHandler('./routes/index'));
router.get('/robots.txt', async (ctx) => {
router.get('/robots.txt', (ctx) => {
if (config.disallowRobot) {
ctx.set('Content-Type', 'text/plain');
ctx.body = 'User-agent: *\nDisallow: /';
lib/routes/199it/index.js
+15 -17
View File
@@ -8,23 +8,21 @@ module.exports = async (ctx) => {
const $ = cheerio.load(res.data);
const list = $('.entry-content').get();
const out = await Promise.all(
list.slice(0, 9).map(async (item) => {
const $ = cheerio.load(item);
const time = $('.post-time time').attr('datetime');
const title = $('.entry-title a').attr('title');
const partial = $('.entry-title a').attr('href');
const address = partial;
const single = {
title,
pubDate: new Date(time).toUTCString(),
link: address,
guid: address,
};
ctx.cache.set(address, JSON.stringify(single));
return Promise.resolve(single);
})
);
const out = list.slice(0, 9).map((item) => {
const $ = cheerio.load(item);
const time = $('.post-time time').attr('datetime');
const title = $('.entry-title a').attr('title');
const partial = $('.entry-title a').attr('href');
const address = partial;
const single = {
title,
pubDate: new Date(time).toUTCString(),
link: address,
guid: address,
};
ctx.cache.set(address, JSON.stringify(single));
return single;
});
ctx.state.data = {
title: '199it',
link: url,
lib/routes/199it/utils.js
+10 -11
View File
@@ -21,19 +21,18 @@ module.exports = async (ctx, keyword, currentUrl) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('div.entry-content').html();
item.description = content('div.entry-content').html();
return item;
})
return item;
})
)
);
lib/routes/1point3acres/blog.js
+12 -13
View File
@@ -24,22 +24,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('h2, .meta_tags, .pagination, .subinfopost, #comments').remove();
content('h2, .meta_tags, .pagination, .subinfopost, #comments').remove();
item.description = content('.post').html();
item.pubDate = new Date(detailResponse.data.match(/"datePublished":"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00)","dateModified":/)[1]).toUTCString();
item.description = content('.post').html();
item.pubDate = new Date(detailResponse.data.match(/"datePublished":"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00)","dateModified":/)[1]).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/36kr/motif.js
+8 -9
View File
@@ -20,15 +20,14 @@ module.exports = async (ctx) => {
title: `36氪专题 - ${motifInfo.categoryTitle}`,
link: motifUrl,
item: await Promise.all(
articleList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
articleList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
)
),
description: `${motifInfo.categoryTitle}`,
lib/routes/36kr/news.js
+7 -8
View File
@@ -81,14 +81,13 @@ module.exports = async (ctx) => {
title: `36氪资讯 - ${cfg.title}`,
link: newsUrl,
item: await Promise.all(
informationList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(contentResponse.data);
item.description = content('div.common-width.content.articleDetailContent.kr-rich-text-wrapper').html();
return item;
})
informationList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(contentResponse.data);
item.description = content('div.common-width.content.articleDetailContent.kr-rich-text-wrapper').html();
return item;
})
)
),
};
lib/routes/36kr/search/article.js
+1 -1
View File
@@ -56,7 +56,7 @@ module.exports = async (ctx) => {
pubDate: new Date(item.publishTime).toUTCString(),
};
const other = await ctx.cache.tryGet(link, async () => await load(link));
const other = await ctx.cache.tryGet(link, () => load(link));
return Promise.resolve(Object.assign({}, single, other));
})
);
lib/routes/36kr/user.js
+8 -9
View File
@@ -20,15 +20,14 @@ module.exports = async (ctx) => {
title: `36氪用户 - ${authorInfo.userNick}`,
link: userUrl,
item: await Promise.all(
articleList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
articleList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
)
),
description: `${authorInfo.label} | ${authorInfo.summary}`,
lib/routes/3k8/latest.js
+13 -14
View File
@@ -23,23 +23,22 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
responseType: 'buffer',
});
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
responseType: 'buffer',
});
const content = cheerio.load(iconv.decode(detailResponse.data, 'gb2312'));
const content = cheerio.load(iconv.decode(detailResponse.data, 'gb2312'));
item.title = content('h1.title').text();
item.pubDate = new Date(content('div.news-mes span').eq(0).text().replace('发布时间：', '').trim() + ' GMT+8').toUTCString();
item.description = content('div.news-content').html();
item.title = content('h1.title').text();
item.pubDate = new Date(content('div.news-mes span').eq(0).text().replace('发布时间：', '').trim() + ' GMT+8').toUTCString();
item.description = content('div.news-content').html();
return item;
})
return item;
})
)
);
lib/routes/3ycy/home.js
+2 -2
View File
@@ -18,12 +18,12 @@ module.exports = async (ctx) => {
}
const items = await Promise.all(
count.map(async (i) => {
count.map((i) => {
const item = $(list[i]);
const title = item.find('h3 a').first();
const link = title.attr('href');
return await ctx.cache.tryGet('3ycy' + link, async () => {
return ctx.cache.tryGet('3ycy' + link, async () => {
const date = item.find('div.postSubtitle').text().trim();
const match = /(\d+\/\d+\/\d{4}\s+\d+:\d+:\d+\s*(?:AM|PM))/i.exec(date);
const pubDate = match && new Date(match[1] + ' GMT+8').toUTCString();
lib/routes/6park/index.js
+15 -16
View File
@@ -29,24 +29,23 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('title').text().replace(' -6park.com', '');
item.author = detailResponse.data.match(/送交者: .*>(.*)<.*\[/)[1];
item.pubDate = new Date(detailResponse.data.match(/于 (.*) 已读/)[1]).toUTCString();
item.description = content('pre')
.html()
.replace(/<font color="#E6E6DD">6park.com<\/font>/g, '');
item.title = content('title').text().replace(' -6park.com', '');
item.author = detailResponse.data.match(/送交者: .*>(.*)<.*\[/)[1];
item.pubDate = new Date(detailResponse.data.match(/于 (.*) 已读/)[1]).toUTCString();
item.description = content('pre')
.html()
.replace(/<font color="#E6E6DD">6park.com<\/font>/g, '');
return item;
})
return item;
})
)
);
lib/routes/8kcos/cat.js
+1 -1
View File
@@ -15,7 +15,7 @@ module.exports = async (ctx) => {
Cheerio.load(resp.body)('#wrap > div > div > div > ul > li.item')
.map((_, e) => {
const { href } = Cheerio.load(e)('h2 > a')[0].attribs;
return ctx.cache.tryGet(href, async () => loadArticle(href));
return ctx.cache.tryGet(href, () => loadArticle(href));
})
.toArray()
),
lib/routes/8kcos/latest.js
+1 -1
View File
@@ -14,7 +14,7 @@ module.exports = async (ctx) => {
Cheerio.load(response.body)('#wrap > div > div > div > ul > li.item')
.map((_, e) => {
const { href } = Cheerio.load(e)('div > h2 > a')[0].attribs;
return ctx.cache.tryGet(href, async () => loadArticle(href));
return ctx.cache.tryGet(href, () => loadArticle(href));
})
.toArray()
)
lib/routes/91ddcc/stage.js
+10 -11
View File
@@ -24,19 +24,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('div.article-con').html();
item.description = content('div.article-con').html();
return item;
})
return item;
})
)
);
lib/routes/93/index.js
+11 -12
View File
@@ -26,20 +26,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.pageContent').html();
item.author = content('.pageTitle ul li').eq(1).text().replace('来源：', '');
item.description = content('.pageContent').html();
item.author = content('.pageTitle ul li').eq(1).text().replace('来源：', '');
return item;
})
return item;
})
)
);
lib/routes/95mm/utils.js
+13 -14
View File
@@ -24,23 +24,22 @@ module.exports = async (ctx, title, rootUrl) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const images = detailResponse.data.match(/src": '(.*?)',"width/g);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const images = detailResponse.data.match(/src": '(.*?)',"width/g);
item.description = '';
item.description = '';
for (const i of images) {
item.description += `<img src="${i.split("'")[1]}">`;
}
for (const i of images) {
item.description += `<img src="${i.split("'")[1]}">`;
}
return item;
})
return item;
})
)
);
lib/routes/abc/index.js
+39 -40
View File
@@ -27,54 +27,53 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('.modal-content').remove();
content('.modal-content').remove();
content('img').each(function () {
content(this).attr('src', content(this).attr('data-src'));
});
content('img').each(function () {
content(this).attr('src', content(this).attr('data-src'));
});
let coverImage, contentBody;
if (content('div[data-component="FeatureMedia"]').html() !== null) {
coverImage = content('div[data-component="FeatureMedia"]').html();
} else {
coverImage = content('.view-hero-media').html();
}
if (content('#body').html() !== null) {
contentBody = content('#body').html();
} else {
contentBody = content('.article-text').html();
}
let coverImage, contentBody;
if (content('div[data-component="FeatureMedia"]').html() !== null) {
coverImage = content('div[data-component="FeatureMedia"]').html();
} else {
coverImage = content('.view-hero-media').html();
}
if (content('#body').html() !== null) {
contentBody = content('#body').html();
} else {
contentBody = content('.article-text').html();
}
const authorsArray = [];
const authorsMatch = detailResponse.data.match(/author":(.*),"dateModified/);
if (authorsMatch === null) {
authorsArray.push(content('meta[name="DCTERMS.contributor"]').attr('content'));
} else {
const authors = JSON.parse(authorsMatch[1]);
try {
for (const author of authors) {
authorsArray.push(author.name);
}
} catch (e) {
authorsArray.push(authors.name);
const authorsArray = [];
const authorsMatch = detailResponse.data.match(/author":(.*),"dateModified/);
if (authorsMatch === null) {
authorsArray.push(content('meta[name="DCTERMS.contributor"]').attr('content'));
} else {
const authors = JSON.parse(authorsMatch[1]);
try {
for (const author of authors) {
authorsArray.push(author.name);
}
} catch (e) {
authorsArray.push(authors.name);
}
}
item.description = coverImage + contentBody;
item.pubDate = new Date(content('meta[property="article:published_time"]').attr('content')).toUTCString();
item.author = authorsArray.join(', ');
item.description = coverImage + contentBody;
item.pubDate = new Date(content('meta[property="article:published_time"]').attr('content')).toUTCString();
item.author = authorsArray.join(', ');
return item;
})
return item;
})
)
);
lib/routes/adquan/index.js
+8 -9
View File
@@ -58,15 +58,14 @@ module.exports = async (ctx) => {
title: `广告网 - ${cfg.title}`,
link: cfg.link,
item: await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
item.pubDate = new Date(content('p.text_time2 span').eq(0).text()).toUTCString();
item.description = content('div.con_Text').html();
return item;
})
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
item.pubDate = new Date(content('p.text_time2 span').eq(0).text()).toUTCString();
item.description = content('div.con_Text').html();
return item;
})
)
),
};
lib/routes/aflcio/blog.js
+11 -12
View File
@@ -24,20 +24,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.section-article-body').html();
item.author = detailResponse.data.match(/entityName":"(.*)","entityType"/)[1];
item.description = content('.section-article-body').html();
item.author = detailResponse.data.match(/entityName":"(.*)","entityType"/)[1];
return item;
})
return item;
})
)
);
lib/routes/agefans/update.js
+10 -11
View File
@@ -23,19 +23,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.div_left').html();
item.description = content('.div_left').html();
return item;
})
return item;
})
)
);
lib/routes/aisixiang/column.js
+2 -2
View File
@@ -22,9 +22,9 @@ module.exports = async (ctx) => {
const columnName = $('.search_list > ul > li:nth-child(1) > a:nth-child(1)').text();
const items = await Promise.all(
list.map(async (e) => {
list.map((e) => {
const link = $(e).attr('href');
return await ctx.cache.tryGet(link, async () => await util.ProcessFeed(link));
return ctx.cache.tryGet(link, () => util.ProcessFeed(link));
})
);
lib/routes/aisixiang/ranking.js
+2 -2
View File
@@ -22,9 +22,9 @@ module.exports = async (ctx) => {
const columnName = $('.tops_text > h3')[0].firstChild.nodeValue;
const items = await Promise.all(
list.map(async (e) => {
list.map((e) => {
const link = $(e).attr('href');
return await ctx.cache.tryGet(link, async () => await util.ProcessFeed(link));
return ctx.cache.tryGet(link, () => util.ProcessFeed(link));
})
);
lib/routes/aisixiang/thinktank.js
+2 -2
View File
@@ -25,9 +25,9 @@ module.exports = async (ctx) => {
const list = tds.parent().next().find('a').slice(0, 5).get();
const items = await Promise.all(
list.map(async (e) => {
list.map((e) => {
const link = $(e).attr('href');
return await ctx.cache.tryGet(link, async () => await util.ProcessFeed(link));
return ctx.cache.tryGet(link, () => util.ProcessFeed(link));
})
);
lib/routes/algocasts/all.js
+12 -14
View File
@@ -8,30 +8,28 @@ const getCacheParsed = async (ctx, key) => {
}
};
const generateResponse = async (items) => ({
const generateResponse = (items) => ({
// 源标题
title: 'AlgoCasts',
// 源链接
link: `https://www.algocasts.com`,
// 源说明
description: `AlgoCasts 旨在用心做好每一个算法讲解视频。每个视频包含两个部分：题目的剖析讲解以及编码，力求在讲解清楚到位的基础上，尽可能地保持视频精简短小，让大家可以在碎片时间里进行学习，并收获这些算法题背后的思想与乐趣。`,
item: await Promise.all(
items.map(async (item) => ({
// 文章标题
title: `${item.title} | AlgoCasts 视频更新`,
// 文章正文
description: item.description || '',
// 文章链接
link: item.link,
}))
),
item: items.map((item) => ({
// 文章标题
title: `${item.title} | AlgoCasts 视频更新`,
// 文章正文
description: item.description || '',
// 文章链接
link: item.link,
})),
});
const makeFull = async (ctx, infos) => {
const makeFull = (ctx, infos) => {
const limit = 10; // 仅获取最近十条信息（避免无意义请求）
infos = infos.slice(0, limit);
return await Promise.all(
return Promise.all(
infos.map(async (item) => {
const cacheKey = `episode-${item.episode}`;
@@ -91,5 +89,5 @@ module.exports = async (ctx) => {
infos.push({ id, title, episode, link });
});
ctx.state.data = await generateResponse(await makeFull(ctx, infos));
ctx.state.data = generateResponse(await makeFull(ctx, infos));
};
lib/routes/aliyun/database_month.js
+1 -1
View File
@@ -33,7 +33,7 @@ module.exports = async (ctx) => {
item.description = itemElement('.content').html();
ctx.cache.set(link, JSON.stringify(item));
return Promise.resolve(item);
return item;
})
);
lib/routes/aliyun/notice.js
+1 -1
View File
@@ -52,7 +52,7 @@ module.exports = async (ctx) => {
item.description = itemElement('#se-knowledge').html();
ctx.cache.set(link, JSON.stringify(item));
return Promise.resolve(item);
return item;
})
);
lib/routes/aljazeera/news.js
+8 -9
View File
@@ -24,15 +24,14 @@ module.exports = async (ctx) => {
title: `Aljazeera半岛网 - 新闻`,
link: link,
item: await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
item.pubDate = new Date(detailResponse.data.match(/"datePublished": "(.*?)",/)[1]).toUTCString();
item.description = content('div.wysiwyg').html();
return item;
})
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
item.pubDate = new Date(detailResponse.data.match(/"datePublished": "(.*?)",/)[1]).toUTCString();
item.description = content('div.wysiwyg').html();
return item;
})
)
),
};
lib/routes/allnow/column.js
+13 -14
View File
@@ -21,22 +21,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
return item;
})
return item;
})
)
);
lib/routes/allnow/index.js
+13 -14
View File
@@ -20,22 +20,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
return item;
})
return item;
})
)
);
lib/routes/allnow/tag.js
+13 -14
View File
@@ -21,22 +21,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
return item;
})
return item;
})
)
);
lib/routes/allnow/user.js
+13 -14
View File
@@ -21,22 +21,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
item.title = content('p.title').text();
item.author = content('a.single-name').eq(0).text();
item.description = content('#article-content').html();
item.pubDate = new Date(detailResponse.data.match(/time:"(.*?)"\}/)[1] + ' GMT+8').toUTCString();
return item;
})
return item;
})
)
);
lib/routes/alter-cn/news.js
+7 -8
View File
@@ -26,14 +26,13 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet('alter-cn' + item.link, async () => {
const res = await got({ method: 'get', url: url.resolve(root_url, item.link) });
const content = cheerio.load(res.data);
item.description = content(content('div.con table.table3')[1]).html();
return item;
})
list.map((item) =>
ctx.cache.tryGet('alter-cn' + item.link, async () => {
const res = await got({ method: 'get', url: url.resolve(root_url, item.link) });
const content = cheerio.load(res.data);
item.description = content(content('div.con table.table3')[1]).html();
return item;
})
)
);
lib/routes/amazfitwatchfaces/utils.js
+11 -12
View File
@@ -22,19 +22,18 @@ module.exports = async (ctx, currentUrl) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
const date = content('i.fa-calendar').parent().find('span').text();
const ymd = date.split(' ')[0];
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
const date = content('i.fa-calendar').parent().find('span').text();
const ymd = date.split(' ')[0];
item.title = `[${content('code').text()}] ${content('title').text().split(' - ')[0]}`;
item.description = `<img src="${content('#watchface-preview').attr('src')}"><br>${content('div.unicodebidi').text()}`;
item.pubDate = new Date(`${ymd.split('.')[2]}-${ymd.split('.')[1]}-${ymd.split('.')[0]} ${date.split(' ')[1]}`).toUTCString();
return item;
})
item.title = `[${content('code').text()}] ${content('title').text().split(' - ')[0]}`;
item.description = `<img src="${content('#watchface-preview').attr('src')}"><br>${content('div.unicodebidi').text()}`;
item.pubDate = new Date(`${ymd.split('.')[2]}-${ymd.split('.')[1]}-${ymd.split('.')[0]} ${date.split(' ')[1]}`).toUTCString();
return item;
})
)
);
lib/routes/amazon/ku.js
+11 -12
View File
@@ -33,20 +33,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.author = content('span.author').text();
item.description = content('#ebooks-img-canvas').html() + content('#detailBullets_feature_div').html();
item.author = content('span.author').text();
item.description = content('#ebooks-img-canvas').html() + content('#detailBullets_feature_div').html();
return item;
})
return item;
})
)
);
lib/routes/aom/journal.js
+15 -16
View File
@@ -78,25 +78,24 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
headers,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
headers,
});
const content = cheerio.load(detailResponse.data);
content('.author-info').remove();
content('.author-info').remove();
item.author = content('.loa-accordion').text();
item.doi = content('meta[name="dc.Identifier"]').attr('content');
item.description = content('.abstractInFull').html();
item.pubDate = new Date(content('meta[name="dc.Date"]').attr('content')).toUTCString();
item.author = content('.loa-accordion').text();
item.doi = content('meta[name="dc.Identifier"]').attr('content');
item.description = content('.abstractInFull').html();
item.pubDate = new Date(content('meta[name="dc.Date"]').attr('content')).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/asahi/area.js
+13 -14
View File
@@ -29,23 +29,22 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('._30SFw, .-Oj2D, .notPrint').remove();
content('._30SFw, .-Oj2D, .notPrint').remove();
item.description = content('._3YqJ1').html();
item.title = content('meta[name="TITLE"]').attr('content');
item.pubDate = Date.parse(content('meta[name="pubdate"]').attr('content'));
item.description = content('._3YqJ1').html();
item.title = content('meta[name="TITLE"]').attr('content');
item.pubDate = Date.parse(content('meta[name="pubdate"]').attr('content'));
return item;
})
return item;
})
)
);
lib/routes/asahi/index.js
+13 -14
View File
@@ -53,23 +53,22 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('._30SFw, .-Oj2D, .notPrint').remove();
content('._30SFw, .-Oj2D, .notPrint').remove();
item.description = content('._3YqJ1').html();
item.title = content('meta[name="TITLE"]').attr('content');
item.pubDate = Date.parse(content('meta[name="pubdate"]').attr('content'));
item.description = content('._3YqJ1').html();
item.title = content('meta[name="TITLE"]').attr('content');
item.pubDate = Date.parse(content('meta[name="pubdate"]').attr('content'));
return item;
})
return item;
})
)
);
lib/routes/asml/press-releases.js
+12 -13
View File
@@ -14,21 +14,20 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('title').text().replace('| ASML', '');
item.description = content('.longcopy').html();
item.pubDate = new Date(detailResponse.data.match(/"datePublished": "(.*)",/)[1]).toUTCString();
item.title = content('title').text().replace('| ASML', '');
item.description = content('.longcopy').html();
item.pubDate = new Date(detailResponse.data.match(/"datePublished": "(.*)",/)[1]).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/babehub/utils.js
+26 -27
View File
@@ -28,38 +28,37 @@ const fetch = async (cache, currentUrl) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await cache.tryGet(item.link, async () => {
try {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
cache.tryGet(item.link, async () => {
try {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const video = content('video');
const video = content('video');
item.description = '';
item.description = '';
if (video.length > 0) {
const poster = detailResponse.data.match(/posterImage: "(.*)",/);
item.itunes_item_image = (poster ? poster[1] : video.attr('poster')) || item.itunes_item_image;
if (video.length > 0) {
const poster = detailResponse.data.match(/posterImage: "(.*)",/);
item.itunes_item_image = (poster ? poster[1] : video.attr('poster')) || item.itunes_item_image;
item.enclosure_type = 'video/mp4';
item.enclosure_url = video.children('source').attr('src');
item.description = `<video controls loop preload="auto"><source src=${item.enclosure_url} type="video/mp4"/></video><img src="${item.itunes_item_image}">`;
}
content('.gallery-e li a').each(function () {
item.description += `<img src="${content(this).attr('href')}">`;
});
return item;
} catch (e) {
return Promise.resolve('');
item.enclosure_type = 'video/mp4';
item.enclosure_url = video.children('source').attr('src');
item.description = `<video controls loop preload="auto"><source src=${item.enclosure_url} type="video/mp4"/></video><img src="${item.itunes_item_image}">`;
}
})
content('.gallery-e li a').each(function () {
item.description += `<img src="${content(this).attr('href')}">`;
});
return item;
} catch (e) {
return Promise.resolve('');
}
})
)
);
lib/routes/baidu/daily.js
+1 -1
View File
@@ -42,7 +42,7 @@ module.exports = async (ctx) => {
item.description = itemElement('.detail-wp').html();
ctx.cache.set(link, JSON.stringify(item));
return Promise.resolve(item);
return item;
})
);
lib/routes/bandcamp/tag.js
+12 -13
View File
@@ -21,21 +21,20 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('.trackTitle').eq(0).text();
item.author = content('h3 span a').text();
item.description = content('#tralbumArt').html() + content('#trackInfo').html();
item.title = content('.trackTitle').eq(0).text();
item.author = content('h3 span a').text();
item.description = content('#tralbumArt').html() + content('#trackInfo').html();
return item;
})
return item;
})
)
);
lib/routes/bangumi/user/blog.js
+7 -8
View File
@@ -23,15 +23,14 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const res = await got({ method: 'get', url: item.link });
const content = cheerio.load(res.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const res = await got({ method: 'get', url: item.link });
const content = cheerio.load(res.data);
item.description = content('#entry_content').html();
return item;
})
item.description = content('#entry_content').html();
return item;
})
)
);
lib/routes/banyuetan/byt.js
+12 -13
View File
@@ -25,21 +25,20 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('#detail_content').html();
item.author = content('.detail_tit_source').text().replace('来源：', '');
item.pubDate = new Date(content('meta[property="og:release_date"]').attr('content')).toUTCString();
item.description = content('#detail_content').html();
item.author = content('.detail_tit_source').text().replace('来源：', '');
item.pubDate = new Date(content('meta[property="og:release_date"]').attr('content')).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/bbc/index.js
+27 -28
View File
@@ -47,39 +47,38 @@ module.exports = async (ctx) => {
}
const items = await Promise.all(
feed.items.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const response = await got({
method: 'get',
url: item.link,
});
feed.items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got({
method: 'get',
url: item.link,
});
const $ = cheerio.load(response.data);
const $ = cheerio.load(response.data);
let description;
let description;
if (response.request.options.url.pathname.startsWith('/news/av')) {
description = item.content;
} else {
description = utils.ProcessFeed($);
}
if (response.request.options.url.pathname.startsWith('/news/av')) {
description = item.content;
} else {
description = utils.ProcessFeed($);
}
let section = 'sport';
const urlSplit = item.link.split('/');
const sectionSplit = urlSplit[urlSplit.length - 1].split('-');
if (sectionSplit.length > 1) {
section = sectionSplit[0];
}
section = section[0].toUpperCase() + section.slice(1);
let section = 'sport';
const urlSplit = item.link.split('/');
const sectionSplit = urlSplit[urlSplit.length - 1].split('-');
if (sectionSplit.length > 1) {
section = sectionSplit[0];
}
section = section[0].toUpperCase() + section.slice(1);
return {
title: `[${section}] ${item.title}`,
description,
pubDate: item.pubDate,
link: item.link,
};
})
return {
title: `[${section}] ${item.title}`,
description,
pubDate: item.pubDate,
link: item.link,
};
})
)
);
lib/routes/bell-labs/events-news.js
+12 -13
View File
@@ -43,22 +43,21 @@ module.exports = async (ctx) => {
if (category === 'press-releases') {
items = await Promise.all(
items.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('.social-media-sharing').remove();
content('.social-media-sharing').remove();
item.description = content('.layout-content').html();
item.pubDate = new Date(content('meta[name="search-updated"]').attr('content')).toUTCString();
item.description = content('.layout-content').html();
item.pubDate = new Date(content('meta[name="search-updated"]').attr('content')).toUTCString();
return item;
})
return item;
})
)
);
}
lib/routes/bendibao/news.js
+16 -17
View File
@@ -52,28 +52,27 @@ module.exports = async (ctx) => {
}
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
try {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
try {
const detailResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(detailResponse.data);
// Some links lead to mobile-view pages.
// eg. http://m.bj.bendibao.com/news/273517.html
// Divs for contents are different from which in desktop-view pages.
// Some links lead to mobile-view pages.
// eg. http://m.bj.bendibao.com/news/273517.html
// Divs for contents are different from which in desktop-view pages.
item.description = content('div.content').html() || content('div.content-box').html();
item.description = content('div.content').html() || content('div.content-box').html();
// Spans for publish dates are the same cases as above.
// Spans for publish dates are the same cases as above.
item.pubDate = new Date((content('span.time').text().replace(/发布时间：/, '') || content('span.public_time').text()) + ' GMT+8').toUTCString();
item.pubDate = new Date((content('span.time').text().replace(/发布时间：/, '') || content('span.public_time').text()) + ' GMT+8').toUTCString();
return item;
} catch (e) {
return Promise.resolve('');
}
})
return item;
} catch (e) {
return Promise.resolve('');
}
})
)
);
lib/routes/bibgame/category.js
+12 -13
View File
@@ -30,22 +30,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('#comment').parent().remove();
content('#comment').parent().remove();
item.description = content('.abstract').html();
item.pubDate = timezone(new Date(content('.time_box').text().trim()), +8);
item.description = content('.abstract').html();
item.pubDate = timezone(new Date(content('.time_box').text().trim()), +8);
return item;
})
return item;
})
)
);
lib/routes/bilibili/followings_article.js
+1 -1
View File
@@ -45,7 +45,7 @@ module.exports = async (ctx) => {
link: link,
author: card.desc.user_profile.info.uname,
};
return Promise.resolve(item);
return item;
})
);
lib/routes/bilibili/followings_video.js
+10 -13
View File
@@ -26,20 +26,17 @@ module.exports = async (ctx) => {
}
const cards = response.data.data.cards;
const out = await Promise.all(
cards.map(async (card) => {
const card_data = JSON.parse(card.card);
const out = cards.map((card) => {
const card_data = JSON.parse(card.card);
const item = {
title: card_data.title,
description: `${card_data.desc}${!disableEmbed ? `<br><br>${utils.iframe(card_data.aid)}` : ''}<br><img src="${card_data.pic}">`,
pubDate: new Date(card_data.pubdate * 1000).toUTCString(),
link: card_data.pubdate > utils.bvidTime && card_data.bvid ? `https://www.bilibili.com/video/${card_data.bvid}` : `https://www.bilibili.com/video/av${card_data.aid}`,
author: card.desc.user_profile.info.uname,
};
return Promise.resolve(item);
})
);
return {
title: card_data.title,
description: `${card_data.desc}${!disableEmbed ? `<br><br>${utils.iframe(card_data.aid)}` : ''}<br><img src="${card_data.pic}">`,
pubDate: new Date(card_data.pubdate * 1000).toUTCString(),
link: card_data.pubdate > utils.bvidTime && card_data.bvid ? `https://www.bilibili.com/video/${card_data.bvid}` : `https://www.bilibili.com/video/av${card_data.aid}`,
author: card.desc.user_profile.info.uname,
};
});
ctx.state.data = {
title: `${name} 关注视频动态`,
lib/routes/biodiscover/index.js
+16 -17
View File
@@ -19,25 +19,24 @@ module.exports = async (ctx) => {
link: listUrl,
description: $('meta[name=description]').attr('content'),
item: await Promise.all(
itemUrls.map(
async (itemUrl) =>
await ctx.cache.tryGet(itemUrl, async () => {
const detailResponse = await got({ url: itemUrl });
const $ = cheerio.load(detailResponse.data);
itemUrls.map((itemUrl) =>
ctx.cache.tryGet(itemUrl, async () => {
const detailResponse = await got({ url: itemUrl });
const $ = cheerio.load(detailResponse.data);
const dateStr = $('.from').children().last().text().replace('·', '').trim();
const dateStr = $('.from').children().last().text().replace('·', '').trim();
return {
title: $('.article_title').text(),
author: $('.from').children().first().text().trim(),
category: $('.article .share .tag a')
.map((_, a) => $(a).text().trim())
.toArray(),
description: $('.article .main_info').html(),
pubDate: timezone(parseRelativeDate(dateStr) || new Date(dateStr), +8),
link: itemUrl,
};
})
return {
title: $('.article_title').text(),
author: $('.from').children().first().text().trim(),
category: $('.article .share .tag a')
.map((_, a) => $(a).text().trim())
.toArray(),
description: $('.article .main_info').html(),
pubDate: timezone(parseRelativeDate(dateStr) || new Date(dateStr), +8),
link: itemUrl,
};
})
)
),
};
lib/routes/bioon/latest.js
+13 -14
View File
@@ -23,24 +23,23 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const content = cheerio.load(detailResponse.data);
const subtitle = content('div.title5 p').text().split(' ');
const pubDate = `${subtitle.slice(-2)[0]} ${subtitle.slice(-1)}`;
const subtitle = content('div.title5 p').text().split(' ');
const pubDate = `${subtitle.slice(-2)[0]} ${subtitle.slice(-1)}`;
item.pubDate = new Date(pubDate + ' GMT+8').toUTCString();
item.description = content('div.text3').html();
item.pubDate = new Date(pubDate + ' GMT+8').toUTCString();
item.description = content('div.text3').html();
return item;
})
return item;
})
)
);
lib/routes/bioone/featured.js
+12 -13
View File
@@ -24,21 +24,20 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('#divARTICLECONTENTTop').html();
item.doi = content('meta[name="dc.Identifier"]').attr('content');
item.pubDate = new Date(content('meta[name="dc.Date"]').attr('content')).toUTCString();
item.description = content('#divARTICLECONTENTTop').html();
item.doi = content('meta[name="dc.Identifier"]').attr('content');
item.pubDate = new Date(content('meta[name="dc.Date"]').attr('content')).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/bjp/apod.js
+1 -1
View File
@@ -37,7 +37,7 @@ module.exports = async (ctx) => {
};
ctx.cache.set(link, JSON.stringify(item));
return Promise.resolve(item);
return item;
})
);
ctx.state.data = {
lib/routes/blizzard/news.js
+12 -13
View File
@@ -26,21 +26,20 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.author = content('.ArticleDetail-bylineAuthor').text();
item.description = content('.ArticleDetail-content').html();
item.pubDate = content('.ArticleDetail-subHeadingLeft time').attr('timestamp');
item.author = content('.ArticleDetail-bylineAuthor').text();
item.description = content('.ArticleDetail-content').html();
item.pubDate = content('.ArticleDetail-subHeadingLeft time').attr('timestamp');
return item;
})
return item;
})
)
);
lib/routes/blogs/wangyin.js
+1 -1
View File
@@ -38,7 +38,7 @@ module.exports = async (ctx) => {
item.description = itemElement('.inner').html();
ctx.cache.set(link, JSON.stringify(item));
return Promise.resolve(item);
return item;
})
);
lib/routes/boston/index.js
+48 -49
View File
@@ -8,57 +8,56 @@ module.exports = async (ctx) => {
const feed = await parser.parseURL(rssUrl);
const items = await Promise.all(
feed.items.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const response = await got({
url: item.link,
method: 'get',
feed.items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got({
url: item.link,
method: 'get',
});
const html = response.body;
const $ = cheerio.load(html);
const content = $('div.content-text-article');
// Disable image lazyloading
content.find('img').each((i, e) => {
e = $(e);
e.attr('src', e.attr('srcset'));
});
// Categories
const categories = [];
$('meta[property="article:tag"]').each((i, e) => {
categories.push(e.attribs.content);
});
// Updated date
const updatedAt = $('meta[property="article:modified_time"]').attr().content;
// Summary
const summary = $('meta[property="og:description"]').attr().content;
// Remove unwanted DOMs
const unwanted_element_selectors = ['.ad-container', '.content-toaster--newsletter'];
unwanted_element_selectors.forEach((selector) => {
content.find(selector).each((i, e) => {
$(e).remove();
});
const html = response.body;
const $ = cheerio.load(html);
const content = $('div.content-text-article');
});
// Disable image lazyloading
content.find('img').each((i, e) => {
e = $(e);
e.attr('src', e.attr('srcset'));
});
// Categories
const categories = [];
$('meta[property="article:tag"]').each((i, e) => {
categories.push(e.attribs.content);
});
// Updated date
const updatedAt = $('meta[property="article:modified_time"]').attr().content;
// Summary
const summary = $('meta[property="og:description"]').attr().content;
// Remove unwanted DOMs
const unwanted_element_selectors = ['.ad-container', '.content-toaster--newsletter'];
unwanted_element_selectors.forEach((selector) => {
content.find(selector).each((i, e) => {
$(e).remove();
});
});
return {
title: item.title.trim(),
id: item.link,
pubDate: new Date(item.pubDate).toUTCString(),
updated: new Date(updatedAt).toUTCString(),
author: item.author.trim(),
link: item.link,
summary: summary.trim(),
description: content.html(),
category: categories,
icon: 'https://www.boston.com/wp-content/themes/bdc/images/favicons/largetile.png',
logo: 'https://www.boston.com/wp-content/themes/bdc/images/favicons/largetile.png',
};
})
return {
title: item.title.trim(),
id: item.link,
pubDate: new Date(item.pubDate).toUTCString(),
updated: new Date(updatedAt).toUTCString(),
author: item.author.trim(),
link: item.link,
summary: summary.trim(),
description: content.html(),
category: categories,
icon: 'https://www.boston.com/wp-content/themes/bdc/images/favicons/largetile.png',
logo: 'https://www.boston.com/wp-content/themes/bdc/images/favicons/largetile.png',
};
})
)
);
lib/routes/caixin/blog.js
+1 -1
View File
@@ -44,7 +44,7 @@ module.exports = async (ctx) => {
link,
author: item['dc:creator'],
};
const other = await ctx.cache.tryGet(link, async () => await load(link, index === 0));
const other = await ctx.cache.tryGet(link, () => load(link, index === 0));
return Promise.resolve(Object.assign({}, single, other));
})
);
lib/routes/caixin/database.js
+11 -12
View File
@@ -24,20 +24,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.pubDate = timezone(parseDate(content('#pubtime_baidu').text()), +8);
item.description = `<p>${content('#subhead').html()}</p>`;
item.pubDate = timezone(parseDate(content('#pubtime_baidu').text()), +8);
item.description = `<p>${content('#subhead').html()}</p>`;
return item;
})
return item;
})
)
);
lib/routes/caixin/yxnews.js
+1 -1
View File
@@ -12,7 +12,7 @@ module.exports = async (ctx) => {
const data = response.data.data.list;
const items = await Promise.all(
data.map(async (item) => ({
data.map((item) => ({
title: item.title,
description: item.text,
link: `http://k.caixin.com/web/detail_${item.oneline_news_code}`,
lib/routes/cartoonmad/comic.js
+2 -2
View File
@@ -16,10 +16,10 @@ const load = ($, baseUrl) => {
};
};
const getChapters = async (id, $, caches) => {
const getChapters = (id, $, caches) => {
const chapters = $('#info').eq(1).find('a');
return await Promise.all(
return Promise.all(
chapters
.slice(chapters.length - 10, chapters.length)
.map(async (_, ele) => {
lib/routes/caus/index.js
+10 -11
View File
@@ -64,19 +64,18 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: `${apiRootUrl}/toronto/display/contentWithRelate?contentId=${item.link}`,
});
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: `${apiRootUrl}/toronto/display/contentWithRelate?contentId=${item.link}`,
});
item.link = `${rootUrl}/detail/${item.link}`;
item.description = detailResponse.data.data.content.content;
item.link = `${rootUrl}/detail/${item.link}`;
item.description = detailResponse.data.data.content.content;
return item;
})
return item;
})
)
);
lib/routes/cbc/topics.js
+1 -2
View File
@@ -53,14 +53,13 @@ module.exports = async (ctx) => {
return Promise.resolve(undefined);
}
const item = {
return {
title: title,
description: description,
pubDate: pubDate,
link: link,
author: author,
};
return Promise.resolve(item);
})
);
lib/routes/cbndata/information.js
+12 -13
View File
@@ -18,22 +18,21 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
item.link = `${rootUrl}/information/${item.title}`;
item.link = `${rootUrl}/information/${item.title}`;
item.title = detailResponse.data.data.title;
item.author = detailResponse.data.data.author;
item.description = detailResponse.data.data.content;
item.title = detailResponse.data.data.title;
item.author = detailResponse.data.data.author;
item.description = detailResponse.data.data.content;
return item;
})
return item;
})
)
);
lib/routes/ccdi/scdc.js
+1 -1
View File
@@ -34,7 +34,7 @@ module.exports = async (ctx) => {
ctx.cache.set(key, item.description);
}
return Promise.resolve(item);
return item;
})
.get()
);
lib/routes/ccf/news.js
+12 -13
View File
@@ -25,22 +25,21 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('.new_info .num').remove();
content('.new_info .num').remove();
item.description = content('.txt').html();
item.pubDate = new Date(content('.new_info span').text()).toUTCString();
item.description = content('.txt').html();
item.pubDate = new Date(content('.new_info span').text()).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/ccg/index.js
+10 -11
View File
@@ -26,19 +26,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.pinpai-page').html();
item.description = content('.pinpai-page').html();
return item;
})
return item;
})
)
);
lib/routes/cctv/jx.js
+10 -11
View File
@@ -23,19 +23,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.tujitop').html();
item.description = content('.tujitop').html();
return item;
})
return item;
})
)
);
lib/routes/cctv/special.js
+57 -58
View File
@@ -30,74 +30,73 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
let date;
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
let date;
if (item.link.indexOf('photo.cctv.com') > 0) {
const imageLink = item.link.split('.shtml')[0] + '.xml?randomNum=26';
const imageResponse = await got({
method: 'get',
url: imageLink,
if (item.link.indexOf('photo.cctv.com') > 0) {
const imageLink = item.link.split('.shtml')[0] + '.xml?randomNum=26';
const imageResponse = await got({
method: 'get',
url: imageLink,
});
const images = cheerio.load(imageResponse.data);
let description;
item.description = '';
images('li')
.get()
.forEach((i) => {
i = images(i);
item.description += `<img src="${i.attr('photourl')}">`;
date = i.attr('time').replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, '');
description = i.html();
});
const images = cheerio.load(imageResponse.data);
let description;
item.description += `<p>${description}</p>`;
} else {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const match = detailResponse.data.match(/publishDate ="(.*)";/);
item.description = '';
if (match === null) {
let dateSpan;
images('li')
.get()
.forEach((i) => {
i = images(i);
item.description += `<img src="${i.attr('photourl')}">`;
date = i.attr('time').replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, '');
description = i.html();
});
if (content('div.info span').html() !== null) {
dateSpan = content('div.info span').text();
} else if (content('div.info1').html() !== null) {
dateSpan = content('div.info1').text().split(' | ')[1];
} else {
dateSpan = content('span.time').text();
}
item.description += `<p>${description}</p>`;
date = dateSpan.replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, '');
} else {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const match = detailResponse.data.match(/publishDate ="(.*)";/);
if (match === null) {
let dateSpan;
if (content('div.info span').html() !== null) {
dateSpan = content('div.info span').text();
} else if (content('div.info1').html() !== null) {
dateSpan = content('div.info1').text().split(' | ')[1];
} else {
dateSpan = content('span.time').text();
}
date = dateSpan.replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, '');
} else {
const dateString = match[1];
date = `${dateString.slice(0, 4)}-${dateString.slice(4, 6)}-${dateString.slice(6, 8)} ${dateString.slice(8, 10)}:${dateString.slice(10, 12)}:${dateString.slice(12, 14)}`;
}
if (content('#content_area').html() !== null) {
item.description = content('#content_area').html();
} else if (content('div.cnt_bd').html() !== null) {
content('div.function,h1,h2').remove();
item.description = content('div.cnt_bd').html();
} else if (content('#text_area').html() !== null) {
item.description = content('#text_area').html();
} else {
item.description = content('div.con').html();
}
const dateString = match[1];
date = `${dateString.slice(0, 4)}-${dateString.slice(4, 6)}-${dateString.slice(6, 8)} ${dateString.slice(8, 10)}:${dateString.slice(10, 12)}:${dateString.slice(12, 14)}`;
}
item.pubDate = new Date(date + ' GMT+8').toUTCString();
if (content('#content_area').html() !== null) {
item.description = content('#content_area').html();
} else if (content('div.cnt_bd').html() !== null) {
content('div.function,h1,h2').remove();
item.description = content('div.cnt_bd').html();
} else if (content('#text_area').html() !== null) {
item.description = content('#text_area').html();
} else {
item.description = content('div.con').html();
}
}
return item;
})
item.pubDate = new Date(date + ' GMT+8').toUTCString();
return item;
})
)
);
lib/routes/cctv/utils/mzzlbg.js
+1 -1
View File
@@ -25,7 +25,7 @@ module.exports = async () => {
});
item.description = `<video src="${videoDetail.hls_url}" controls="controls" poster="${video.img.replace(/\?.+/g, '')}" style="width: 100%"></video>`;
return Promise.resolve(item);
return item;
})
);
lib/routes/cctv/utils/xinwen1j1.js
+3 -4
View File
@@ -27,9 +27,8 @@ async function load(link) {
}
// 同步 列表
const ProcessFeed = async (list, caches) =>
// host = "hostname"
await Promise.all(
const ProcessFeed = (list, caches) =>
Promise.all(
list.map(async (item) => {
const $ = cheerio.load(item);
const title = $('a').text();
@@ -47,7 +46,7 @@ const ProcessFeed = async (list, caches) =>
// 使用tryGet方法从缓存获取内容。
// 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
const other = await caches.tryGet(itemUrl, async () => await load(itemUrl));
const other = await caches.tryGet(itemUrl, () => load(itemUrl));
// 合并解析后的结果集作为该篇文章最终的输出结果
return Promise.resolve(Object.assign({}, single, other));
lib/routes/cctv/xwlb.js
+1 -1
View File
@@ -36,7 +36,7 @@ module.exports = async (ctx) => {
return alist.join('<br/>\n');
}),
};
return Promise.resolve(item);
return item;
})
);
lib/routes/cgtn/most.js
+10 -11
View File
@@ -25,19 +25,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('#cmsMainContent').html();
item.description = content('#cmsMainContent').html();
return item;
})
return item;
})
)
);
lib/routes/cgtn/opinions.js
+11 -12
View File
@@ -26,20 +26,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.author = content('.news-author-name').text();
item.description = content('#cmsMainContent').html();
item.author = content('.news-author-name').text();
item.description = content('#cmsMainContent').html();
return item;
})
return item;
})
)
);
lib/routes/cgtn/pick.js
+10 -11
View File
@@ -22,19 +22,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('#cmsMainContent').html();
item.description = content('#cmsMainContent').html();
return item;
})
return item;
})
)
);
lib/routes/cgtn/top.js
+10 -11
View File
@@ -22,19 +22,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('#cmsMainContent').html();
item.description = content('#cmsMainContent').html();
return item;
})
return item;
})
)
);
lib/routes/chaoli/index.js
+12 -13
View File
@@ -25,21 +25,20 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.author = content('.info h3').eq(0).text();
item.description = content('.postBody').eq(0).html();
item.pubDate = new Date(parseInt(content('.info .time').attr('data-timestamp')) * 1000).toUTCString();
item.author = content('.info h3').eq(0).text();
item.description = content('.postBody').eq(0).html();
item.pubDate = new Date(parseInt(content('.info .time').attr('data-timestamp')) * 1000).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/chaping/banner.js
+12 -13
View File
@@ -14,21 +14,20 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
bannerList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = JSON.parse(detailResponse.data.match(/"current":(.*?),"optionsList":/)[1]);
bannerList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = JSON.parse(detailResponse.data.match(/"current":(.*?),"optionsList":/)[1]);
item.description = content.content;
item.author = content.article_author.name;
item.pubDate = new Date(content.time_publish_timestamp * 1000).toUTCString();
item.description = content.content;
item.author = content.article_author.name;
item.pubDate = new Date(content.time_publish_timestamp * 1000).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/chaping/news.js
+10 -11
View File
@@ -29,19 +29,18 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = JSON.parse(detailResponse.data.match(/"current":(.*?),"optionsList":/)[1]);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = JSON.parse(detailResponse.data.match(/"current":(.*?),"optionsList":/)[1]);
item.description = content.content;
item.description = content.content;
return item;
})
return item;
})
)
);
lib/routes/chicagotribune/index.js
+49 -50
View File
@@ -11,58 +11,57 @@ module.exports = async (ctx) => {
const feed = await parser.parseURL(rssUrl);
const items = await Promise.all(
feed.items.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const response = await got({
url: item.link,
method: 'get',
feed.items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got({
url: item.link,
method: 'get',
});
const html = response.body;
const $ = cheerio.load(html);
const content = $('article > main > section#left');
const categories = $('meta[property="article:section"]')
.attr('content')
.split(',')
.map((e) => e.trim())
.filter((e) => e !== '');
// Disable image lazyloading
content.find('img').each((i, e) => {
e = $(e);
e.attr('src', e.attr('data-src'));
});
// Remove unwanted DOMs
const unwanted_element_selectors = [
'[data-pb-name="Recommender"]',
'.inline-ad',
'[data-type="newsletter-promo-card"]',
'[data-type="interstitial_link"]',
'[data-type="recommender"]',
'.recommender',
'[class*="pb-f-ads-"]',
];
unwanted_element_selectors.forEach((selector) => {
content.find(selector).each((i, e) => {
$(e).remove();
});
const html = response.body;
const $ = cheerio.load(html);
const content = $('article > main > section#left');
});
const categories = $('meta[property="article:section"]')
.attr('content')
.split(',')
.map((e) => e.trim())
.filter((e) => e !== '');
// Disable image lazyloading
content.find('img').each((i, e) => {
e = $(e);
e.attr('src', e.attr('data-src'));
});
// Remove unwanted DOMs
const unwanted_element_selectors = [
'[data-pb-name="Recommender"]',
'.inline-ad',
'[data-type="newsletter-promo-card"]',
'[data-type="interstitial_link"]',
'[data-type="recommender"]',
'.recommender',
'[class*="pb-f-ads-"]',
];
unwanted_element_selectors.forEach((selector) => {
content.find(selector).each((i, e) => {
$(e).remove();
});
});
return {
title: item.title.trim(),
id: item.guid,
pubDate: new Date(item.pubDate).toUTCString(),
author: item.creator.trim(),
link: item.link,
summary: item.content.trim(),
description: content.html(),
category: categories,
icon: 'https://www.chicagotribune.com/pb/resources/images/ct_icons/144-iTunesArtwork.png',
logo: 'https://upload.wikimedia.org/wikipedia/commons/c/c4/Chicago_Tribune_Logo.svg',
};
})
return {
title: item.title.trim(),
id: item.guid,
pubDate: new Date(item.pubDate).toUTCString(),
author: item.creator.trim(),
link: item.link,
summary: item.content.trim(),
description: content.html(),
category: categories,
icon: 'https://www.chicagotribune.com/pb/resources/images/ct_icons/144-iTunesArtwork.png',
logo: 'https://upload.wikimedia.org/wikipedia/commons/c/c4/Chicago_Tribune_Logo.svg',
};
})
)
);
lib/routes/chiculture/topic.js
+23 -24
View File
@@ -19,36 +19,35 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const pubDate = detailResponse.data.match(/上載日期：(.*)<\/p>/);
const pubDate = detailResponse.data.match(/上載日期：(.*)<\/p>/);
if (pubDate) {
item.pubDate = date(pubDate[1]);
} else if (item.title.indexOf('一周時事通識') > -1) {
for (const tag of item.pubDate) {
if (tag.title.match(/^\d{4}年$/)) {
item.pubDate = date(`${tag.title.replace('年', '')}-${item.title.replace(/）/g, '').split('- ')[1].split('/').reverse().join('-')}`);
break;
}
if (pubDate) {
item.pubDate = date(pubDate[1]);
} else if (item.title.indexOf('一周時事通識') > -1) {
for (const tag of item.pubDate) {
if (tag.title.match(/^\d{4}年$/)) {
item.pubDate = date(`${tag.title.replace('年', '')}-${item.title.replace(/）/g, '').split('- ')[1].split('/').reverse().join('-')}`);
break;
}
} else if (item.title.match(/^\d{4}年新聞回顧$/)) {
item.pubDate = date(`${item.title.split('年')[0]}-12-31`);
} else {
item.pubDate = '';
}
} else if (item.title.match(/^\d{4}年新聞回顧$/)) {
item.pubDate = date(`${item.title.split('年')[0]}-12-31`);
} else {
item.pubDate = '';
}
item.description = content('#article_main_content').html();
item.description = content('#article_main_content').html();
return item;
})
return item;
})
)
);
lib/routes/chinafile/index.js
+36 -37
View File
@@ -13,50 +13,49 @@ module.exports = async (ctx) => {
const feed = await parser.parseURL(rssUrl);
const items = await Promise.all(
feed.items.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
let url = item.link;
const response = await got({ url });
const html = response.body;
const $ = cheerio.load(html);
const content = $('article');
feed.items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
let url = item.link;
const response = await got({ url });
const html = response.body;
const $ = cheerio.load(html);
const content = $('article');
// Cover
const cover = $('.view-featured-photo');
// Cover
const cover = $('.view-featured-photo');
if (cover.length > 0) {
cover.insertBefore(content[0].childNodes[0]);
$(cover).remove();
}
if (cover.length > 0) {
cover.insertBefore(content[0].childNodes[0]);
$(cover).remove();
}
// Summary
const summary = $('meta[name="description"]').attr('content');
const updatedAt = $('meta[name="og:updated_time"]').attr('content');
// Summary
const summary = $('meta[name="description"]').attr('content');
const updatedAt = $('meta[name="og:updated_time"]').attr('content');
const categories = $('meta[name="news_keywords"]')
.attr('content')
.split(',')
.map((c) => c.trim());
const categories = $('meta[name="news_keywords"]')
.attr('content')
.split(',')
.map((c) => c.trim());
url = $('link[rel="canonical"]').attr('href');
url = $('link[rel="canonical"]').attr('href');
const pubDate = dayjs(`${item.pubDate.replace(' - ', ' ').replace('am', ' am').replace('pm', ' pm')}`).toISOString();
const pubDate = dayjs(`${item.pubDate.replace(' - ', ' ').replace('am', ' am').replace('pm', ' pm')}`).toISOString();
return {
title: item.title,
id: item.guid,
pubDate,
updated: updatedAt,
author: item.creator,
link: url,
summary,
description: content.html(),
category: categories,
icon: 'https://www.chinafile.com/sites/default/files/chinafile_favicon.png',
logo: 'https://www.chinafile.com/sites/all/themes/cftwo/assets/images/logos/logo-large.png',
};
})
return {
title: item.title,
id: item.guid,
pubDate,
updated: updatedAt,
author: item.creator,
link: url,
summary,
description: content.html(),
category: categories,
icon: 'https://www.chinafile.com/sites/default/files/chinafile_favicon.png',
logo: 'https://www.chinafile.com/sites/all/themes/cftwo/assets/images/logos/logo-large.png',
};
})
)
);
lib/routes/chinalaborwatch/reports.js
+12 -13
View File
@@ -26,23 +26,22 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.pubDate = new Date(content('#ContentPlaceHolder1_LabelDate').text().replace(/年|月/g, '-').replace('日', '')).toUTCString();
item.pubDate = new Date(content('#ContentPlaceHolder1_LabelDate').text().replace(/年|月/g, '-').replace('日', '')).toUTCString();
content('h1, #ContentPlaceHolder1_LabelDate').remove();
content('h1, #ContentPlaceHolder1_LabelDate').remove();
item.description = content('.mainContent').html();
item.description = content('.mainContent').html();
return item;
})
return item;
})
)
);
lib/routes/chsi/zszcgd.js
+10 -11
View File
@@ -28,19 +28,18 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.list').html();
item.description = content('.list').html();
return item;
})
return item;
})
)
);
lib/routes/chuapp/index.js
+9 -10
View File
@@ -34,16 +34,15 @@ module.exports = async (ctx) => {
.get();
const item = await Promise.all(
articles.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const res = await got.get(`http://www.chuapp.com${item.link}`);
const s = cheerio.load(res.data);
item.description = s('.content .the-content').html();
item.pubDate = new Date(s('.friendly_time').attr('data-time'));
item.author = s('.author-time .fn-left').text();
return Promise.resolve(item);
})
articles.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const res = await got.get(`http://www.chuapp.com${item.link}`);
const s = cheerio.load(res.data);
item.description = s('.content .the-content').html();
item.pubDate = new Date(s('.friendly_time').attr('data-time'));
item.author = s('.author-time .fn-left').text();
return item;
})
)
);
lib/routes/citavi/index.js
+11 -12
View File
@@ -28,20 +28,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('div.panel-body').html();
item.pubDate = new Date(date(content('small.gray').eq(0).text().split(' • ')[1])).toUTCString();
item.description = content('div.panel-body').html();
item.pubDate = new Date(date(content('small.gray').eq(0).text().split(' • ')[1])).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/cktest/policy.js
+14 -15
View File
@@ -15,24 +15,23 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const dateArray = content('meta[name="firstpublishedtime"]').attr('content').split('-');
const time = dateArray.pop();
const dateArray = content('meta[name="firstpublishedtime"]').attr('content').split('-');
const time = dateArray.pop();
item.pubDate = `${dateArray.join('-')} ${time}`;
item.title = content('title').text().split('_')[0];
item.description = content('.wrap').html() || content('.policyLibraryOverview_content').html();
item.pubDate = `${dateArray.join('-')} ${time}`;
item.title = content('title').text().split('_')[0];
item.description = content('.wrap').html() || content('.policyLibraryOverview_content').html();
return item;
})
return item;
})
)
);
lib/routes/clb/commentary.js
+11 -12
View File
@@ -24,20 +24,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('.field-name-body').html();
item.pubDate = new Date(content('meta[property="article:published_time"]').attr('content')).toUTCString();
item.description = content('.field-name-body').html();
item.pubDate = new Date(content('meta[property="article:published_time"]').attr('content')).toUTCString();
return item;
})
return item;
})
)
);
lib/routes/cls/depth.js
+10 -11
View File
@@ -41,18 +41,17 @@ module.exports = async (ctx) => {
);
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('div.detail-content').html();
return item;
})
item.description = content('div.detail-content').html();
return item;
})
)
);
lib/routes/cmes/news.js
+14 -15
View File
@@ -35,25 +35,24 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
content('h3, script, .bshare-custom').remove();
content('h3, script, .bshare-custom').remove();
item.description = content('.xhjj').html() || content('.detail-page').html();
item.description = content('.xhjj').html() || content('.detail-page').html();
if (content('#articleTime').html()) {
item.pubDate = new Date(content('#articleTime').text()).toUTCString();
}
if (content('#articleTime').html()) {
item.pubDate = new Date(content('#articleTime').text()).toUTCString();
}
return item;
})
return item;
})
)
);
lib/routes/cna/index.js
+11 -12
View File
@@ -30,20 +30,19 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const topImage = content('.fullPic').html();
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const topImage = content('.fullPic').html();
item.description = (topImage === null ? '' : topImage) + content('.paragraph').eq(0).html();
item.description = (topImage === null ? '' : topImage) + content('.paragraph').eq(0).html();
return item;
})
return item;
})
)
);
lib/routes/comicskingdom/index.js
+21 -22
View File
@@ -27,30 +27,29 @@ module.exports = async (ctx) => {
throw `Comic Not Found - ${name}`;
}
const items = await Promise.all(
links.map(
async (link) =>
await ctx.cache.tryGet(link, async () => {
const detailResponse = await got({
method: 'get',
url: link,
});
const content = cheerio.load(detailResponse.data);
links.map((link) =>
ctx.cache.tryGet(link, async () => {
const detailResponse = await got({
method: 'get',
url: link,
});
const content = cheerio.load(detailResponse.data);
const title = content('title').text();
const image = content('meta[property="og:image"]').attr('content');
const description = `<img src="${image}" />`;
// Pull the date out of the URL
const pubDate = new Date(link.split('/').slice(-3).join('/')).toUTCString();
const title = content('title').text();
const image = content('meta[property="og:image"]').attr('content');
const description = `<img src="${image}" />`;
// Pull the date out of the URL
const pubDate = new Date(link.split('/').slice(-3).join('/')).toUTCString();
return {
title: title,
author: author,
category: 'comic',
description: description,
pubDate: pubDate,
link: link,
};
})
return {
title: title,
author: author,
category: 'comic',
description: description,
pubDate: pubDate,
link: link,
};
})
)
);
lib/routes/commonapp/blog.js
+10 -11
View File
@@ -18,20 +18,19 @@ module.exports = async (ctx) => {
}));
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
const content = cheerio.load(detailResponse.data);
item.description = content('.inner-page').html();
item.description = content('.inner-page').html();
return item;
})
return item;
})
)
);
lib/routes/cool18/index.js
+15 -16
View File
@@ -29,24 +29,23 @@ module.exports = async (ctx) => {
.get();
const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.title = content('title').text().replace(' - cool18.com', '');
item.author = detailResponse.data.match(/送交者: .*>(.*)<.*\[/)[1];
item.pubDate = new Date(detailResponse.data.match(/于 (.*) 已读/)[1]).toUTCString();
item.description = content('pre')
.html()
.replace(/<font color="#E6E6DD">cool18.com<\/font>/g, '');
item.title = content('title').text().replace(' - cool18.com', '');
item.author = detailResponse.data.match(/送交者: .*>(.*)<.*\[/)[1];
item.pubDate = new Date(detailResponse.data.match(/于 (.*) 已读/)[1]).toUTCString();
item.description = content('pre')
.html()
.replace(/<font color="#E6E6DD">cool18.com<\/font>/g, '');
return item;
})
return item;
})
)
);
lib/routes/coolapk/dyh.js
+1 -1
View File
@@ -19,7 +19,7 @@ module.exports = async (ctx) => {
const data = response.data.data;
let out = await Promise.all(
data.map(async (item) => {
data.map((item) => {
if (!targetTitle) {
targetTitle = item.targetRow.title;
}
lib/routes/coolapk/hot.js
+1 -1
View File
@@ -86,7 +86,7 @@ module.exports = async (ctx) => {
}
}
let out = await Promise.all(t.map(async (item) => utils.parseDynamic(item, ctx)));
let out = await Promise.all(t.map((item) => utils.parseDynamic(item, ctx)));
out = out.filter((i) => i);
lib/routes/coolapk/huati.js
+1 -1
View File
@@ -11,7 +11,7 @@ module.exports = async (ctx) => {
});
const data = response.data.data;
let out = await Promise.all(data.map(async (item) => utils.parseDynamic(item, ctx)));
let out = await Promise.all(data.map((item) => utils.parseDynamic(item, ctx)));
out = out.filter((i) => i); // 去除空值
if (out.length === 0) {

Some files were not shown because too many files have changed in this diff Show More