opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-09-24 23:12:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

test: wechat-mp

Browse Source
This commit is contained in:
DIYgod
2024-02-25 02:08:35 +08:00
parent bdf8ace129
commit 01e142ccaf
22 changed files with 60 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/utils/wechat-mp.test.ts
+141
View File
@@ -0,0 +1,141 @@
import { describe, expect, it } from '@jest/globals';
import { load } from 'cheerio';
import nock from 'nock';
import { fixArticleContent, fetchArticle, finishArticleItem, normalizeUrl } from '@/utils/wechat-mp';
// date from the cache will be an ISO8601 string, so we need to use this function
const compareDate = (date1, date2) => {
date1 = typeof date1 === 'string' ? new Date(date1) : date1;
date2 = typeof date2 === 'string' ? new Date(date2) : date2;
return date1.getTime() === date2.getTime();
};
describe('wechat-mp', () => {
it('fixArticleContent', () => {
const divHeader = '<div class="rich_media_content " id="js_content">';
const divFooter = '</div>';
const codeSection =
'<section class="code-snippet__fix code-snippet__js">' +
'<ul class="code-snippet__line-index code-snippet__js">' +
'<li></li><li></li><li></li>' +
'</ul >' +
'<pre class="code-snippet__js">' +
'<code><span class="code-snippet_outer">Line1 {</span></code>' +
'<code><span class="code-snippet__keyword">Line2</span></code>' +
'<code><span class="code-snippet_outer">Line3 }</span></code>' +
'</pre></section>';
const expectedCodeSection =
'<p class="code-snippet__fix code-snippet__js">' +
'<pre class="code-snippet__js">' +
'<code><span class="code-snippet_outer">Line1 {</span></code>' +
'<br>' +
'<code><span class="code-snippet__keyword">Line2</span></code>' +
'<br>' +
'<code><span class="code-snippet_outer">Line3 }</span></code>' +
'<br>' +
'</pre></p>';
const htmlSection =
codeSection +
'<section>test</section>' +
'<section><p>test</p></section>' +
'<section><div>test</div></section>' +
'<section><section><section>test</section></section></section>' +
'<div><section><p>test</p></section></div>' +
'<p>test</p>' +
'<div><p>test</p></div>' +
'<script>const test = "test"</script>';
const expectedHtmlSection =
expectedCodeSection + '<p>test</p>' + '<div><p>test</p></div>' + '<div><div>test</div></div>' + '<div><div><p>test</p></div></div>' + '<div><div><p>test</p></div></div>' + '<p>test</p>' + '<div><p>test</p></div>';
let $ = load(divHeader + htmlSection + divFooter);
expect(fixArticleContent(htmlSection)).toBe(expectedHtmlSection);
expect(fixArticleContent($('div#js_content.rich_media_content'))).toBe(expectedHtmlSection);
const htmlImg = '<img alt="test" data-src="http://rsshub.test/test.jpg" src="http://rsshub.test/test.jpg">' + '<img alt="test" data-src="http://rsshub.test/test.jpg">' + '<img alt="test" src="http://rsshub.test/test.jpg">';
const expectedHtmlImg = Array.from({ length: 3 + 1 }).join('<img alt="test" src="http://rsshub.test/test.jpg">');
$ = load(divHeader + htmlImg + divFooter);
expect(fixArticleContent(htmlImg)).toBe(expectedHtmlImg);
expect(fixArticleContent($('div#js_content.rich_media_content'))).toBe(expectedHtmlImg);
expect(fixArticleContent(htmlImg, true)).toBe(htmlImg);
expect(fixArticleContent($('div#js_content.rich_media_content'), true)).toBe(htmlImg);
expect(fixArticleContent('')).toBe('');
expect(fixArticleContent()).toBe('');
expect(fixArticleContent($('div#something_not_in.the_document_tree'))).toBe('');
});
it('normalizeUrl', () => {
const mpRoot = 'https://mp.weixin.qq.com';
const mpArticleRoot = mpRoot + '/s';
const shortUrl = mpArticleRoot + '/-rwvHhqYbKGCVFeXRNknYQ';
const shortUrlWithQueryAndHash = shortUrl + '?foo=bar#baz';
expect(normalizeUrl(shortUrlWithQueryAndHash)).toBe(shortUrl);
const longUrlShortened = mpArticleRoot + '?__biz=MzA4MjQxNjQzMA==' + '&mid=2768628484' + '&idx=1' + '&sn=93dcc54ce807f7793739ee2fd2377056';
const longUrl = longUrlShortened + '&chksm=bf774d458800c453c94cae866093680e6cac6a1f02cab7e82683f82f35f7f487e2daa1dcde20' + '&scene=75' + '#wechat_redirect';
expect(normalizeUrl(longUrl)).toBe(longUrlShortened);
const temporaryUrlShortened =
mpArticleRoot + '?src=11' + '&timestamp=1620536401' + '&ver=3057' + '&signature=vCDI8FQcumnNGv4ScvFP-swQRlirdQSqTfjS8m-oFzgHMkqlNM3ljzjSevcjXLC-z-n0RzzMkNt-lwKMUaskfaqFFrpYZNq4ZCKkFFGj8L*KvH780aEUBJFvWTGmMGLC';
const temporaryUrl = temporaryUrlShortened + '&new=1#foo';
expect(normalizeUrl(temporaryUrl)).toBe(temporaryUrlShortened);
const somethingElse = mpRoot + '/something/else?__biz=foo&mid=bar&idx=baz&sn=qux';
const somethingElseWithHash = somethingElse + '#foo';
expect(normalizeUrl(somethingElseWithHash.replace('https://', 'http://'))).toBe(somethingElse);
const notWechatMp = 'https://im.not.wechat.mp/and/an/error/is/expected';
expect(() => normalizeUrl(notWechatMp)).toThrow();
expect(normalizeUrl(notWechatMp, true)).toBe(notWechatMp);
});
it('fetchArticle_&_finishArticleItem', async () => {
const ct = 1_636_626_300;
const exampleMpArticlePage =
'\n' +
'<meta name="description" content="summary" />\n' +
'<meta name="author" content="author" />\n' +
'<meta property="og:title" content="title" />\n' +
'<meta property="twitter:card" content="summary" />\n' +
'<div class="rich_media_content" id="js_content" style="visibility: hidden;">description</div>\n' +
'<div class="profile_inner"><strong class="profile_nickname">mpName</strong></div>\n' +
'<script type="text/javascript" nonce="000000000">\n' +
'var appmsg_type = "9";\n' +
`var ct = "${ct}";\n` +
'</script>';
nock('https://mp.weixin.qq.com')
.get('/rsshub_test/wechatMp_fetchArticle')
.reply(() => [200, exampleMpArticlePage]);
const httpsUrl = 'https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle';
const httpUrl = httpsUrl.replace(/^https:\/\//, 'http://');
const expectedItem: {
title: string;
summary: string;
author: string;
description: string;
mpName?: string;
link: string;
} = {
title: 'title',
summary: 'summary',
author: 'author',
description: 'description',
mpName: 'mpName',
link: httpsUrl,
};
const expectedDate = new Date(ct * 1000);
const fetchArticleItem = await fetchArticle(httpUrl);
expect(compareDate(fetchArticleItem.pubDate, expectedDate)).toBe(true);
delete fetchArticleItem.pubDate;
expect(fetchArticleItem).toEqual(expectedItem);
delete expectedItem.mpName;
const finishedArticleItem = await finishArticleItem({ link: httpUrl });
expect(compareDate(finishedArticleItem.pubDate, expectedDate)).toBe(true);
delete finishedArticleItem.pubDate;
expect(finishedArticleItem).toEqual(expectedItem);
});
});
lib/utils/wechat-mp.ts
+24 -10
View File
@@ -26,8 +26,9 @@
*/
import got from '@/utils/got';
import { load } from 'cheerio';
import { load, type Cheerio, type Element } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import cache from '@/utils/cache';
const replaceTag = ($, oldTag, newTagName) => {
oldTag = $(oldTag);
@@ -76,12 +77,17 @@ const detectSourceUrl = ($) => {
* @param {boolean} skipImg - Whether to skip fixing images.
* @return {string} - The fixed html, a string.
*/
const fixArticleContent = (html, skipImg = false) => {
html = html && html.html ? html.html() : html; // do not disturb the original tree
if (!html) {
const fixArticleContent = (html?: string | Cheerio<Element>, skipImg = false) => {
let htmlResult = '';
if (typeof html === 'string') {
htmlResult = html;
} else if (html?.html) {
htmlResult = html.html() || '';
}
if (!htmlResult) {
return '';
}
const $ = load(html, undefined, false);
const $ = load(htmlResult, undefined, false);
if (!skipImg) {
// fix img lazy loading
$('img[data-src]').each((_, img) => {
@@ -187,9 +193,9 @@ const normalizeUrl = (url, bypassHostCheck = false) => {
* @param {boolean} bypassHostCheck - Whether to bypass host check.
* @return {Promise<object>} - An object containing the article and its metadata.
*/
const fetchArticle = (ctx, url, bypassHostCheck = false) => {
const fetchArticle = (url, bypassHostCheck = false) => {
url = normalizeUrl(url, bypassHostCheck);
return ctx.cache.tryGet(url, async () => {
return cache.tryGet(url, async () => {
const response = await got(url);
// @ts-expect-error custom field
const $ = load(response.data);
@@ -225,7 +231,15 @@ const fetchArticle = (ctx, url, bypassHostCheck = false) => {
let mpName = $('.profile_nickname').first().text();
mpName = mpName && mpName.trim();
return { title, author, description, summary, pubDate, mpName, link: response.url };
});
}) as Promise<{
title: string;
author: string;
description: string;
summary: string;
pubDate?: Date;
mpName?: string;
link: string;
}>;
};
/**
@@ -244,8 +258,8 @@ const fetchArticle = (ctx, url, bypassHostCheck = false) => {
* @param {boolean} skipLink - Whether to skip overriding `item.link` with the normalized url.
* @return {Promise<object>} - The incoming `item` object, with the article and its metadata filled in.
*/
const finishArticleItem = async (ctx, item, setMpNameAsAuthor = false, skipLink = false) => {
const { title, author, description, summary, pubDate, mpName, link } = await fetchArticle(ctx, item.link);
const finishArticleItem = async (item, setMpNameAsAuthor = false, skipLink = false) => {
const { title, author, description, summary, pubDate, mpName, link } = await fetchArticle(item.link);
item.title = title || item.title;
item.description = description || item.description;
item.summary = summary || item.summary;
lib/v2/gov/general/general.js
+1 -1
View File
@@ -187,7 +187,7 @@ const gdgov = async (info, ctx) => {
});
} else if (idlink.host === 'mp.weixin.qq.com') {
const { finishArticleItem } = require('@/utils/wechat-mp');
return finishArticleItem(ctx, { link });
return finishArticleItem({ link });
} else {
return ctx.cache.tryGet(link, async () => {
// 获取网页
lib/v2/gov/nmpa/generic.js
+1 -1
View File
@@ -45,7 +45,7 @@ module.exports = async (ctx) => {
return item;
});
} else if (/^https:\/\/mp\.weixin\.qq\.com\//.test(item.link)) {
return finishArticleItem(ctx, item);
return finishArticleItem(item);
} else {
return item;
}
lib/v2/hrbeu/job/list.js
+1 -1
View File
@@ -48,7 +48,7 @@ module.exports = async (ctx) => {
const content = cheerio.load(detailResponse.data);
item.description = content('.article').html();
} else if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
await finishArticleItem(ctx, item);
await finishArticleItem(item);
} else {
item.description = '本文需跳转，请点击标题后阅读';
}
lib/v2/hrbeu/uae/news.js
+1 -1
View File
@@ -49,7 +49,7 @@ module.exports = async (ctx) => {
const $1 = cheerio.load(resp.data);
item.description = $1('div#print').html();
} else if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
await finishArticleItem(ctx, item);
await finishArticleItem(item);
} else {
item.description = '本文需跳转，请点击标题后阅读';
}
lib/v2/newrank/wechat.js
+1 -1
View File
@@ -55,7 +55,7 @@ module.exports = async (ctx) => {
link: item.url,
pubDate: item.publicTime,
}));
await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: name + ' - 微信公众号',
lib/v2/nua/utils.js
+1 -1
View File
@@ -59,7 +59,7 @@ const ProcessFeed = (items, artiContent, ctx) =>
return item;
}
case 'wechat-mp':
return fetchArticle(ctx, item.link);
return fetchArticle(item.link);
case 'unknown':
default:
item.description = `暂不支持解析该内容，请点击 ${arti_link('原文', item.link)}`;
lib/v2/pku/nsd.js
+1 -1
View File
@@ -40,7 +40,7 @@ module.exports = async (ctx) => {
list.map((item) => {
switch (item.type) {
case 'wechat-mp':
return finishArticleItem(ctx, item);
return finishArticleItem(item);
case 'pku-news':
return ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ url: item.link, https: { rejectUnauthorized: false } });
lib/v2/sdu/cs.js
+1 -1
View File
@@ -31,7 +31,7 @@ module.exports = async (ctx) => {
item.map((item) =>
ctx.cache.tryGet(item.link, async () => {
if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
return finishArticleItem(ctx, item);
return finishArticleItem(item);
} else if (new URL(item.link).hostname !== 'www.cs.sdu.edu.cn') {
return item;
}
lib/v2/sjtu/gs.js
+1 -1
View File
@@ -36,7 +36,7 @@ module.exports = async (ctx) => {
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
item.description = (await fetchArticle(ctx, item.link)).description;
item.description = (await fetchArticle(item.link)).description;
} else {
const detailResponse = await got({
method: 'get',
lib/v2/slowmist/slowmist.js
+1 -1
View File
@@ -39,7 +39,7 @@ module.exports = async (ctx) => {
pubDate: parseDate(item.date),
}));
items = await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
items = await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title,
lib/v2/tju/oaa/index.js
+1 -1
View File
@@ -82,7 +82,7 @@ module.exports = async (ctx) => {
list.map((item) => {
switch (item.type) {
case 'wechat-mp':
return finishArticleItem(ctx, item);
return finishArticleItem(item);
case 'tju-oaa':
case 'in-site':
return ctx.cache.tryGet(item.link, async () => {
lib/v2/wechat/data258.js
+1 -1
View File
@@ -119,7 +119,7 @@ module.exports = async (ctx) => {
throw err;
}
await Promise.all(items.map((item) => finishArticleItem(ctx, item, !!categoryPage)));
await Promise.all(items.map((item) => finishArticleItem(item, !!categoryPage)));
ctx.state.data = {
title,
lib/v2/wechat/ershcimi.js
+1 -1
View File
@@ -24,7 +24,7 @@ module.exports = async (ctx) => {
})
.get();
await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: `微信公众号 - ${$('span.name').text()}`,
lib/v2/wechat/feeddd.js
+1 -1
View File
@@ -27,7 +27,7 @@ module.exports = async (ctx) => {
guid: item.id,
}));
items = await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
items = await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: response.data.title,
lib/v2/wechat/feeds.js
+1 -1
View File
@@ -12,7 +12,7 @@ module.exports = async (ctx) => {
link: item.link,
guid: item.link,
}));
await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: feed.title,
lib/v2/wechat/mp.js
+1 -1
View File
@@ -32,7 +32,7 @@ module.exports = async (ctx) => {
link: item.link,
guid: item.link,
};
return finishArticleItem(ctx, single);
return finishArticleItem(single);
})
);
ctx.state.data = {
lib/v2/wechat/msgalbum.js
+1 -1
View File
@@ -23,7 +23,7 @@ module.exports = async (ctx) => {
link,
guid: link,
};
return finishArticleItem(ctx, single);
return finishArticleItem(single);
})
);
ctx.state.data = {
lib/v2/wechat/sogou.js
+1 -1
View File
@@ -47,7 +47,7 @@ module.exports = async (ctx) => {
guid: link,
};
await finishArticleItem(ctx, item);
await finishArticleItem(item);
ctx.state.data = {
title: `${title} 的微信公众号`,
lib/v2/wechat/tgchannel.js
+1 -1
View File
@@ -126,7 +126,7 @@ module.exports = async (ctx) => {
if (link !== undefined) {
try {
return await finishArticleItem(ctx, single);
return await finishArticleItem(single);
} catch {
single.description = item.find('.tgme_widget_message_text').html();
}
lib/v2/wechat/wechat2rss.js
+1 -1
View File
@@ -16,7 +16,7 @@ module.exports = async (ctx) => {
link: i.link,
}));
items = await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
items = await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title,