mirror of
https://github.com/DIYgod/RSSHub.git
synced 2026-10-16 07:44:44 +08:00
fix(worker): close shared cache and route compatibility gaps
This commit is contained in:
@@ -6,6 +6,8 @@ import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import timezone from '@/utils/timezone';
|
||||
|
||||
import { getForumUrl } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/:id?',
|
||||
categories: ['multimedia'],
|
||||
@@ -64,14 +66,10 @@ async function handler(ctx) {
|
||||
const id = ctx.req.param('id') ?? '3';
|
||||
|
||||
const rootUrl = 'https://hjd2048.com';
|
||||
// 获取地址发布页指向的 URL
|
||||
const domainInfo = await cache.tryGet('2048:domainInfo', async () => {
|
||||
// Resolve the address page before caching the forum URL.
|
||||
const domainInfo = await cache.tryGet('2048:domainInfo:v2', async () => {
|
||||
const response = await ofetch('https://2048.info');
|
||||
const $ = load(response);
|
||||
const onclickValue = $('.button').first().attr('onclick');
|
||||
const targetUrl = onclickValue?.match(/window\.open\('([^']+)'/)?.[1];
|
||||
|
||||
return { url: new URL(targetUrl!, 'https://2048.info').href };
|
||||
return { url: getForumUrl(response) };
|
||||
});
|
||||
// 获取重定向后的url
|
||||
const redirectResponse = await ofetch.raw(domainInfo.url);
|
||||
@@ -91,7 +89,7 @@ async function handler(ctx) {
|
||||
86400, // fixed cookie duration: 24 hours
|
||||
false
|
||||
);
|
||||
const currentUrl = `${redirected.url}thread.php?fid-${id}.html`;
|
||||
const currentUrl = new URL(`thread.php?fid-${id}.html`, redirected.url).href;
|
||||
|
||||
const response = await ofetch.raw(currentUrl, {
|
||||
headers: {
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { load } from 'cheerio';
|
||||
|
||||
export function getForumUrl(html: string) {
|
||||
const $ = load(html);
|
||||
const button = $('.button').first();
|
||||
const target = button.attr('href') || button.attr('onclick')?.match(/window\.open\(\s*(['"])(.*?)\1/)?.[2];
|
||||
if (!target) {
|
||||
throw new Error('The 2048 address page did not contain a forum link.');
|
||||
}
|
||||
const url = new URL(target, 'https://2048.info');
|
||||
if (!['https:', 'http:'].includes(url.protocol)) {
|
||||
throw new Error('The 2048 address page contained an unsupported forum URL.');
|
||||
}
|
||||
return url.href;
|
||||
}
|
||||
@@ -49,15 +49,12 @@ async function handler(ctx) {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
params: {
|
||||
title: 'AIEA Seminars',
|
||||
},
|
||||
title: 'AIEA Seminars',
|
||||
item: {
|
||||
item: `.seminar-contents .seminar-partWrap:nth-child(${nth_child}) > .seminar-list`,
|
||||
title: `$('.seminar-list-title > span').text()`,
|
||||
link: `$('a[href^="/0504"]').attr('href')`,
|
||||
description: `$('.seminar-list .txt > .title').text()`,
|
||||
title: ($) => $('.seminar-list-title > span').text(),
|
||||
link: ($) => $('a[href^="/0504"]').attr('href'),
|
||||
description: ($) => $('.seminar-list .txt > .title').text(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import buildData from '@/utils/common-config';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
const baseUrl = 'https://www.airchina.com.cn';
|
||||
|
||||
@@ -36,18 +37,17 @@ async function handler() {
|
||||
const data = await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
description: '%description%',
|
||||
params: {
|
||||
title: '国航服务公告',
|
||||
description: '中国国际航空公司服务公告',
|
||||
},
|
||||
title: '国航服务公告',
|
||||
description: '中国国际航空公司服务公告',
|
||||
item: {
|
||||
item: '.serviceMsg li',
|
||||
title: `$('a').text()`,
|
||||
link: `$('a').attr('href')`,
|
||||
pubDate: `parseDate($('span').text(), 'YYYY-MM-DD')`,
|
||||
guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
|
||||
title: ($) => $('a').text(),
|
||||
link: ($) => $('a').attr('href'),
|
||||
pubDate: ($) => parseDate($('span').text(), 'YYYY-MM-DD'),
|
||||
guid: ($) => {
|
||||
const href = $('a').attr('href');
|
||||
return href ? Buffer.from(href).toString('base64') : undefined;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -38,9 +38,9 @@ async function handler(ctx) {
|
||||
title: 'Brooklyn Museum - Exhibitions',
|
||||
item: {
|
||||
item: '.exhibitions .image-card',
|
||||
title: `$('h2 > a, h3 > a').text()`,
|
||||
link: `$('h2 > a, h3 > a').attr('href')`,
|
||||
description: `$('h6').text()`,
|
||||
title: ($) => $('h2 > a, h3 > a').text(),
|
||||
link: ($) => $('h2 > a, h3 > a').attr('href'),
|
||||
description: ($) => $('h6').text(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import type { Route } from '@/types';
|
||||
import { ViewType } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseDateInTimezone } from '@/utils/parse-date-in-timezone';
|
||||
|
||||
import { getEpsOrPeStr, getRatingChangeStr } from '../utils';
|
||||
|
||||
@@ -81,18 +81,18 @@ async function handler(ctx) {
|
||||
return {
|
||||
title: `[${item.orgSName}]${stockName}${item.title}`,
|
||||
link: `${baseUrl}/report/${linkType[category]}` + (category === 'stock' ? `/${item.infoCode}.html` : `.jshtml?encodeUrl=${item.encodeUrl}`),
|
||||
pubDate: parseDate(item.publishDate),
|
||||
pubDate: parseDateInTimezone(item.publishDate, 8),
|
||||
author: item.researcher,
|
||||
originItem: item, // temp use
|
||||
};
|
||||
});
|
||||
|
||||
const items = await Promise.all(
|
||||
list.map((item) => {
|
||||
list.map(async (item) => {
|
||||
const tempOriginItem = item.originItem;
|
||||
delete item.originItem; // temp use
|
||||
|
||||
return cache.tryGet(item.link, async () => {
|
||||
const cached = await cache.tryGet(item.link, async () => {
|
||||
try {
|
||||
const { data: response } = await got(item.link);
|
||||
const $ = load(response);
|
||||
@@ -158,6 +158,7 @@ async function handler(ctx) {
|
||||
return item;
|
||||
}
|
||||
});
|
||||
return { ...cached, pubDate: item.pubDate };
|
||||
})
|
||||
);
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Route } from '@/types';
|
||||
import buildData from '@/utils/common-config';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/zhengceku/:department',
|
||||
@@ -26,16 +27,13 @@ async function handler(ctx) {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
title: ($) => $('.channel_tab > .noline > a').text().trim() + ' - 政府文件库',
|
||||
description: '政府文件库, 当页的所有列表',
|
||||
params: {
|
||||
title: `$('.channel_tab > .noline > a').text().trim() + ' - 政府文件库'`,
|
||||
},
|
||||
item: {
|
||||
item: '.news_box > .list > ul > li:not(.line)',
|
||||
title: `$('h4 > a').text()`,
|
||||
link: `$('h4 > a').attr('href')`,
|
||||
pubDate: `parseDate($('h4 > .date').text().trim())`,
|
||||
title: ($) => $('h4 > a').text(),
|
||||
link: ($) => $('h4 > a').attr('href'),
|
||||
pubDate: ($) => parseDate($('h4 > .date').text().trim()),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { load } from 'cheerio';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseDateInTimezone } from '@/utils/parse-date-in-timezone';
|
||||
|
||||
const sorts = {
|
||||
featured: '精选',
|
||||
@@ -64,7 +64,7 @@ async function handler(ctx) {
|
||||
title: `${item.name}: ${item.title}`,
|
||||
author: item.author,
|
||||
link: `${rootUrl}/repository/${item.item_id}`,
|
||||
pubDate: parseDate(item.updated_at),
|
||||
pubDate: parseDateInTimezone(item.updated_at, 8),
|
||||
name: `${item.author}/${item.name}`,
|
||||
description: item.summary,
|
||||
language: item.primary_lang,
|
||||
|
||||
@@ -6,6 +6,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
import type { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import { parseDateInTimezone } from '@/utils/parse-date-in-timezone';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news/:category?',
|
||||
@@ -67,7 +68,8 @@ async function handler(ctx) {
|
||||
return {
|
||||
title: item.title.rendered,
|
||||
link: item.link,
|
||||
pubDate: item.date_gmt,
|
||||
// WordPress date_gmt is UTC even when its string omits the Z suffix.
|
||||
pubDate: item.date_gmt ? parseDateInTimezone(item.date_gmt, 0) : undefined,
|
||||
description,
|
||||
category: item._embedded['wp:term'][0].map((term) => term.name) ?? [],
|
||||
};
|
||||
|
||||
+11
-11
@@ -1,5 +1,6 @@
|
||||
import type { Route } from '@/types';
|
||||
import buildData from '@/utils/common-config';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
const baseUrl = 'https://www.iiilab.com/';
|
||||
|
||||
@@ -24,19 +25,18 @@ async function handler() {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
description: '%description%',
|
||||
params: {
|
||||
title: '发现',
|
||||
description: '人人都是自媒体-发现',
|
||||
},
|
||||
title: '发现',
|
||||
description: '人人都是自媒体-发现',
|
||||
item: {
|
||||
item: '.aw-common-list > div',
|
||||
title: `$('a').first().text()`,
|
||||
link: `$('a').first().attr('href')`,
|
||||
description: `$('.markitup-box').text()`,
|
||||
pubDate: `parseDate($('.text-color-999').first().text(), 'YYYY-MM-DD HH:mm')`,
|
||||
guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
|
||||
title: ($) => $('a').first().text(),
|
||||
link: ($) => $('a').first().attr('href'),
|
||||
description: ($) => $('.markitup-box').text(),
|
||||
pubDate: ($) => parseDate($('.text-color-999').first().text(), 'YYYY-MM-DD HH:mm'),
|
||||
guid: ($) => {
|
||||
const href = $('a').attr('href');
|
||||
return href ? Buffer.from(href).toString('base64') : undefined;
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import { ViewType } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import { getSubPath } from '@/utils/common-utils';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseDateInTimezone } from '@/utils/parse-date-in-timezone';
|
||||
|
||||
const toSize = (raw) => {
|
||||
const matches = raw.match(/(\d+(\.\d+)?)(\D\w*)/);
|
||||
@@ -109,14 +109,15 @@ async function handler(ctx) {
|
||||
return {
|
||||
link: $item.attr('href'),
|
||||
guid: $item.find('date').first().text(),
|
||||
pubDate: parseDate($item.find('date').last().text()),
|
||||
// This is a calendar release date, represented at UTC midnight.
|
||||
pubDate: parseDateInTimezone($item.find('date').last().text(), 0),
|
||||
title: '',
|
||||
};
|
||||
});
|
||||
|
||||
items = await Promise.all(
|
||||
items.map((item) =>
|
||||
cache.tryGet(item.link!, async () => {
|
||||
items.map(async (item) => {
|
||||
const cached = await cache.tryGet(item.link!, async () => {
|
||||
const detailResponse = await got({
|
||||
method: 'get',
|
||||
url: item.link,
|
||||
@@ -206,8 +207,9 @@ async function handler(ctx) {
|
||||
});
|
||||
|
||||
return item;
|
||||
})
|
||||
)
|
||||
});
|
||||
return { ...cached, pubDate: item.pubDate };
|
||||
})
|
||||
);
|
||||
|
||||
const title = $('head title').text();
|
||||
|
||||
@@ -28,8 +28,8 @@ async function handler() {
|
||||
title: 'Jewish Museums - Exhibitions',
|
||||
item: {
|
||||
item: '#current article.exhibition, #upcoming article, #past article.exhibition',
|
||||
title: `$('h3').text()`,
|
||||
link: `$('h3').parent().attr('href')`,
|
||||
title: ($) => $('h3').text(),
|
||||
link: ($) => $('h3').parent().attr('href'),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -38,9 +38,9 @@ async function handler(ctx) {
|
||||
title: 'New Museum - Exhibitions',
|
||||
item: {
|
||||
item: '.exh',
|
||||
title: `$('.exh .title').text()`,
|
||||
link: `$('.exh > a').attr('href')`,
|
||||
description: `$('.exh .body-reveal').text()`,
|
||||
title: ($) => $('.exh .title').text(),
|
||||
link: ($) => $('.exh > a').attr('href'),
|
||||
description: ($) => $('.exh .body-reveal').text(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -31,18 +31,14 @@ async function handler() {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
description: '%description%',
|
||||
params: {
|
||||
title: '博客',
|
||||
description: 'Oct0pu5的博客',
|
||||
},
|
||||
title: '博客',
|
||||
description: 'Oct0pu5的博客',
|
||||
item: {
|
||||
item: '.recent-posts > .recent-post-item',
|
||||
title: `$('.recent-post-info > a').text()`,
|
||||
link: `$('.recent-post-info > a').attr('href')`,
|
||||
description: `$('.recent-post-info > .content').text()`,
|
||||
pubDate: `Date.parse($('div.recent-post-info > div.article-meta-wrap > span.post-meta-date > time').text().trim())`,
|
||||
title: ($) => $('.recent-post-info > a').text(),
|
||||
link: ($) => $('.recent-post-info > a').attr('href'),
|
||||
description: ($) => $('.recent-post-info > .content').text(),
|
||||
pubDate: ($) => Date.parse($('div.recent-post-info > div.article-meta-wrap > span.post-meta-date > time').text().trim()),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { Context } from 'hono';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import buildData from '@/utils/common-config';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news/:language?',
|
||||
@@ -21,15 +22,15 @@ async function handler(ctx: Context) {
|
||||
link,
|
||||
url: link,
|
||||
title: 'Polimi News',
|
||||
params: {
|
||||
homeLink: 'https://www.polimi.it',
|
||||
},
|
||||
item: {
|
||||
item: '.card--editorial-photo',
|
||||
title: `$('.card-title').text()`,
|
||||
link: `'%homeLink%' + $('.card-footer a').attr('href')`,
|
||||
description: `$('.news-bodytext').html()`,
|
||||
pubDate: `parseDate($('time').attr('datetime'))`,
|
||||
title: ($) => $('.card-title').text(),
|
||||
link: ($) => 'https://www.polimi.it' + $('.card-footer a').attr('href'),
|
||||
description: ($) => $('.news-bodytext').html(),
|
||||
pubDate: ($) => {
|
||||
const datetime = $('time').attr('datetime');
|
||||
return datetime ? parseDate(datetime) : undefined;
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -23,17 +23,13 @@ async function handler(ctx: Context) {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
description: `$('#detail_intro .detail_body').text().trim()`,
|
||||
title: ($) => $('#book_left > #h1').text().trim() + ' - 单品活动信息 - 缺书网',
|
||||
description: ($) => $('#detail_intro .detail_body').text().trim(),
|
||||
allowEmpty: true,
|
||||
params: {
|
||||
title: `$('#book_left > #h1').text().trim() + ' - 单品活动信息 - 缺书网'`,
|
||||
host,
|
||||
},
|
||||
item: {
|
||||
item: '.stacked.right_state > a',
|
||||
title: `$('.right_item .bodycol_258').text()`,
|
||||
link: `'%host%' + $('.right_item').parent().attr('href')`,
|
||||
title: ($) => $('.right_item .bodycol_258').text(),
|
||||
link: ($) => host + $('.right_item').parent().attr('href'),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -17,16 +17,20 @@ async function handler() {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
params: {
|
||||
title: '图书促销 - 缺书网',
|
||||
host,
|
||||
},
|
||||
title: '图书促销 - 缺书网',
|
||||
item: {
|
||||
item: '#tb_sale tr',
|
||||
title: `$('.news_sale_detail .sale_btn').text() + ':' + $('.news_sale_title a').text()`,
|
||||
link: `'%host%' + $('.news_sale_title a').attr('href')`,
|
||||
description: `$('.news_sale_detail .sale_btn').text() + ':' + $('.news_sale_title a').text() + '<br>' + $('.news_sale_detail .sale_time_end').first().text() + '<br>' + '发布时间:' + $('.news_sale_detail .sale_time_end.inline_right').text()`,
|
||||
title: ($) => $('.news_sale_detail .sale_btn').text() + ':' + $('.news_sale_title a').text(),
|
||||
link: ($) => host + $('.news_sale_title a').attr('href'),
|
||||
description: ($) =>
|
||||
$('.news_sale_detail .sale_btn').text() +
|
||||
':' +
|
||||
$('.news_sale_title a').text() +
|
||||
'<br>' +
|
||||
$('.news_sale_detail .sale_time_end').first().text() +
|
||||
'<br>' +
|
||||
'发布时间:' +
|
||||
$('.news_sale_detail .sale_time_end.inline_right').text(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ async function handler(ctx) {
|
||||
return {
|
||||
item: items,
|
||||
title: `${author} - ${route.name}`,
|
||||
link: currentUrl,
|
||||
link: infoUrl,
|
||||
description: $('meta[name="description"]').prop('content'),
|
||||
language: 'zh' as const satisfies Language,
|
||||
image,
|
||||
|
||||
@@ -56,9 +56,9 @@ async function handler(ctx) {
|
||||
rootUrl,
|
||||
}),
|
||||
author: item.siteNameDisplay,
|
||||
category: [...(item.entityList.map((c) => c.name) ?? []), ...(item.tagList.map((c) => c.name) ?? [])],
|
||||
category: [...(item.entityList?.map((c) => c.name) ?? []), ...(item.tagList?.map((c) => c.name) ?? [])],
|
||||
guid: item.uid,
|
||||
pubDate: parseDate(item.publishDate),
|
||||
pubDate: item.publishDate ? parseDate(item.publishDate) : undefined,
|
||||
}));
|
||||
|
||||
items = await processItems(items);
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import dayjs from 'dayjs';
|
||||
import utc from 'dayjs/plugin/utc.js';
|
||||
import { renderToString } from 'hono/jsx/dom/server';
|
||||
|
||||
dayjs.extend(utc);
|
||||
|
||||
type NewsItem = {
|
||||
url?: string;
|
||||
title?: string;
|
||||
@@ -28,7 +31,7 @@ export const renderDescription = ({ description, news, timeline, rootUrl }: Desc
|
||||
renderToString(
|
||||
<>
|
||||
{description ? <p>{description}</p> : null}
|
||||
{news ? (
|
||||
{news?.length ? (
|
||||
<>
|
||||
<h3>媒体报道</h3>
|
||||
<table cellspacing="8">
|
||||
@@ -47,7 +50,7 @@ export const renderDescription = ({ description, news, timeline, rootUrl }: Desc
|
||||
</table>
|
||||
</>
|
||||
) : null}
|
||||
{timeline ? (
|
||||
{timeline?.topics?.length ? (
|
||||
<>
|
||||
<h3>事件追踪</h3>
|
||||
<table cellspacing="10">
|
||||
@@ -55,7 +58,7 @@ export const renderDescription = ({ description, news, timeline, rootUrl }: Desc
|
||||
{timeline.topics?.map((topic, index) => (
|
||||
<tr key={String(topic.uid ?? topic.title ?? index)}>
|
||||
<th align="left">
|
||||
<small>{topic.publishDate ? dayjs(topic.publishDate).format('YYYY-MM-DD HH:mm:ss') : ''}</small>
|
||||
<small>{topic.publishDate ? dayjs(topic.publishDate).utcOffset(8).format('YYYY-MM-DD HH:mm:ss') : ''}</small>
|
||||
</th>
|
||||
<td>
|
||||
<a href={topic.uid ? new URL(`topic/${topic.uid}`, rootUrl).href : undefined}>{topic.title}</a>
|
||||
|
||||
+69
-34
@@ -1,3 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import type { DataItem } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
@@ -8,45 +11,77 @@ const domain = 'readhub.cn';
|
||||
const rootUrl = `https://${domain}`;
|
||||
const apiRootUrl = `https://api.${domain}`;
|
||||
const apiTopicUrl = new URL('topic/list', apiRootUrl).href;
|
||||
const apiDetailUrl = new URL('topic/detail', apiRootUrl).href;
|
||||
const apiTimelineUrl = new URL('topic/timeline/list', apiRootUrl).href;
|
||||
|
||||
/**
|
||||
* Process items asynchronously.
|
||||
*
|
||||
* @param {Array<Object>} items - The array of items to process.
|
||||
* @returns {Promise<Array<Object>>} Returns a Promise that resolves to an array of processed items.
|
||||
*/
|
||||
const processItems = async (items) =>
|
||||
await Promise.all(
|
||||
items.map((item) =>
|
||||
cache.tryGet(item.link, async () => {
|
||||
try {
|
||||
if (!item.link.startsWith(rootUrl)) {
|
||||
throw new Error(`"${item.link}" is an external URL`);
|
||||
}
|
||||
const namedEntry = z.object({ name: z.string() });
|
||||
const newsSchema = z.object({ url: z.url(), title: z.string(), siteNameDisplay: z.string().nullish() });
|
||||
const topicSchema = z.object({
|
||||
uid: z.string().min(1),
|
||||
title: z.string().min(1),
|
||||
summary: z.string().min(1),
|
||||
url: z.url().nullish(),
|
||||
publishDate: z.iso.datetime({ offset: true }),
|
||||
siteNameDisplay: z.string().nullish(),
|
||||
newsAggList: z.array(newsSchema),
|
||||
entityList: z.array(namedEntry).nullish(),
|
||||
tagList: z.array(namedEntry).nullish(),
|
||||
});
|
||||
const timelineEntrySchema = z.object({ uid: z.string().min(1), title: z.string(), publishDate: z.iso.datetime({ offset: true }).nullish() });
|
||||
const timelineDataSchema = z.object({ items: z.array(timelineEntrySchema) });
|
||||
const topicDataSchema = z.object({ items: z.array(topicSchema) });
|
||||
const timelineSchema = z.object({ data: timelineDataSchema });
|
||||
const detailSchema = z.object({ data: topicDataSchema });
|
||||
|
||||
const { data: detailResponse } = await got(item.link);
|
||||
type ReadhubItem = DataItem & { link: string; guid: string };
|
||||
|
||||
const data = JSON.parse(detailResponse.match(/\{\\"topic\\":(.*?)\}\]\\n"\]\)<\/script>/)[1].replaceAll(String.raw`\"`, '"'));
|
||||
async function fetchTopic(uid: string): Promise<ReadhubItem> {
|
||||
const { data: detailResponse } = await got(apiDetailUrl, { searchParams: { uid } });
|
||||
const detail = detailSchema.safeParse(detailResponse);
|
||||
if (!detail.success) {
|
||||
throw new Error('Readhub topic detail response is invalid or missing publication dates');
|
||||
}
|
||||
const matches = detail.data.data.items.filter((topic) => topic.uid === uid);
|
||||
if (matches.length !== 1) {
|
||||
throw new Error('Readhub topic detail response does not contain the requested topic');
|
||||
}
|
||||
const topic = matches[0];
|
||||
|
||||
item.title = data.title;
|
||||
item.link = data.url ?? new URL(`topic/${data.uid}`, rootUrl).href;
|
||||
item.description = renderDescription({
|
||||
description: data.summary,
|
||||
news: data.newsAggList,
|
||||
timeline: data.timeline,
|
||||
rootUrl,
|
||||
});
|
||||
item.author = data.siteNameDisplay;
|
||||
item.category = [...(data.entityList.map((c) => c.name) ?? []), ...(data.tagList.map((c) => c.name) ?? [])];
|
||||
item.guid = `readhub-${data.uid}`;
|
||||
item.pubDate = parseDate(data.publishDate.replaceAll(/\s/g, ''));
|
||||
} catch {
|
||||
item.guid = `readhub-${item.guid}`;
|
||||
}
|
||||
const { data: timelineResponse } = await got(apiTimelineUrl, { searchParams: { topic_uid: uid, size: 10 } });
|
||||
const timeline = timelineSchema.safeParse(timelineResponse);
|
||||
if (!timeline.success) {
|
||||
throw new Error('Readhub topic timeline response is invalid');
|
||||
}
|
||||
|
||||
return item;
|
||||
})
|
||||
)
|
||||
return {
|
||||
title: topic.title,
|
||||
link: topic.url ?? new URL(`topic/${topic.uid}`, rootUrl).href,
|
||||
description: renderDescription({
|
||||
description: topic.summary,
|
||||
news: topic.newsAggList.map((news) => ({ ...news, siteNameDisplay: news.siteNameDisplay ?? undefined })),
|
||||
timeline: { topics: timeline.data.data.items.map((entry) => ({ ...entry, publishDate: entry.publishDate ?? undefined })) },
|
||||
rootUrl,
|
||||
}),
|
||||
author: topic.siteNameDisplay ?? undefined,
|
||||
category: [...(topic.entityList ?? []), ...(topic.tagList ?? [])].map((entry) => entry.name),
|
||||
guid: `readhub-${topic.uid}`,
|
||||
pubDate: parseDate(topic.publishDate),
|
||||
};
|
||||
}
|
||||
|
||||
const processItems = (items: ReadhubItem[]): Promise<ReadhubItem[]> =>
|
||||
Promise.all(
|
||||
items.map((item) => {
|
||||
const url = new URL(item.link);
|
||||
const topicPath = /^\/topic\/([^/]+)\/?$/.exec(url.pathname);
|
||||
if (url.origin !== rootUrl || !topicPath) {
|
||||
return { ...item, guid: `readhub-${item.guid}` };
|
||||
}
|
||||
|
||||
// Keep failed enrichment out of the cache, including old summary-only entries.
|
||||
const uid = decodeURIComponent(topicPath[1]);
|
||||
return cache.tryGet(`readhub:topic:v2:${uid}`, () => fetchTopic(uid));
|
||||
})
|
||||
);
|
||||
|
||||
export { apiRootUrl, apiTopicUrl, processItems, rootUrl };
|
||||
|
||||
@@ -5,10 +5,8 @@ import ConfigNotFoundError from '@/errors/types/config-not-found';
|
||||
import type { Route } from '@/types';
|
||||
import { ViewType } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import timezone from '@/utils/timezone';
|
||||
|
||||
import { getHeaders } from './utils';
|
||||
import { getHeaders, parseSearchDate } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/keyword/:keyword',
|
||||
@@ -63,14 +61,15 @@ async function handler(ctx) {
|
||||
return {
|
||||
title: `${keyword} - 什么值得买`,
|
||||
link: `https://search.smzdm.com/?c=home&s=${encodeURIComponent(keyword)}&order=time`,
|
||||
item:
|
||||
list &&
|
||||
list.toArray().map((item) => {
|
||||
item: list
|
||||
.toArray()
|
||||
.filter((item) => $(item).find('.feed-block-title a').first().attr('href'))
|
||||
.map((item) => {
|
||||
const $item = $(item);
|
||||
return {
|
||||
title: `${$item.find('.feed-block-title a').eq(0).text().trim()} - ${$item.find('.feed-block-title a').eq(1).text().trim()}`,
|
||||
description: `${$item.find('.feed-block-descripe').contents().eq(2).text().trim()}<br>${$item.find('.feed-block-extras span').text().trim()}<br><img src="http:${$item.find('.z-feed-img img').attr('src')}">`,
|
||||
pubDate: timezone(parseDate($item.find('.feed-block-extras').contents().eq(0).text().trim(), ['MM-DD HH:mm', 'HH:mm']), 8),
|
||||
pubDate: parseSearchDate($item.find('.feed-block-extras').contents().eq(0).text().trim()),
|
||||
link: $item.find('.feed-block-title a').attr('href'),
|
||||
};
|
||||
}),
|
||||
|
||||
@@ -1,4 +1,17 @@
|
||||
import { config } from '@/config';
|
||||
import { parseDateInTimezone } from '@/utils/parse-date-in-timezone';
|
||||
|
||||
export const parseSearchDate = (value: string, now = new Date()): Date | undefined => {
|
||||
const today = new Date(now.getTime() + 8 * 60 * 60 * 1000).toISOString().slice(0, 10);
|
||||
if (/^\d{2}:\d{2}$/.test(value)) {
|
||||
return parseDateInTimezone(`${today}T${value}`, 8);
|
||||
}
|
||||
if (/^\d{2}-\d{2} \d{2}:\d{2}$/.test(value)) {
|
||||
// Search results omit the year; a later month/day belongs to last year.
|
||||
const year = Number(today.slice(0, 4)) - Number(value.slice(0, 5) > today.slice(5));
|
||||
return parseDateInTimezone(`${year}-${value}`, 8);
|
||||
}
|
||||
};
|
||||
|
||||
export const getHeaders = () => ({
|
||||
accept: 'application/json, text/javascript, */*; q=0.01',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Route } from '@/types';
|
||||
import buildData from '@/utils/common-config';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/',
|
||||
@@ -21,10 +22,13 @@ async function handler() {
|
||||
description: 'Stratechery provides analysis of the strategy and business side of technology and media, and the impact of technology on society. ',
|
||||
item: {
|
||||
item: 'article',
|
||||
title: `$('article > header > h1 > a').text()`,
|
||||
link: `$('article > header > h1 > a').attr('href')`,
|
||||
pubDate: `parseDate($('article .entry-date').attr('datetime'))`,
|
||||
description: `$('article > .entry-content').html().replace(/%/g, '%')`,
|
||||
title: ($) => $('article > header > h1 > a').text(),
|
||||
link: ($) => $('article > header > h1 > a').attr('href'),
|
||||
pubDate: ($) => {
|
||||
const datetime = $('article .entry-date').attr('datetime');
|
||||
return datetime ? parseDate(datetime) : undefined;
|
||||
},
|
||||
description: ($) => $('article > .entry-content').html()!.replaceAll('%', '%'),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -8,8 +8,30 @@ import { baseUrl, gqlFeatures, gqlMap, initGqlMap } from './constants';
|
||||
import type { ApiParams } from './utils';
|
||||
import { gatherLegacyFromData, paginationTweets, twitterGot } from './utils';
|
||||
|
||||
const getUserData = (id) =>
|
||||
cache.tryGet(`twitter-userdata-${id}`, () => {
|
||||
const getUserResult = (response: any) => {
|
||||
const errors = response?.errors;
|
||||
if (errors !== undefined && errors !== null && (!Array.isArray(errors) || errors.length > 0)) {
|
||||
const codes = Array.isArray(errors) ? [...new Set(errors.map((error) => error?.code).filter((code) => Number.isSafeInteger(code)))] : [];
|
||||
throw new Error(`Twitter API user lookup failed${codes.length ? ` (codes: ${codes.join(', ')})` : ''}`);
|
||||
}
|
||||
|
||||
const user = response?.data?.user;
|
||||
if (user === null || user?.result === null) {
|
||||
throw new InvalidParameterError("This account doesn't exist");
|
||||
}
|
||||
const result = user?.result;
|
||||
if (result?.__typename === 'UserUnavailable') {
|
||||
throw new InvalidParameterError('Twitter user is unavailable');
|
||||
}
|
||||
if (!result || typeof result.rest_id !== 'string' || result.rest_id.length === 0) {
|
||||
throw new Error('Twitter API returned an incomplete user response');
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const getUserData = async (id: string) => {
|
||||
// Other Twitter API adapters cache different schemas under the legacy key.
|
||||
const data = await cache.tryGet(`twitter:web:userdata:v2:${id}`, async () => {
|
||||
const params = {
|
||||
variables: id.startsWith('+')
|
||||
? JSON.stringify({
|
||||
@@ -26,31 +48,28 @@ const getUserData = (id) =>
|
||||
}),
|
||||
};
|
||||
|
||||
if (config.twitter.thirdPartyApi) {
|
||||
const endpoint = id.startsWith('+') ? gqlMap.UserByRestId : gqlMap.UserByScreenName;
|
||||
const response = config.twitter.thirdPartyApi
|
||||
? await ofetch(`${config.twitter.thirdPartyApi}${id.startsWith('+') ? gqlMap.UserByRestId : gqlMap.UserByScreenName}`, {
|
||||
method: 'GET',
|
||||
params,
|
||||
headers: {
|
||||
'accept-encoding': 'gzip',
|
||||
},
|
||||
})
|
||||
: await twitterGot(`${baseUrl}${id.startsWith('+') ? gqlMap.UserByRestId : gqlMap.UserByScreenName}`, params, {
|
||||
allowNoAuth: !id.startsWith('+'),
|
||||
});
|
||||
|
||||
return ofetch(`${config.twitter.thirdPartyApi}${endpoint}`, {
|
||||
method: 'GET',
|
||||
params,
|
||||
headers: {
|
||||
'accept-encoding': 'gzip',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return twitterGot(`${baseUrl}${id.startsWith('+') ? gqlMap.UserByRestId : gqlMap.UserByScreenName}`, params, {
|
||||
allowNoAuth: !id.startsWith('+'),
|
||||
});
|
||||
// HTTP 200 can still contain GraphQL errors; never cache those responses.
|
||||
getUserResult(response);
|
||||
return response;
|
||||
});
|
||||
return getUserResult(data);
|
||||
};
|
||||
|
||||
const cacheTryGet = async (_id, params, operationName, func) => {
|
||||
const userData: any = await getUserData(_id);
|
||||
const id = userData.data?.user?.result?.rest_id;
|
||||
if (id === undefined) {
|
||||
cache.set(`twitter-userdata-${_id}`, '', config.cache.contentExpire);
|
||||
throw new InvalidParameterError('User not found');
|
||||
}
|
||||
return cache.tryGet(getTwitterUserCacheKey(id, operationName, params), () => func(id, params), config.cache.routeExpire, false);
|
||||
const user = await getUserData(_id);
|
||||
return cache.tryGet(getTwitterUserCacheKey(user.rest_id, operationName, params), () => func(user.rest_id, params), config.cache.routeExpire, false);
|
||||
};
|
||||
|
||||
const getUserTweets = (id: string, params?: ApiParams) =>
|
||||
@@ -164,19 +183,11 @@ const getList = async (id: string, params?: ApiParams) =>
|
||||
);
|
||||
|
||||
const getUser = async (id: string) => {
|
||||
const userData: any = await getUserData(id);
|
||||
|
||||
if (!userData.data.user) {
|
||||
throw new InvalidParameterError("This account doesn't exist");
|
||||
}
|
||||
if (userData.data.user.result.__typename === 'UserUnavailable') {
|
||||
throw new InvalidParameterError(userData.data.user.result.message || 'User is unavailable');
|
||||
}
|
||||
|
||||
const user = await getUserData(id);
|
||||
return {
|
||||
profile_image_url: userData.data?.user?.result?.avatar?.image_url,
|
||||
description: userData.data?.user?.result?.profile_bio?.description,
|
||||
...userData.data?.user?.result?.core,
|
||||
profile_image_url: user.avatar?.image_url,
|
||||
description: user.profile_bio?.description,
|
||||
...user.core,
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -34,13 +34,23 @@ async function handler(ctx) {
|
||||
await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: `$('.name-main').text().trim() + ' - 用户 - 绿洲'`,
|
||||
description: `$('.desc').text().trim()`,
|
||||
title: ($) => $('.name-main').text().trim() + ' - 用户 - 绿洲',
|
||||
description: ($) => $('.desc').text().trim(),
|
||||
item: {
|
||||
item: '.container .status-item',
|
||||
title: `$('.status-item-title').contents().filter((_, node) => node.type === 'text').text()`,
|
||||
description: `$('.status-item-title').contents().filter((_, node) => node.type === 'text').text() + '<br>' + $('.status-img').html()`,
|
||||
link: `'https://oasis.weibo.cn/v1/h5/share?sid=' + $('.status-item-title').parent().data('id')`,
|
||||
title: ($) =>
|
||||
$('.status-item-title')
|
||||
.contents()
|
||||
.filter((_, node) => node.type === 'text')
|
||||
.text(),
|
||||
description: ($) =>
|
||||
$('.status-item-title')
|
||||
.contents()
|
||||
.filter((_, node) => node.type === 'text')
|
||||
.text() +
|
||||
'<br>' +
|
||||
$('.status-img').html(),
|
||||
link: ($) => 'https://oasis.weibo.cn/v1/h5/share?sid=' + $('.status-item-title').parent().data('id'),
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
@@ -44,14 +44,16 @@ async function handler(ctx) {
|
||||
response = await ofetch(link);
|
||||
|
||||
$ = load(response);
|
||||
const nextDataElement = $('#__NEXT_DATA__').get(0);
|
||||
page_data = JSON.parse(nextDataElement.children[0].data);
|
||||
|
||||
// If no episodes found, we should try episode URL
|
||||
if (!page_data.props.pageProps.podcast?.episodes) {
|
||||
throw new Error('No episodes found in podcast data');
|
||||
page_data = JSON.parse($('#__NEXT_DATA__').text());
|
||||
} catch (error) {
|
||||
// An episode ID may return 404 at the podcast URL. Preserve access and
|
||||
// transport errors instead of hiding them behind a second failed request.
|
||||
if (!(error instanceof Error) || !('status' in error) || error.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
|
||||
if (!page_data?.props?.pageProps?.podcast?.episodes) {
|
||||
// Try as episode instead
|
||||
link = `https://www.xiaoyuzhoufm.com/episode/${id}`;
|
||||
response = await ofetch(link);
|
||||
@@ -65,11 +67,14 @@ async function handler(ctx) {
|
||||
response = await ofetch(link);
|
||||
|
||||
$ = load(response);
|
||||
const nextDataElement = $('#__NEXT_DATA__').get(0);
|
||||
page_data = JSON.parse(nextDataElement.children[0].data);
|
||||
page_data = JSON.parse($('#__NEXT_DATA__').text());
|
||||
}
|
||||
}
|
||||
|
||||
if (!page_data?.props?.pageProps?.podcast?.episodes) {
|
||||
throw new Error('Xiaoyuzhou did not return podcast data for this podcast or episode ID.');
|
||||
}
|
||||
|
||||
let episodes = page_data.props.pageProps.podcast.episodes.map((item) => ({
|
||||
title: item.title,
|
||||
enclosure_url: item.enclosure.url,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Route } from '@/types';
|
||||
import buildData from '@/utils/common-config';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
const baseUrl = 'https://www.xunhupay.com';
|
||||
|
||||
@@ -32,19 +33,18 @@ async function handler() {
|
||||
return await buildData({
|
||||
link,
|
||||
url: link,
|
||||
title: '%title%',
|
||||
description: '%description%',
|
||||
params: {
|
||||
title: '博客',
|
||||
description: '虎皮椒-博客',
|
||||
},
|
||||
title: '博客',
|
||||
description: '虎皮椒-博客',
|
||||
item: {
|
||||
item: '.blog-post > article',
|
||||
title: `$('h5').text()`,
|
||||
link: `$('a').attr('href')`,
|
||||
description: `$('.content').text()`,
|
||||
pubDate: `parseDate($('.date').text(), 'YYYY-MM-DD')`,
|
||||
guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
|
||||
title: ($) => $('h5').text(),
|
||||
link: ($) => $('a').attr('href'),
|
||||
description: ($) => $('.content').text(),
|
||||
pubDate: ($) => parseDate($('.date').text(), 'YYYY-MM-DD'),
|
||||
guid: ($) => {
|
||||
const href = $('a').attr('href');
|
||||
return href ? Buffer.from(href).toString('base64') : undefined;
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
// oxlint-disable unicorn/prefer-math-trunc unicorn-js/no-this-outside-of-class
|
||||
// xxhash-wasm shim for Cloudflare Workers
|
||||
// Uses Web Crypto API instead of WebAssembly
|
||||
|
||||
type XXHash<T> = {
|
||||
update(input: string | Uint8Array): XXHash<T>;
|
||||
digest(): T;
|
||||
};
|
||||
|
||||
type XXHashAPI = {
|
||||
h32(input: string, seed?: number): number;
|
||||
h32ToString(input: string, seed?: number): string;
|
||||
h32Raw(inputBuffer: Uint8Array, seed?: number): number;
|
||||
create32(seed?: number): XXHash<number>;
|
||||
h64(input: string, seed?: bigint): bigint;
|
||||
h64ToString(input: string, seed?: bigint): string;
|
||||
h64Raw(inputBuffer: Uint8Array, seed?: bigint): bigint;
|
||||
create64(seed?: bigint): XXHash<bigint>;
|
||||
};
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
// Simple sync hash for h32 methods (fallback)
|
||||
const simpleHash32 = (input: Uint8Array, seed = 0): number => {
|
||||
let hash = seed;
|
||||
for (const byte of input) {
|
||||
hash = Math.trunc((hash << 5) - hash + byte);
|
||||
}
|
||||
return hash >>> 0;
|
||||
};
|
||||
|
||||
function xxhash(): XXHashAPI {
|
||||
return {
|
||||
h32: (input: string, seed?: number): number => simpleHash32(encoder.encode(input), seed),
|
||||
h32ToString: (input: string, seed?: number): string => simpleHash32(encoder.encode(input), seed).toString(16).padStart(8, '0'),
|
||||
h32Raw: (inputBuffer: Uint8Array, seed?: number): number => simpleHash32(inputBuffer, seed),
|
||||
create32: (seed?: number): XXHash<number> => {
|
||||
const chunks: Uint8Array[] = [];
|
||||
return {
|
||||
update(input: string | Uint8Array) {
|
||||
chunks.push(input instanceof Uint8Array ? input : encoder.encode(input));
|
||||
return this;
|
||||
},
|
||||
digest() {
|
||||
const totalLength = chunks.reduce((sum, arr) => sum + arr.length, 0);
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return simpleHash32(combined, seed);
|
||||
},
|
||||
};
|
||||
},
|
||||
// h64 methods use async SHA-256 but return sync - this is a limitation
|
||||
// In practice, only h64ToString is used and it's called with await xxhash()
|
||||
h64: (_input: string, _seed?: bigint): bigint => {
|
||||
throw new Error('h64 is not supported in Worker shim, use h64ToString instead');
|
||||
},
|
||||
h64ToString: (input: string, _seed?: bigint): string => {
|
||||
// This needs to be sync to match the API, but we use a simple hash
|
||||
// The actual usage in cache.ts awaits xxhash() first, so this works
|
||||
let hash = 0n;
|
||||
const data = encoder.encode(input);
|
||||
for (const byte of data) {
|
||||
hash = ((hash << 5n) - hash + BigInt(byte)) & 0xff_ff_ff_ff_ff_ff_ff_ffn;
|
||||
}
|
||||
return hash.toString(16).padStart(16, '0');
|
||||
},
|
||||
h64Raw: (_inputBuffer: Uint8Array, _seed?: bigint): bigint => {
|
||||
throw new Error('h64Raw is not supported in Worker shim');
|
||||
},
|
||||
create64: (_seed?: bigint): XXHash<bigint> => {
|
||||
throw new Error('create64 is not supported in Worker shim');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export default xxhash;
|
||||
Vendored
+14
@@ -117,4 +117,18 @@ describe('http cache module', () => {
|
||||
expect(cache.status.available).toBe(true);
|
||||
expect(urls[0]).not.toContain('?refresh=1');
|
||||
});
|
||||
|
||||
it('reports Cloudflare routing errors instead of treating them as cache misses', async () => {
|
||||
setHttpCacheEnv();
|
||||
const { errorSpy } = await spyOnLogger();
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => new Response('error code: 1042', { status: 404 }))
|
||||
);
|
||||
const cache = (await import('@/utils/cache/http')).default;
|
||||
cache.init();
|
||||
|
||||
await expect(cache.get('missing')).resolves.toBeNull();
|
||||
expect(errorSpy).toHaveBeenCalledWith('HTTP cache get returned HTTP 404 without a cache miss response.');
|
||||
});
|
||||
});
|
||||
|
||||
Vendored
+10
-1
@@ -106,7 +106,16 @@ export default {
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
return '';
|
||||
try {
|
||||
const data = await response.json();
|
||||
if (data && typeof data === 'object' && 'hit' in data && data.hit === false) {
|
||||
return '';
|
||||
}
|
||||
} catch {
|
||||
// Cloudflare routing errors can also return 404, without reaching the cache service.
|
||||
}
|
||||
logger.error('HTTP cache get returned HTTP 404 without a cache miss response.');
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
|
||||
+43
-110
@@ -2,128 +2,61 @@ import iconv from 'iconv-lite';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import configUtils, { getProp, replaceParams, transElemText } from '@/utils/common-config';
|
||||
import buildData from '@/utils/common-config';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import timezone from '@/utils/timezone';
|
||||
|
||||
describe('index', () => {
|
||||
it('transElemText', () => {
|
||||
const $ = () => 'RSSHub';
|
||||
expect(transElemText($, '$()')).toBe('RSSHub');
|
||||
});
|
||||
const url = 'http://rsshub.test/buildData';
|
||||
|
||||
it('replaceParams', () => {
|
||||
const $ = () => 'RSSHub';
|
||||
const data = {
|
||||
params: {
|
||||
title: 'RSSHub',
|
||||
},
|
||||
title: '%title%',
|
||||
};
|
||||
expect(replaceParams(data, data.title, $)).toBe('RSSHub');
|
||||
});
|
||||
|
||||
it('getProp', () => {
|
||||
const $ = () => 'RSSHub';
|
||||
const data = {
|
||||
title: 'RSSHub',
|
||||
};
|
||||
expect(getProp(data, ['title'], $)).toBe('RSSHub');
|
||||
expect(getProp(data, 'title', $)).toBe('RSSHub');
|
||||
});
|
||||
|
||||
it('all', () => {
|
||||
const $ = () => 'RSSHub';
|
||||
const data = {
|
||||
params: {
|
||||
title: '$()',
|
||||
},
|
||||
title: '%title%',
|
||||
};
|
||||
expect(getProp(data, ['title'], $)).toBe('RSSHub');
|
||||
});
|
||||
|
||||
it('buildData', async () => {
|
||||
const data = await configUtils({
|
||||
link: 'http://rsshub.test/buildData',
|
||||
url: 'http://rsshub.test/buildData',
|
||||
title: '%title%',
|
||||
params: {
|
||||
title: 'buildData',
|
||||
},
|
||||
describe('common-config', () => {
|
||||
it('extracts page and item values with callbacks and preserves literal text', async () => {
|
||||
const suffix = 'buildData';
|
||||
const literal = `$('script').text() %one% + %two%`;
|
||||
const data = await buildData({
|
||||
link: url,
|
||||
url,
|
||||
title: ($) => $('.content li a').first().text() + ' - ' + suffix,
|
||||
description: literal,
|
||||
item: {
|
||||
item: '.content li',
|
||||
title: `$('a').text() + ' - %title%'`,
|
||||
link: `$('a').attr('href')`,
|
||||
description: `$('.description').html()`,
|
||||
pubDate: `timezone(parseDate($('.date').text(), 'YYYY-MM-DD'), 0)`,
|
||||
title: ($) => $('a').text() + ' - ' + suffix,
|
||||
link: ($) => $('a').attr('href'),
|
||||
description: ($) => $('.description').html(),
|
||||
pubDate: ($) => timezone(parseDate($('.date').text(), 'YYYY-MM-DD'), 0),
|
||||
},
|
||||
});
|
||||
|
||||
expect(data).toMatchObject({
|
||||
link: 'http://rsshub.test/buildData',
|
||||
title: 'buildData',
|
||||
expect(data).toEqual({
|
||||
link: url,
|
||||
title: '1 - buildData',
|
||||
description: literal,
|
||||
allowEmpty: false,
|
||||
item: [
|
||||
{
|
||||
description: 'RSSHub1',
|
||||
guid: undefined,
|
||||
link: '/1',
|
||||
pubDate: new Date('2025-01-01T00:00:00Z'),
|
||||
title: '1 - buildData',
|
||||
},
|
||||
{
|
||||
description: 'RSSHub2',
|
||||
guid: undefined,
|
||||
link: '/2',
|
||||
pubDate: new Date('2025-01-02T00:00:00Z'),
|
||||
title: '2 - buildData',
|
||||
},
|
||||
{ title: '1 - buildData', link: '/1', description: 'RSSHub1', pubDate: new Date('2025-01-01T00:00:00Z'), guid: undefined },
|
||||
{ title: '2 - buildData', link: '/2', description: 'RSSHub2', pubDate: new Date('2025-01-02T00:00:00Z'), guid: undefined },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('charset', () => {
|
||||
const html = `<div class="content">
|
||||
<ul>
|
||||
<li>
|
||||
<a href="/1">1</a>
|
||||
<div class="description">中文RSSHub1</div>
|
||||
<div class="date">2025-01-01</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>`;
|
||||
|
||||
it('parses charset from content-type', async () => {
|
||||
const { default: server } = await import('@/setup.test');
|
||||
server.use(
|
||||
http.get(
|
||||
'http://rsshub.test/buildData-gbk',
|
||||
() =>
|
||||
new HttpResponse(iconv.encode(html, 'gbk'), {
|
||||
headers: {
|
||||
'content-type': 'text/html; charset=gbk',
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
const data = await configUtils({
|
||||
link: 'http://rsshub.test/buildData-gbk',
|
||||
url: 'http://rsshub.test/buildData-gbk',
|
||||
title: '%title%',
|
||||
params: {
|
||||
title: 'buildData',
|
||||
},
|
||||
item: {
|
||||
item: '.content li',
|
||||
title: `$('a').text() + ' - %title%'`,
|
||||
link: `$('a').attr('href')`,
|
||||
description: `$('.description').html()`,
|
||||
pubDate: `timezone(parseDate($('.date').text(), 'YYYY-MM-DD'), 0)`,
|
||||
},
|
||||
it('preserves constants, numeric dates, missing fields and allowEmpty', async () => {
|
||||
const date = 1_735_689_600_000;
|
||||
const data = await buildData({
|
||||
url,
|
||||
title: 'Feed',
|
||||
allowEmpty: true,
|
||||
item: { item: '.content li', title: 'Item', description: undefined, pubDate: date },
|
||||
});
|
||||
expect(data.allowEmpty).toBe(true);
|
||||
expect(data.item).toEqual(Array.from({ length: 2 }, () => ({ title: 'Item', description: undefined, pubDate: date, link: undefined, guid: undefined })));
|
||||
const empty = await buildData({ url, title: 'Empty', item: { item: '.missing', title: 'Item' } });
|
||||
expect(empty.item).toEqual([]);
|
||||
expect(empty.allowEmpty).toBe(false);
|
||||
});
|
||||
|
||||
expect(data.title).toBe('buildData');
|
||||
expect(data.item[0].title).toBe('1 - buildData');
|
||||
expect(data.item[0].description).toBe('中文RSSHub1');
|
||||
it('decodes non-UTF-8 responses before calling selectors', async () => {
|
||||
const { default: server } = await import('@/setup.test');
|
||||
server.use(http.get('http://rsshub.test/buildData-gbk', () => new HttpResponse(iconv.encode('<h1>中文标题</h1><ul><li><a href="/1">中文条目</a></li></ul>', 'gbk'), { headers: { 'content-type': 'text/html; charset=gbk' } })));
|
||||
const data = await buildData({ url: 'http://rsshub.test/buildData-gbk', title: ($) => $('h1').text(), item: { item: 'li', title: ($) => $('a').text(), link: ($) => $('a').attr('href') } });
|
||||
expect(data.title).toBe('中文标题');
|
||||
expect(data.item[0]).toMatchObject({ title: '中文条目', link: '/1' });
|
||||
});
|
||||
});
|
||||
|
||||
+32
-57
@@ -1,52 +1,29 @@
|
||||
import { load } from 'cheerio';
|
||||
import { type Cheerio, load } from 'cheerio';
|
||||
import type { AnyNode } from 'domhandler';
|
||||
import iconv from 'iconv-lite';
|
||||
|
||||
import type { Data, DataItem } from '@/types';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate as _parseDate } from '@/utils/parse-date';
|
||||
import _timezone from '@/utils/timezone';
|
||||
|
||||
function transElemText($, prop) {
|
||||
const regex = /\$\(.*\)/;
|
||||
let result = prop;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const parseDate = _parseDate;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const timezone = _timezone;
|
||||
if (regex.test(result)) {
|
||||
// eslint-disable-next-line no-eval
|
||||
result = eval(result);
|
||||
}
|
||||
return result;
|
||||
type Selector = (selector: string) => Cheerio<AnyNode>;
|
||||
type Value<T> = T | ((select: Selector) => T);
|
||||
type ItemFields = Pick<DataItem, 'title' | 'description' | 'pubDate' | 'link' | 'guid'>;
|
||||
|
||||
type BuildDataConfig = Pick<Data, 'link' | 'author' | 'allowEmpty'> & {
|
||||
url: string;
|
||||
title: Value<Data['title']>;
|
||||
description?: Value<Data['description']>;
|
||||
item: { item: string } & { [Key in keyof ItemFields]: Value<ItemFields[Key]> };
|
||||
};
|
||||
|
||||
function resolveValue<T>(value: Value<T>, select: Selector): T {
|
||||
return typeof value === 'function' ? (value as (select: Selector) => T)(select) : value;
|
||||
}
|
||||
|
||||
function replaceParams(data, prop, $) {
|
||||
const regex = /%(.*)%/g;
|
||||
let result = prop;
|
||||
let group = regex.exec(prop);
|
||||
while (group) {
|
||||
// FIXME Multi vars
|
||||
result = result.replace(group[0], () => transElemText($, data.params[group![1]]));
|
||||
group = regex.exec(prop);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function getProp(data, prop, $) {
|
||||
let result = data;
|
||||
if (Array.isArray(prop)) {
|
||||
for (const e of prop) {
|
||||
result = transElemText($, result[e]);
|
||||
}
|
||||
} else {
|
||||
result = transElemText($, result[prop]);
|
||||
}
|
||||
return replaceParams(data, result, $);
|
||||
}
|
||||
|
||||
export default async function buildData(data) {
|
||||
export default async function buildData(data: BuildDataConfig) {
|
||||
const response = await ofetch.raw(data.url);
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
// 若没有指定编码,则默认utf-8
|
||||
// Default to UTF-8 when no encoding is specified.
|
||||
let charset = 'utf-8';
|
||||
for (const attr of contentType.split(';')) {
|
||||
if (attr.includes('charset=')) {
|
||||
@@ -55,24 +32,22 @@ export default async function buildData(data) {
|
||||
}
|
||||
const responseData = charset === 'utf-8' ? response._data : iconv.decode(Buffer.from(await ofetch(data.url, { responseType: 'arrayBuffer' })), charset);
|
||||
const $ = load(responseData);
|
||||
const $item = $(data.item.item);
|
||||
// 这里应该是可以通过参数注入一些代码的,不过应该无伤大雅
|
||||
return {
|
||||
link: data.link,
|
||||
title: getProp(data, 'title', $),
|
||||
description: getProp(data, 'description', $),
|
||||
title: resolveValue(data.title, $),
|
||||
description: resolveValue(data.description, $),
|
||||
allowEmpty: data.allowEmpty || false,
|
||||
item: $item.toArray().map((e) => {
|
||||
const $elem = (selector) => $(e).find(selector);
|
||||
return {
|
||||
title: getProp(data, ['item', 'title'], $elem),
|
||||
description: getProp(data, ['item', 'description'], $elem),
|
||||
pubDate: getProp(data, ['item', 'pubDate'], $elem),
|
||||
link: getProp(data, ['item', 'link'], $elem),
|
||||
guid: getProp(data, ['item', 'guid'], $elem),
|
||||
};
|
||||
}),
|
||||
item: $(data.item.item)
|
||||
.toArray()
|
||||
.map((element) => {
|
||||
const select: Selector = (selector) => $(element).find(selector);
|
||||
return {
|
||||
title: resolveValue(data.item.title, select),
|
||||
description: resolveValue(data.item.description, select),
|
||||
pubDate: resolveValue(data.item.pubDate, select),
|
||||
link: resolveValue(data.item.link, select),
|
||||
guid: resolveValue(data.item.guid, select),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export { getProp, replaceParams, transElemText };
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { parseDateInTimezone } from './parse-date-in-timezone';
|
||||
|
||||
describe('parseDateInTimezone', () => {
|
||||
it.each([
|
||||
['2026-08-28T08:07:22', 8, '2026-08-28T00:07:22.000Z'],
|
||||
['2026-09-07 00:00:00.000', 8, '2026-09-06T16:00:00.000Z'],
|
||||
['2026-09-06T15:41:43', 0, '2026-09-06T15:41:43.000Z'],
|
||||
['2026-09-18', 0, '2026-09-18T00:00:00.000Z'],
|
||||
['2026-09-07T00:00:00', -3.5, '2026-09-07T03:30:00.000Z'],
|
||||
['2026-09-07T08:00:00+08:00', 8, '2026-09-07T00:00:00.000Z'],
|
||||
['2026-09-07T08:00:00+0800', 0, '2026-09-07T00:00:00.000Z'],
|
||||
['2026-09-07T08:00:00-05:00', 8, '2026-09-07T13:00:00.000Z'],
|
||||
['2026-09-07T00:00:00Z', 8, '2026-09-07T00:00:00.000Z'],
|
||||
['Mon, 07 Sep 2026 08:00:00 GMT+8', 8, '2026-09-07T00:00:00.000Z'],
|
||||
])('parses %s in offset %s independently of the host timezone', (input, offset, expected) => {
|
||||
expect(parseDateInTimezone(input, offset).toISOString()).toBe(expected);
|
||||
});
|
||||
|
||||
it('preserves absolute Date and millisecond timestamp inputs without mutating them', () => {
|
||||
const date = new Date('2026-09-07T00:00:00Z');
|
||||
expect(parseDateInTimezone(date, 8).getTime()).toBe(date.getTime());
|
||||
expect(parseDateInTimezone(date.getTime(), -5).getTime()).toBe(date.getTime());
|
||||
expect(date.toISOString()).toBe('2026-09-07T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('does not invent a date for an empty or unparseable string', () => {
|
||||
expect(Number.isNaN(parseDateInTimezone('', 8).getTime())).toBe(true);
|
||||
expect(Number.isNaN(parseDateInTimezone('unknown', 8).getTime())).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
import dayjs from 'dayjs';
|
||||
import utc from 'dayjs/plugin/utc';
|
||||
|
||||
dayjs.extend(utc);
|
||||
|
||||
// Interpret a source's wall clock without using the host's timezone. Dates,
|
||||
// timestamps and ISO strings with an explicit offset already identify an instant.
|
||||
export const parseDateInTimezone = (date: string | number | Date, offset: number): Date => {
|
||||
const parsed = dayjs.utc(date);
|
||||
if (typeof date !== 'string' || /(?:Z|[+-]\d{2}:?\d{2}|(?:GMT|UTC)(?:[+-]\d{1,4})?)$/i.test(date.trim())) {
|
||||
return parsed.toDate();
|
||||
}
|
||||
return parsed.subtract(offset, 'hour').toDate();
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { getForumUrl } from '../lib/routes/2048/utils';
|
||||
|
||||
describe('2048 forum address discovery', () => {
|
||||
it('follows the current address page anchor', () => {
|
||||
expect(getForumUrl('<a class="button" href="/bbs.php" target="_blank">Forum</a>')).toBe('https://2048.info/bbs.php');
|
||||
});
|
||||
|
||||
it('accepts the previous onclick format', () => {
|
||||
expect(getForumUrl('<button class="button" onclick="window.open(\'https://forum.example/\')">Forum</button>')).toBe('https://forum.example/');
|
||||
});
|
||||
|
||||
it('rejects missing links before they can be cached as /undefined', () => {
|
||||
expect(() => getForumUrl('<html>Unavailable</html>')).toThrow('did not contain a forum link');
|
||||
});
|
||||
|
||||
it('rejects script URLs', () => {
|
||||
expect(() => getForumUrl('<a class="button" href="javascript:void(0)">Forum</a>')).toThrow('unsupported forum URL');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { Context } from 'hono';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { Data, Route } from '../lib/types';
|
||||
|
||||
const mocks = vi.hoisted(() => ({ raw: vi.fn(), get: vi.fn() }));
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: Object.assign(vi.fn(), { raw: mocks.raw }) }));
|
||||
vi.mock('../lib/utils/got', () => ({ default: mocks.get }));
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: (_key: string, load: () => unknown) => load() } }));
|
||||
vi.mock('../lib/routes/weibo/utils', () => ({ default: { sinaimgTvax: (data: Data) => data } }));
|
||||
|
||||
const fixtures = [
|
||||
{
|
||||
name: 'government policy library',
|
||||
module: () => import('../lib/routes/gov/zhengce/zhengceku'),
|
||||
params: { department: 'bmwj' },
|
||||
html: '<div class="channel_tab"><div class="noline"><a>部门文件</a></div></div><div class="news_box"><div class="list"><ul><li><h4><a href="/policy">政策正文</a><span class="date">2025-01-01</span></h4></li><li class="line">separator</li></ul></div></div>',
|
||||
title: '部门文件 - 政府文件库',
|
||||
item: { title: '政策正文', link: '/policy', pubDate: new Date(2025, 0, 1) },
|
||||
},
|
||||
{
|
||||
name: 'Polimi',
|
||||
module: () => import('../lib/routes/polimi/news'),
|
||||
params: { language: 'en' },
|
||||
html: '<article class="card--editorial-photo"><div class="card-title">News</div><div class="card-footer"><a href="/story">Read</a></div><div class="news-bodytext"><b>Body</b></div><time datetime="2025-01-01T00:00:00Z"></time></article>',
|
||||
title: 'Polimi News',
|
||||
item: { title: 'News', link: 'https://www.polimi.it/story', description: '<b>Body</b>', pubDate: new Date('2025-01-01T00:00:00Z') },
|
||||
},
|
||||
{
|
||||
name: 'Jewish Museum',
|
||||
module: () => import('../lib/routes/jewishmuseum/exhibitions'),
|
||||
html: '<section id="current"><article class="exhibition"><a href="/show"><h3>Exhibition</h3></a></article></section>',
|
||||
title: 'Jewish Museums - Exhibitions',
|
||||
item: { title: 'Exhibition', link: '/show' },
|
||||
},
|
||||
{
|
||||
name: 'Brooklyn Museum',
|
||||
module: () => import('../lib/routes/brooklynmuseum/exhibitions'),
|
||||
html: '<div class="exhibitions"><article class="image-card"><h2><a href="/show">Exhibition</a></h2><h6>Description</h6></article></div>',
|
||||
title: 'Brooklyn Museum - Exhibitions',
|
||||
item: { title: 'Exhibition', link: '/show', description: 'Description' },
|
||||
},
|
||||
{
|
||||
name: 'New Museum',
|
||||
module: () => import('../lib/routes/newmuseum/exhibitions'),
|
||||
html: '<div class="exh"><a href="/show"><span class="title">Exhibition</span></a><div class="body-reveal">Description</div></div>',
|
||||
title: 'New Museum - Exhibitions',
|
||||
item: { title: 'Exhibition', link: '/show', description: 'Description' },
|
||||
},
|
||||
{
|
||||
name: 'Stratechery',
|
||||
module: () => import('../lib/routes/stratechery/index'),
|
||||
html: '<article><header><h1><a href="/post">Post</a></h1></header><time class="entry-date" datetime="2025-01-01T00:00:00Z"></time><div class="entry-content"><p>100% growth</p></div></article>',
|
||||
title: 'Stratechery by Ben Thompson',
|
||||
item: { title: 'Post', link: '/post', description: '<p>100% growth</p>', pubDate: new Date('2025-01-01T00:00:00Z') },
|
||||
},
|
||||
{
|
||||
name: 'Queshu book',
|
||||
module: () => import('../lib/routes/queshu/book'),
|
||||
params: { bookid: '123' },
|
||||
html: '<div id="book_left"><h1 id="h1">Book</h1></div><div id="detail_intro"><div class="detail_body">Description</div></div><div class="stacked right_state"><a href="/sale"><div class="right_item"><div class="bodycol_258">Offer</div></div></a></div>',
|
||||
title: 'Book - 单品活动信息 - 缺书网',
|
||||
item: { title: 'Offer', link: 'http://www.queshu.com/sale' },
|
||||
},
|
||||
{
|
||||
name: 'Queshu sale',
|
||||
module: () => import('../lib/routes/queshu/sale'),
|
||||
html: '<table id="tb_sale"><tr><td class="news_sale_title"><a href="/book">Book</a></td><td class="news_sale_detail"><span class="sale_btn">Offer</span><span class="sale_time_end">Ends today</span><span class="sale_time_end inline_right">Yesterday</span></td></tr></table>',
|
||||
title: '图书促销 - 缺书网',
|
||||
item: { title: 'Offer:Book', link: 'http://www.queshu.com/book', description: 'Offer:Book<br>Ends today<br>发布时间:Yesterday' },
|
||||
},
|
||||
{
|
||||
name: 'AIEA',
|
||||
module: () => import('../lib/routes/aiea/index'),
|
||||
params: { period: 'upcoming' },
|
||||
html: '<div class="seminar-contents"><div class="seminar-partWrap"><div class="seminar-list"><div class="seminar-list-title"><span>Seminar</span></div><a href="/0504/1">Details</a><div class="txt"><span class="title">Description</span></div></div></div></div>',
|
||||
title: 'AIEA Seminars',
|
||||
item: { title: 'Seminar', link: '/0504/1', description: 'Description' },
|
||||
},
|
||||
{
|
||||
name: 'Oasis',
|
||||
module: () => import('../lib/routes/weibo/oasis/user'),
|
||||
params: { userid: '123' },
|
||||
html: '<div class="name-main">Author</div><div class="desc">Profile</div><div class="container"><div class="status-item"><div data-id="456"><div class="status-item-title">Post<span>Ignored child</span></div></div><div class="status-img"><img src="/image.jpg"></div></div></div>',
|
||||
title: 'Author - 用户 - 绿洲',
|
||||
item: { title: 'Post', link: 'https://oasis.weibo.cn/v1/h5/share?sid=456', description: 'Post<br><img src="/image.jpg">' },
|
||||
},
|
||||
{
|
||||
name: 'Oct0pu5',
|
||||
module: () => import('../lib/routes/oct0pu5/rss'),
|
||||
html: '<div class="recent-posts"><article class="recent-post-item"><div class="recent-post-info"><a href="/post">Post</a><div class="content">Description</div><div class="article-meta-wrap"><span class="post-meta-date"><time>2025-01-01T00:00:00Z</time></span></div></div></article></div>',
|
||||
title: '博客',
|
||||
item: { title: 'Post', link: '/post', description: 'Description', pubDate: 1_735_689_600_000 },
|
||||
},
|
||||
{
|
||||
name: 'Air China',
|
||||
module: () => import('../lib/routes/airchina/index'),
|
||||
html: '<ul class="serviceMsg"><li><a href="/1">First</a><span>2025-01-01</span></li><li><a href="/2">Second</a><span>2025-01-02</span></li></ul>',
|
||||
title: '国航服务公告',
|
||||
item: { title: 'First', link: '/1', pubDate: new Date(2025, 0, 1), description: 'Article detail' },
|
||||
uniqueGuid: true,
|
||||
},
|
||||
{
|
||||
name: 'Xunhupay',
|
||||
module: () => import('../lib/routes/xunhupay/index'),
|
||||
html: '<div class="blog-post"><article><h5>First</h5><a href="/1"></a><div class="content">Description</div><div class="date">2025-01-01</div></article><article><h5>Second</h5><a href="/2"></a><div class="date">2025-01-02</div></article></div>',
|
||||
title: '博客',
|
||||
item: { title: 'First', link: '/1', pubDate: new Date(2025, 0, 1), description: 'Description' },
|
||||
uniqueGuid: true,
|
||||
},
|
||||
{
|
||||
name: 'iiilab',
|
||||
module: () => import('../lib/routes/iiilab/index'),
|
||||
html: '<div class="aw-common-list"><div><a href="/1">First</a><div class="markitup-box">Description</div><div class="text-color-999">2025-01-01 00:00</div></div><div><a href="/2">Second</a><div class="text-color-999">2025-01-02 00:00</div></div></div>',
|
||||
title: '发现',
|
||||
item: { title: 'First', link: '/1', pubDate: new Date(2025, 0, 1), description: 'Description' },
|
||||
uniqueGuid: true,
|
||||
},
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.get.mockResolvedValue({ data: '<div class="serviceMsg">Article detail</div>' });
|
||||
});
|
||||
|
||||
describe('common-config route callbacks', () => {
|
||||
it.each(fixtures)('$name extracts its HTML fixture without evaluating selector strings', async (fixture) => {
|
||||
mocks.raw.mockResolvedValue({ headers: new Headers({ 'content-type': 'text/html' }), _data: fixture.html });
|
||||
const { route } = await fixture.module();
|
||||
const params: Record<string, string | undefined> = fixture.params ?? {};
|
||||
const ctx = { req: { param: (key?: string) => (key ? params[key] : params), query: () => {} } } as unknown as Context;
|
||||
const data = (await (route.handler as Exclude<Route['handler'], string>)(ctx)) as Data;
|
||||
expect(mocks.raw).toHaveBeenCalledTimes(1);
|
||||
expect(data.title).toBe(fixture.title);
|
||||
expect(data.item).toHaveLength(fixture.uniqueGuid ? 2 : 1);
|
||||
expect(data.item![0]).toMatchObject(fixture.item);
|
||||
if (fixture.uniqueGuid) {
|
||||
expect(data.item!.map((item) => item.guid)).toEqual(['LzE=', 'LzI=']);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest';
|
||||
|
||||
import { route } from '../lib/routes/gov/zhengce/zhengceku';
|
||||
import type { Data, Route } from '../lib/types';
|
||||
import buildData from '../lib/utils/common-config';
|
||||
|
||||
// These unused network/devtools exports are tree-shaken in the production bundle.
|
||||
vi.mock('undici', () => ({}));
|
||||
vi.mock('node-network-devtools', () => ({ register: vi.fn() }));
|
||||
vi.mock('../lib/config', () => ({ config: { requestRetry: 0, loggerLevel: 'error' } }));
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
it('extracts the policy library in workerd with page/item callbacks and dates', async () => {
|
||||
const fetchMock = vi.fn(() =>
|
||||
Promise.resolve(
|
||||
new Response(
|
||||
'<div class="channel_tab"><div class="noline"><a>部门文件</a></div></div><div class="news_box"><div class="list"><ul><li><h4><a href="/policy">政策</a><span class="date">2025-01-01</span></h4></li></ul></div></div>',
|
||||
{ headers: { 'content-type': 'text/html' } }
|
||||
)
|
||||
)
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
const data = (await (route.handler as Exclude<Route['handler'], string>)({ req: { param: () => 'bmwj' } } as never)) as Data;
|
||||
expect(data.title).toBe('部门文件 - 政府文件库');
|
||||
expect(data.item).toEqual([{ title: '政策', link: '/policy', pubDate: new Date('2025-01-01T00:00:00Z'), description: undefined, guid: undefined }]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('treats JavaScript-looking text and percent signs as literal data in workerd', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(new Response('<li><a>100% growth %remaining%</a></li>', { headers: { 'content-type': 'text/html' } })))
|
||||
);
|
||||
const title = `$('a').text(); globalThis.shouldNotRun = true`;
|
||||
const result = await buildData({ url: 'https://example.test/', title, item: { item: 'li', title: ($) => $('a').text() } });
|
||||
expect(result.title).toBe(title);
|
||||
expect(result.item[0].title).toBe('100% growth %remaining%');
|
||||
expect(Reflect.has(globalThis, 'shouldNotRun')).toBe(false);
|
||||
});
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"data": {
|
||||
"self": { "title": "相关事件", "topicTitle": "Topic" },
|
||||
"items": [
|
||||
{ "uid": "earlier-1", "title": "Earlier event", "publishDate": "2026-09-05T02:03:04.000Z" },
|
||||
{ "uid": "earlier-2", "title": "Another event", "publishDate": "2026-09-04T15:04:05+08:00" }
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"data": {
|
||||
"items": [
|
||||
{
|
||||
"uid": "topic-42",
|
||||
"title": "Topic \"quoted\" & 中文",
|
||||
"summary": "First paragraph.\nA quoted \"statement\", path C:\\topic, and <markup>.",
|
||||
"createdAt": "2026-09-06T08:00:00.000Z",
|
||||
"publishDate": "2026-09-06T09:40:34.084Z",
|
||||
"siteCount": 2,
|
||||
"siteNameDisplay": "Source A",
|
||||
"newsAggList": [
|
||||
{ "uid": "news-1", "url": "https://example.com/report-1", "title": "Report A", "siteNameDisplay": "Source A" },
|
||||
{ "uid": "news-2", "url": "https://example.com/report-2", "title": "Report B", "siteNameDisplay": "Source B" }
|
||||
],
|
||||
"entityList": [{ "name": "Entity" }],
|
||||
"tagList": [{ "name": "Technology" }],
|
||||
"itemId": 42
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { route as daily } from '../lib/routes/readhub/daily';
|
||||
import { processItems } from '../lib/routes/readhub/util';
|
||||
import cache from '../lib/utils/cache';
|
||||
import got from '../lib/utils/got';
|
||||
import timelineFixture from './fixtures/readhub/timeline.json';
|
||||
import topicFixture from './fixtures/readhub/topic.json';
|
||||
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: vi.fn() } }));
|
||||
vi.mock('../lib/utils/got', () => ({ default: vi.fn() }));
|
||||
|
||||
// Response shapes from public Readhub detail/timeline APIs on 2026-09-07.
|
||||
// Values are synthetic; escaped text and explicit offsets exercise preservation.
|
||||
const detailUrl = 'https://api.readhub.cn/topic/detail';
|
||||
const timelineUrl = 'https://api.readhub.cn/topic/timeline/list';
|
||||
const link = 'https://readhub.cn/topic/topic-42';
|
||||
const input = { title: 'Daily summary', link, guid: 'topic-42', description: '<p>Incomplete summary</p>' };
|
||||
const cached = new Map<string, unknown>();
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
cached.clear();
|
||||
vi.mocked(cache.tryGet).mockImplementation(async (key, fetchValue) => {
|
||||
if (cached.has(key)) {
|
||||
return cached.get(key);
|
||||
}
|
||||
const value = await fetchValue();
|
||||
cached.set(key, value);
|
||||
return value;
|
||||
});
|
||||
vi.mocked(got).mockImplementation((url) => {
|
||||
if (url === detailUrl) {
|
||||
return Promise.resolve({ data: structuredClone(topicFixture) });
|
||||
}
|
||||
if (url === timelineUrl) {
|
||||
return Promise.resolve({ data: structuredClone(timelineFixture) });
|
||||
}
|
||||
throw new Error('Unexpected upstream request');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Readhub API enrichment', () => {
|
||||
it('preserves the full summary, media, timeline, tags, title, GUID, and upstream publication date', async () => {
|
||||
const [item] = await processItems([input]);
|
||||
|
||||
expect(item).toMatchObject({
|
||||
title: topicFixture.data.items[0].title,
|
||||
link,
|
||||
author: 'Source A',
|
||||
category: ['Entity', 'Technology'],
|
||||
guid: 'readhub-topic-42',
|
||||
pubDate: new Date('2026-09-06T09:40:34.084Z'),
|
||||
});
|
||||
expect(item.description).toContain('First paragraph.\nA quoted "statement", path C:\\topic, and <markup>.');
|
||||
expect(item.description).toContain('<h3>媒体报道</h3>');
|
||||
expect(item.description).toContain('<a href="https://example.com/report-1">Report A</a>');
|
||||
expect(item.description).toContain('Source B');
|
||||
expect(item.description).toContain('<h3>事件追踪</h3>');
|
||||
expect(item.description).toContain('<a href="https://readhub.cn/topic/earlier-1">Earlier event</a>');
|
||||
expect(item.description).toContain('2026-09-05 10:03:04');
|
||||
expect(item.description).toContain('2026-09-04 15:04:05');
|
||||
expect(got).toHaveBeenNthCalledWith(1, detailUrl, { searchParams: { uid: 'topic-42' } });
|
||||
expect(got).toHaveBeenNthCalledWith(2, timelineUrl, { searchParams: { topic_uid: 'topic-42', size: 10 } });
|
||||
expect(input.guid).toBe('topic-42');
|
||||
});
|
||||
|
||||
it('does not reuse or extend old cached summary-only items', async () => {
|
||||
cached.set(link, { ...input, guid: 'readhub-topic-42' });
|
||||
const [first] = await processItems([input]);
|
||||
const [second] = await processItems([{ ...input, title: 'Another route title' }]);
|
||||
|
||||
expect(first).toEqual(second);
|
||||
expect(first.pubDate).toBeInstanceOf(Date);
|
||||
expect(cache.tryGet).toHaveBeenCalledWith('readhub:topic:v2:topic-42', expect.any(Function));
|
||||
expect(got).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('selects the requested UID instead of another item in the response', async () => {
|
||||
const response = structuredClone(topicFixture);
|
||||
response.data.items.unshift({ ...response.data.items[0], uid: 'unrelated', title: 'Unrelated' });
|
||||
vi.mocked(got).mockResolvedValueOnce({ data: response });
|
||||
expect((await processItems([input]))[0].title).toBe(topicFixture.data.items[0].title);
|
||||
});
|
||||
|
||||
it.each(['missing', 'duplicate'])('rejects a %s requested topic without caching the summary', async (kind) => {
|
||||
const response = structuredClone(topicFixture);
|
||||
if (kind === 'missing') {
|
||||
response.data.items[0].uid = 'unrelated';
|
||||
} else {
|
||||
response.data.items.push(response.data.items[0]);
|
||||
}
|
||||
vi.mocked(got).mockResolvedValueOnce({ data: response });
|
||||
await expect(processItems([input])).rejects.toThrow('does not contain the requested topic');
|
||||
expect(cached.size).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ data: { items: [{ ...topicFixture.data.items[0], publishDate: undefined }] } },
|
||||
{ data: { items: [{ ...topicFixture.data.items[0], publishDate: 'not a date' }] } },
|
||||
{ data: { items: [{ ...topicFixture.data.items[0], summary: '' }] } },
|
||||
{ data: { items: [{ ...topicFixture.data.items[0], newsAggList: undefined }] } },
|
||||
'<html>Unexpected upstream page</html>',
|
||||
])('rejects invalid detail data instead of returning an incomplete item', async (response) => {
|
||||
vi.mocked(got).mockResolvedValueOnce({ data: response });
|
||||
await expect(processItems([input])).rejects.toThrow('Readhub topic detail response is invalid');
|
||||
expect(cached.size).toBe(0);
|
||||
});
|
||||
|
||||
it('retries after a failed enrichment rather than caching the fallback', async () => {
|
||||
vi.mocked(got).mockRejectedValueOnce(new Error('Upstream unavailable'));
|
||||
await expect(processItems([input])).rejects.toThrow('Upstream unavailable');
|
||||
expect(cached.size).toBe(0);
|
||||
expect((await processItems([input]))[0].pubDate).toEqual(new Date('2026-09-06T09:40:34.084Z'));
|
||||
});
|
||||
|
||||
it('rejects a malformed timeline instead of dropping related events', async () => {
|
||||
vi.mocked(got)
|
||||
.mockResolvedValueOnce({ data: topicFixture })
|
||||
.mockResolvedValueOnce({ data: { data: {} } });
|
||||
await expect(processItems([input])).rejects.toThrow('Readhub topic timeline response is invalid');
|
||||
expect(cached.size).toBe(0);
|
||||
});
|
||||
|
||||
it('accepts an explicitly empty timeline and optional tags without inventing related items', async () => {
|
||||
const response = { data: { items: [{ ...topicFixture.data.items[0], entityList: undefined, tagList: null }] } };
|
||||
vi.mocked(got)
|
||||
.mockResolvedValueOnce({ data: response })
|
||||
.mockResolvedValueOnce({ data: { data: { items: [] } } });
|
||||
const [item] = await processItems([input]);
|
||||
expect(item.category).toEqual([]);
|
||||
expect(item.description).toContain('媒体报道');
|
||||
expect(item.description).not.toContain('事件追踪');
|
||||
expect(item.pubDate).toEqual(new Date('2026-09-06T09:40:34.084Z'));
|
||||
});
|
||||
|
||||
it.each(['https://example.com/article', 'https://readhub.cn.example.com/topic/topic-42'])('preserves external category items without fetching their pages: %s', async (externalLink) => {
|
||||
const external = { ...input, link: externalLink, pubDate: new Date('2026-09-01T00:00:00Z') };
|
||||
expect(await processItems([external])).toEqual([{ ...external, guid: 'readhub-topic-42' }]);
|
||||
expect(got).not.toHaveBeenCalled();
|
||||
expect(cache.tryGet).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('enriches daily summaries that contain no date and keeps the public feed URL', async () => {
|
||||
const upstreamMock = vi.mocked(got).getMockImplementation()!;
|
||||
vi.mocked(got).mockImplementation((url, options) => {
|
||||
if (url === 'https://api.readhub.cn/daily') {
|
||||
return Promise.resolve({ data: { data: { items: [{ uid: 'topic-42', title: input.title, summary: 'Daily summary' }] } } });
|
||||
}
|
||||
if (url === 'https://readhub.cn/daily') {
|
||||
return Promise.resolve({ data: '<meta name="application-name" content="Readhub"><link rel="apple-touch-icon" href="/icon.png">' });
|
||||
}
|
||||
return upstreamMock(url, options);
|
||||
});
|
||||
const feed = await daily.handler({ req: { query: vi.fn() } } as never);
|
||||
|
||||
expect(feed.link).toBe('https://readhub.cn/daily');
|
||||
expect(feed.item).toHaveLength(1);
|
||||
expect(feed.item![0].pubDate).toEqual(new Date('2026-09-06T09:40:34.084Z'));
|
||||
expect(feed.item![0].description).toContain('Report A');
|
||||
expect(feed.item![0].description).toContain('Earlier event');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
// Run the same enrichment and failure cases in workerd.
|
||||
import './readhub.test';
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { Context } from 'hono';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { route as eastmoney } from '../lib/routes/eastmoney/report';
|
||||
import { route as hellogithub } from '../lib/routes/hellogithub';
|
||||
import { route as iCable } from '../lib/routes/i-cable/news';
|
||||
import { route as javbus } from '../lib/routes/javbus';
|
||||
import { route as smzdm } from '../lib/routes/smzdm/keyword';
|
||||
import { parseSearchDate } from '../lib/routes/smzdm/utils';
|
||||
import type { Data, Route } from '../lib/types';
|
||||
|
||||
const mocks = vi.hoisted(() => ({ request: vi.fn(), tryGet: vi.fn() }));
|
||||
vi.mock('../lib/utils/got', () => ({ default: mocks.request }));
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: mocks.tryGet } }));
|
||||
vi.mock('../lib/config', () => ({ config: { smzdm: { cookie: 'test-only-cookie' }, feature: { allow_user_supply_unsafe_domain: false }, cache: { routeExpire: 900 } } }));
|
||||
|
||||
const invoke = async (route: Route, path: string, params: Record<string, string> = {}) =>
|
||||
(await route.handler({ req: { path, query: vi.fn(), param: (name?: string) => (name ? params[name] : params) } } as unknown as Context)) as Data;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mocks.request.mockRejectedValue(new Error('Unexpected upstream request'));
|
||||
mocks.tryGet.mockImplementation(async (_key, callback) => await callback());
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('External network access is disabled')));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('explicit source dates', () => {
|
||||
it('interprets HelloGitHub wall clock dates as UTC+8 and preserves explicit offsets', async () => {
|
||||
mocks.request.mockResolvedValueOnce({
|
||||
data: { data: ['2026-08-28T08:07:22', '2026-08-28T00:07:22Z', '2026-08-28T08:07:22+08:00'].map((updated_at, index) => ({ item_id: String(index), name: 'Project', title: 'Summary', author: 'Author', updated_at })) },
|
||||
});
|
||||
const result = await invoke(hellogithub, '/hellogithub/home');
|
||||
expect(result.item.map((item) => (item.pubDate as Date).toISOString())).toEqual(Array.from({ length: 3 }, () => '2026-08-28T00:07:22.000Z'));
|
||||
});
|
||||
|
||||
it('uses the WordPress GMT field as UTC and leaves absent dates absent', async () => {
|
||||
mocks.tryGet.mockResolvedValueOnce({ data: [{ id: 1, name: 'News', link: 'https://example.com/news' }] });
|
||||
mocks.request.mockResolvedValueOnce({
|
||||
data: ['2026-09-06T15:41:43', '2026-09-06T23:41:43+08:00', undefined].map((date_gmt) => ({
|
||||
title: { rendered: 'Title' },
|
||||
link: 'https://example.com/article',
|
||||
date_gmt,
|
||||
content: { rendered: '<p>Article</p>' },
|
||||
_embedded: { 'wp:term': [[]] },
|
||||
})),
|
||||
});
|
||||
const result = await invoke(iCable, '/i-cable/news');
|
||||
expect(result.item.slice(0, 2).map((item) => (item.pubDate as Date).toISOString())).toEqual(Array.from({ length: 2 }, () => '2026-09-06T15:41:43.000Z'));
|
||||
expect(result.item[2].pubDate).toBeUndefined();
|
||||
});
|
||||
|
||||
it('corrects Eastmoney list dates even when full article entries were cached with the old host timezone', async () => {
|
||||
const cached = { title: 'Cached report', link: 'https://example.com/report.pdf', description: '<p>Cached body</p>', pubDate: '2026-09-07T00:00:00.000Z' };
|
||||
mocks.request.mockResolvedValueOnce({
|
||||
data: `<script>var initdata=${JSON.stringify({ data: [{ publishDate: '2026-09-07 00:00:00.000', orgSName: 'Org', title: 'Report', encodeUrl: 'id', researcher: 'Author' }] })};</script>`,
|
||||
});
|
||||
mocks.tryGet.mockResolvedValueOnce(cached);
|
||||
const result = await invoke(eastmoney, '/eastmoney/report/industry', { category: 'industry' });
|
||||
expect(result.item[0]).toMatchObject({ description: cached.description, link: cached.link, pubDate: new Date('2026-09-06T16:00:00Z') });
|
||||
expect(cached.pubDate).toBe('2026-09-07T00:00:00.000Z');
|
||||
expect(mocks.request).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves the Javbus calendar release date at UTC midnight on cache hits', async () => {
|
||||
const cached = { title: 'Cached release', link: 'https://www.javbus.com/release', description: '<p>Cached body</p>', pubDate: '2026-09-17T16:00:00.000Z' };
|
||||
mocks.request.mockResolvedValueOnce({ data: '<title>Releases</title><a class="movie-box" href="https://www.javbus.com/release"><date>release-id</date><date>2026-09-18</date></a>' });
|
||||
mocks.tryGet.mockResolvedValueOnce(cached);
|
||||
const result = await invoke(javbus, '/javbus/star/example');
|
||||
expect(result.item[0]).toMatchObject({ description: cached.description, pubDate: new Date('2026-09-18T00:00:00Z') });
|
||||
expect(cached.pubDate).toBe('2026-09-17T16:00:00.000Z');
|
||||
expect(mocks.request).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('skips SMZDM layout rows without article links and resolves time-only dates in the source calendar day', async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-06T18:10:00Z'));
|
||||
mocks.request.mockResolvedValueOnce({
|
||||
data: `
|
||||
<div class="feed-row-wide"><div class="feed-block-extras">00:40</div></div>
|
||||
<div class="feed-row-wide"><div class="feed-block-title"><a href="https://example.com/deal">Deal</a><a>Price</a></div>
|
||||
<div class="feed-block-extras">00:40<span>Source</span></div><div class="z-feed-img"><img src="//example.com/image.jpg"></div></div>`,
|
||||
});
|
||||
const result = await invoke(smzdm, '/smzdm/keyword/example', { keyword: 'example' });
|
||||
expect(result.item).toHaveLength(1);
|
||||
expect(result.item[0]).toMatchObject({ link: 'https://example.com/deal', title: 'Deal - Price', pubDate: new Date('2026-09-06T16:40:00Z') });
|
||||
});
|
||||
|
||||
it('resolves SMZDM month/day values at the source year boundary', () => {
|
||||
const now = new Date('2025-12-31T18:10:00Z');
|
||||
expect(parseSearchDate('00:40', now)?.toISOString()).toBe('2025-12-31T16:40:00.000Z');
|
||||
expect(parseSearchDate('12-31 23:40', now)?.toISOString()).toBe('2025-12-31T15:40:00.000Z');
|
||||
expect(parseSearchDate('01-01 01:40', now)?.toISOString()).toBe('2025-12-31T17:40:00.000Z');
|
||||
});
|
||||
|
||||
it('leaves unknown SMZDM dates absent instead of inventing the current time', () => {
|
||||
expect(parseSearchDate('')).toBeUndefined();
|
||||
expect(parseSearchDate('unknown')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,151 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { config } from '../lib/config';
|
||||
import InvalidParameterError from '../lib/errors/types/invalid-parameter';
|
||||
import api from '../lib/routes/twitter/api/web-api/api';
|
||||
import { gatherLegacyFromData, paginationTweets, twitterGot } from '../lib/routes/twitter/api/web-api/utils';
|
||||
import cache from '../lib/utils/cache';
|
||||
import ofetch from '../lib/utils/ofetch';
|
||||
|
||||
vi.mock('../lib/config', () => ({ config: { twitter: {}, cache: { routeExpire: 300 } } }));
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: vi.fn(), set: vi.fn() } }));
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: vi.fn() }));
|
||||
vi.mock('../lib/routes/twitter/utils', () => ({ getTwitterUserCacheKey: vi.fn(() => 'tweet-cache') }));
|
||||
vi.mock('../lib/routes/twitter/api/web-api/constants', () => ({
|
||||
baseUrl: 'https://x.com/i/api',
|
||||
gqlMap: { UserByScreenName: '/graphql/user-by-name', UserByRestId: '/graphql/user-by-id' },
|
||||
gqlFeatures: { UserByScreenName: {}, UserByRestId: {} },
|
||||
initGqlMap: vi.fn(),
|
||||
}));
|
||||
vi.mock('../lib/routes/twitter/api/web-api/utils', () => ({ twitterGot: vi.fn(), paginationTweets: vi.fn(), gatherLegacyFromData: vi.fn() }));
|
||||
|
||||
const cached = new Map<string, unknown>();
|
||||
const user = {
|
||||
data: {
|
||||
user: {
|
||||
result: {
|
||||
__typename: 'User',
|
||||
rest_id: '123',
|
||||
core: { name: 'Example', screen_name: 'example' },
|
||||
avatar: { image_url: 'https://example.com/avatar.png' },
|
||||
profile_bio: { description: 'Example bio' },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
cached.clear();
|
||||
config.twitter.thirdPartyApi = undefined;
|
||||
vi.mocked(cache.tryGet).mockImplementation(async (key, fetchValue) => {
|
||||
if (cached.has(key)) {
|
||||
return cached.get(key);
|
||||
}
|
||||
const value = await fetchValue();
|
||||
cached.set(key, value);
|
||||
return value;
|
||||
});
|
||||
vi.mocked(twitterGot).mockResolvedValue(structuredClone(user));
|
||||
vi.mocked(ofetch).mockResolvedValue(structuredClone(user));
|
||||
});
|
||||
|
||||
describe('Twitter web user lookup', () => {
|
||||
it('preserves valid user metadata and caches only the verified web response', async () => {
|
||||
cached.set('twitter-userdata-example', { errors: [{ code: 88, message: 'Legacy cached failure' }] });
|
||||
expect(await api.getUser('example')).toEqual({
|
||||
name: 'Example',
|
||||
screen_name: 'example',
|
||||
profile_image_url: 'https://example.com/avatar.png',
|
||||
description: 'Example bio',
|
||||
});
|
||||
await api.getUser('example');
|
||||
expect(twitterGot).toHaveBeenCalledTimes(1);
|
||||
expect(cache.tryGet).toHaveBeenCalledWith('twitter:web:userdata:v2:example', expect.any(Function));
|
||||
expect(cached.get('twitter-userdata-example')).toEqual({ errors: [{ code: 88, message: 'Legacy cached failure' }] });
|
||||
expect(cache.set).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[{ errors: [{ code: 88, message: 'Private upstream details' }] }, 'Twitter API user lookup failed (codes: 88)'],
|
||||
[{ ...user, errors: [{ code: 89 }] }, 'Twitter API user lookup failed (codes: 89)'],
|
||||
[{ errors: [{ message: 'Private upstream details' }] }, 'Twitter API user lookup failed'],
|
||||
[{ errors: { message: 'Malformed errors' } }, 'Twitter API user lookup failed'],
|
||||
[{ data: {} }, 'Twitter API returned an incomplete user response'],
|
||||
[{ data: { user: {} } }, 'Twitter API returned an incomplete user response'],
|
||||
[{ data: { user: { result: { __typename: 'User' } } } }, 'Twitter API returned an incomplete user response'],
|
||||
[undefined, 'Twitter API returned an incomplete user response'],
|
||||
['<html>Unexpected upstream response</html>', 'Twitter API returned an incomplete user response'],
|
||||
])('rejects an upstream failure without caching or reporting a missing account', async (response, message) => {
|
||||
vi.mocked(twitterGot).mockResolvedValueOnce(response);
|
||||
let result: unknown;
|
||||
try {
|
||||
await api.getUser('example');
|
||||
} catch (error) {
|
||||
result = error;
|
||||
}
|
||||
expect(result).toBeInstanceOf(Error);
|
||||
expect(result).not.toBeInstanceOf(InvalidParameterError);
|
||||
expect((result as Error).message).toBe(message);
|
||||
expect(cached.size).toBe(0);
|
||||
expect(cache.set).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[{ data: { user: null } }, "This account doesn't exist"],
|
||||
[{ data: { user: { result: null } } }, "This account doesn't exist"],
|
||||
[{ data: { user: { result: { __typename: 'UserUnavailable', message: 'Upstream details' } } } }, 'Twitter user is unavailable'],
|
||||
])('distinguishes an explicitly missing or unavailable user', async (response, message) => {
|
||||
vi.mocked(twitterGot).mockResolvedValueOnce(response);
|
||||
await expect(api.getUser('example')).rejects.toThrow(new InvalidParameterError(message));
|
||||
expect(cached.size).toBe(0);
|
||||
});
|
||||
|
||||
it('retries normally after a GraphQL error and accepts an empty errors array', async () => {
|
||||
vi.mocked(twitterGot)
|
||||
.mockResolvedValueOnce({ errors: [{ code: 88 }] })
|
||||
.mockResolvedValueOnce({ ...user, errors: [] });
|
||||
await expect(api.getUser('example')).rejects.toThrow('Twitter API user lookup failed');
|
||||
expect((await api.getUser('example')).name).toBe('Example');
|
||||
expect(twitterGot).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('validates cached data before dereferencing it', async () => {
|
||||
cached.set('twitter:web:userdata:v2:example', { data: {} });
|
||||
await expect(api.getUser('example')).rejects.toThrow('Twitter API returned an incomplete user response');
|
||||
expect(twitterGot).not.toHaveBeenCalled();
|
||||
expect(cache.set).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves numeric-ID lookup parameters and auth requirements', async () => {
|
||||
await api.getUser('+123');
|
||||
expect(twitterGot).toHaveBeenCalledWith('https://x.com/i/api/graphql/user-by-id', expect.objectContaining({ variables: JSON.stringify({ userId: '123', withSafetyModeUserFields: true }) }), {
|
||||
allowNoAuth: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('validates third-party GraphQL errors without calling the direct API', async () => {
|
||||
config.twitter.thirdPartyApi = 'https://api.example.com';
|
||||
vi.mocked(ofetch).mockResolvedValueOnce({ errors: [{ code: 88 }] });
|
||||
await expect(api.getUser('example')).rejects.toThrow('Twitter API user lookup failed (codes: 88)');
|
||||
expect(cached.size).toBe(0);
|
||||
expect(twitterGot).not.toHaveBeenCalled();
|
||||
expect(ofetch).toHaveBeenCalledWith('https://api.example.com/graphql/user-by-name', expect.objectContaining({ method: 'GET' }));
|
||||
});
|
||||
|
||||
it('does not request tweets or write an empty legacy cache entry when user lookup fails', async () => {
|
||||
vi.mocked(twitterGot).mockResolvedValueOnce({ errors: [{ code: 88 }] });
|
||||
await expect(api.getUserTweets('example')).rejects.toThrow('Twitter API user lookup failed');
|
||||
expect(paginationTweets).not.toHaveBeenCalled();
|
||||
expect(cache.set).not.toHaveBeenCalled();
|
||||
expect(cached.size).toBe(0);
|
||||
});
|
||||
|
||||
it('passes a verified user ID to the existing tweet cache and mapper', async () => {
|
||||
vi.mocked(paginationTweets).mockResolvedValueOnce({ tweets: 'upstream result' });
|
||||
vi.mocked(gatherLegacyFromData).mockReturnValueOnce([{ id_str: 'tweet-1' }]);
|
||||
expect(await api.getUserTweets('example')).toEqual([{ id_str: 'tweet-1' }]);
|
||||
expect(paginationTweets).toHaveBeenCalledWith('UserTweets', '123', expect.objectContaining({ count: 20 }));
|
||||
expect(cache.tryGet).toHaveBeenCalledWith('tweet-cache', expect.any(Function), 300, false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
// Run user-response validation and cache behavior in workerd as well.
|
||||
import './twitter-web-user.test';
|
||||
@@ -0,0 +1,40 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { route } from '../lib/routes/xiaoyuzhou/podcast';
|
||||
import ofetch from '../lib/utils/ofetch';
|
||||
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: vi.fn() }));
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: (_key: string, fetchValue: () => unknown) => fetchValue() } }));
|
||||
|
||||
const context = { req: { param: () => 'source-id' } } as any;
|
||||
const podcastHtml = `<script id="__NEXT_DATA__">${JSON.stringify({ props: { pageProps: { podcast: { pid: 'podcast-id', title: 'Podcast', image: {}, episodes: [] } } } })}</script>`;
|
||||
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
describe('Xiaoyuzhou podcast and episode resolution', () => {
|
||||
it('preserves a podcast access error without requesting the same ID as an episode', async () => {
|
||||
const blocked = Object.assign(new Error('Forbidden'), { status: 403 });
|
||||
vi.mocked(ofetch).mockRejectedValue(blocked);
|
||||
|
||||
await expect(route.handler!(context)).rejects.toBe(blocked);
|
||||
expect(ofetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resolves episode IDs when the podcast URL returns 404', async () => {
|
||||
vi.mocked(ofetch)
|
||||
.mockRejectedValueOnce(Object.assign(new Error('Not Found'), { status: 404 }))
|
||||
.mockResolvedValueOnce('<a class="name" href="/podcast/podcast-id">Podcast</a>')
|
||||
.mockResolvedValueOnce(podcastHtml);
|
||||
|
||||
await expect(route.handler!(context)).resolves.toMatchObject({ title: 'Podcast', link: 'https://www.xiaoyuzhoufm.com/podcast/podcast-id' });
|
||||
expect(ofetch).toHaveBeenNthCalledWith(2, 'https://www.xiaoyuzhoufm.com/episode/source-id');
|
||||
expect(ofetch).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('does not hide malformed page data behind an unrelated episode request', async () => {
|
||||
vi.mocked(ofetch).mockResolvedValue('<html>Unexpected response</html>');
|
||||
|
||||
await expect(route.handler!(context)).rejects.toBeInstanceOf(SyntaxError);
|
||||
expect(ofetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import xxhash from 'xxhash-wasm';
|
||||
|
||||
// Empty/a/abc are standard XXH vectors; the remaining vectors were generated
|
||||
// with xxhash-wasm 1.1.0's Node entry. This suite also runs via its workerd entry.
|
||||
const vectors = [
|
||||
{ input: '', h32: '02cc5d05', h64: 'ef46db3751d8e999', seed32: 'bd209070', seed64: '51e24c0e9077a48c' },
|
||||
{ input: 'a', h32: '550d7456', h64: 'd24ec4f1a98c6e5b', seed32: '0d5a8e75', seed64: '63a379f532ba23d4' },
|
||||
{ input: 'abc', h32: '32d153ff', h64: '44bc2cf5ad770999', seed32: '11364062', seed64: '1fc03ef74cebaa7d' },
|
||||
{ input: '0123456789abcdef'.repeat(5), h32: '3fe8e7af', h64: '0d4cc7d5057880df', seed32: 'e0c25edb', seed64: '3d2aa4692d5507dc' },
|
||||
{ input: 'RSSHub 中文 🐧', h32: '57c604bf', h64: '24862e5b32f4f03f', seed32: '129c225f', seed64: '163f1ecdcce47f0a' },
|
||||
{ input: '/gov/zhengce/zhengceku/bmwj:json', h32: 'e008b760', h64: '390f38bd8df8e23e', seed32: 'abc5ecf2', seed64: 'b1183e2c2e5ba95d' },
|
||||
{ input: '/gov/zhengce/zhengceku/bmwj:json:47', h32: 'f144b479', h64: '24d1ed2f1309ed35', seed32: 'f6461457', seed64: '2675f5df8c4442da' },
|
||||
];
|
||||
|
||||
const hash = await xxhash();
|
||||
|
||||
describe('official xxhash Node and Worker parity', () => {
|
||||
it.each(vectors)('matches known 32/64-bit and seeded vectors for $input', (vector) => {
|
||||
const bytes = new TextEncoder().encode(vector.input);
|
||||
expect(hash.h32ToString(vector.input)).toBe(vector.h32);
|
||||
expect(hash.h64ToString(vector.input)).toBe(vector.h64);
|
||||
expect(hash.h32(vector.input)).toBe(Number.parseInt(vector.h32, 16));
|
||||
expect(hash.h64(vector.input)).toBe(BigInt('0x' + vector.h64));
|
||||
expect(hash.h32Raw(bytes)).toBe(Number.parseInt(vector.h32, 16));
|
||||
expect(hash.h64Raw(bytes)).toBe(BigInt('0x' + vector.h64));
|
||||
expect(hash.h32ToString(vector.input, 0x12_34_56_78)).toBe(vector.seed32);
|
||||
expect(hash.h64ToString(vector.input, 0x01_23_45_67_89_ab_cd_efn)).toBe(vector.seed64);
|
||||
});
|
||||
|
||||
it('supports streaming mixed strings and byte subarrays across block boundaries', () => {
|
||||
const prefix = '0123456789abcdef'.repeat(5);
|
||||
const bytes = new TextEncoder().encode('skipRSSHub 中文 🐧skip');
|
||||
const suffix = bytes.subarray(4, -4);
|
||||
const h32 = hash.create32(0x12_34_56_78);
|
||||
const h64 = hash.create64(0x01_23_45_67_89_ab_cd_efn);
|
||||
for (const chunk of [prefix.slice(0, 7), prefix.slice(7), suffix]) {
|
||||
h32.update(chunk);
|
||||
h64.update(chunk);
|
||||
}
|
||||
expect(h32.digest()).toBe(hash.h32(prefix + 'RSSHub 中文 🐧', 0x12_34_56_78));
|
||||
expect(h64.digest()).toBe(hash.h64(prefix + 'RSSHub 中文 🐧', 0x01_23_45_67_89_ab_cd_efn));
|
||||
});
|
||||
|
||||
it('does not retain the polynomial shim collision', () => {
|
||||
expect(hash.h64ToString('Aa')).not.toBe(hash.h64ToString('BB'));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
// Run the same reference vectors against the package's static-WASM workerd export.
|
||||
import './xxhash-wasm.test';
|
||||
@@ -84,7 +84,6 @@ export default defineConfig({
|
||||
'dotenv/config': path.resolve('./lib/shims/dotenv-config.ts'),
|
||||
'@sentry/node': path.resolve('./lib/shims/sentry-node.ts'),
|
||||
'@honeybadger-io/js': path.resolve('./lib/shims/honeybadger.ts'),
|
||||
'xxhash-wasm': path.resolve('./lib/shims/xxhash-wasm.ts'),
|
||||
typescript: path.resolve('./lib/shims/typescript.ts'),
|
||||
// Routes file with Worker-specific build (match relative import from lib/)
|
||||
'../assets/build/routes.js': path.resolve('./assets/build/routes-worker.js'),
|
||||
@@ -94,6 +93,8 @@ export default defineConfig({
|
||||
deps: {
|
||||
onlyBundle: false,
|
||||
neverBundle: [
|
||||
// Let Wrangler select the package's static-WASM workerd export.
|
||||
'xxhash-wasm',
|
||||
// Exclude non-code files that might be accidentally imported
|
||||
/\/_README$/,
|
||||
/\.node$/,
|
||||
|
||||
@@ -58,6 +58,7 @@ export default defineConfig({
|
||||
compatibilityDate: '2026-08-22',
|
||||
compatibilityFlags: ['nodejs_compat'],
|
||||
kvNamespaces: ['CACHE'],
|
||||
modulesRules: [{ type: 'CompiledWasm', include: ['**/*.wasm'] }],
|
||||
},
|
||||
}),
|
||||
workerAliasPlugin(),
|
||||
@@ -69,6 +70,6 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
test: {
|
||||
include: ['lib/**/*.worker.test.ts', 'tests/**/*.worker.test.ts', 'lib/utils/parse-script-data.test.ts'],
|
||||
include: ['lib/**/*.worker.test.ts', 'tests/**/*.worker.test.ts', 'lib/utils/parse-script-data.test.ts', 'lib/utils/parse-date-in-timezone.test.ts', 'tests/source-date-routes.test.ts'],
|
||||
},
|
||||
});
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
name = "rsshub"
|
||||
main = "dist-worker/worker.mjs"
|
||||
compatibility_date = "2026-09-01"
|
||||
compatibility_flags = ["nodejs_compat"]
|
||||
compatibility_flags = ["nodejs_compat", "global_fetch_strictly_public"]
|
||||
|
||||
find_additional_modules = true
|
||||
base_dir = "dist-worker/modules"
|
||||
|
||||
Reference in New Issue
Block a user