mirror of
https://github.com/DIYgod/RSSHub.git
synced 2026-10-13 08:48:59 +08:00
perf: replace acorn with oxc-parser (#23302)
* perf: speed up js parsing * fix: drop parseScriptData for dongqiudi * refactor: rename evaluate-script * refactor: rename exports * chore: move oxc-parser to dependencies
This commit is contained in:
@@ -43,6 +43,7 @@ updates:
|
||||
- '@opentelemetry/*'
|
||||
oxc:
|
||||
patterns:
|
||||
- '@oxc-parser/*'
|
||||
- '@oxlint/*'
|
||||
- 'oxc-parser'
|
||||
- 'oxfmt'
|
||||
|
||||
@@ -42,7 +42,7 @@ async function handler(ctx) {
|
||||
cache.tryGet(item.link, async () => {
|
||||
try {
|
||||
const { data: response } = await got(item.link);
|
||||
processFeedType2(item, response);
|
||||
await processFeedType2(item, response);
|
||||
} catch (error) {
|
||||
if (!(error instanceof FetchError) || error.statusCode !== 404) {
|
||||
throw error;
|
||||
|
||||
@@ -49,7 +49,7 @@ async function handler(ctx) {
|
||||
list.map((item) =>
|
||||
cache.tryGet(item.link, async () => {
|
||||
const { data: response } = await got(item.link);
|
||||
processFeedType2(item, response);
|
||||
await processFeedType2(item, response);
|
||||
return item;
|
||||
})
|
||||
)
|
||||
|
||||
@@ -3,34 +3,24 @@ import { load } from 'cheerio';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
import timezone from '@/utils/timezone';
|
||||
|
||||
const readNuxtData = (response: string) => {
|
||||
const $ = load(response);
|
||||
const script = $('script')
|
||||
.toArray()
|
||||
.map((element) => $(element).text())
|
||||
.find((source) => /\b__NUXT__\s*=/.test(source));
|
||||
return script ? parseScriptData<{ data?: any[] } | undefined>(script, '__NUXT__')?.data?.[0] : undefined;
|
||||
};
|
||||
|
||||
const processVideo = (content) => {
|
||||
content('div.video').each((i, v) => {
|
||||
const processVideo = ($, scope) => {
|
||||
scope.find('div.video').each((i, v) => {
|
||||
let link = new URL(v.attribs.src);
|
||||
if (link.host === 'm.miguvideo.com') {
|
||||
content(`<a href="${link.href}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
|
||||
content(v).remove();
|
||||
$(`<a href="${link.href}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
|
||||
$(v).remove();
|
||||
} else {
|
||||
link = v.attribs.src;
|
||||
switch (v.attribs.site) {
|
||||
case 'qiniu':
|
||||
content(`<video width="100%" controls="controls"> <source src="${link}" type="video/mp4"> Your RSS reader does not support video playback. </video>`).insertAfter(v);
|
||||
content(v).remove();
|
||||
$(`<video width="100%" controls="controls"> <source src="${link}" type="video/mp4"> Your RSS reader does not support video playback. </video>`).insertAfter(v);
|
||||
$(v).remove();
|
||||
break;
|
||||
case 'youku':
|
||||
content(`<iframe height='100%' width='100%' src='${link}' frameborder=0 scrolling=no webkitallowfullscreen=true allowfullscreen=true></iframe>`).insertAfter(v);
|
||||
content(v).remove();
|
||||
$(`<iframe height='100%' width='100%' src='${link}' frameborder=0 scrolling=no webkitallowfullscreen=true allowfullscreen=true></iframe>`).insertAfter(v);
|
||||
$(v).remove();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
@@ -39,16 +29,14 @@ const processVideo = (content) => {
|
||||
});
|
||||
|
||||
// Process iframes
|
||||
content('iframe.media-iframe, .edui-faked-video').each((i, v) => {
|
||||
scope.find('iframe.media-iframe, .edui-faked-video').each((i, v) => {
|
||||
const link = v.attribs.src;
|
||||
if (link.startsWith('http://ssports.iqiyi.com/')) {
|
||||
content(`<a href="${link.link}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
|
||||
$(`<a href="${link.link}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
|
||||
}
|
||||
|
||||
content(v).remove();
|
||||
$(v).remove();
|
||||
});
|
||||
|
||||
return content;
|
||||
};
|
||||
|
||||
const processHref = (content) => {
|
||||
@@ -77,19 +65,16 @@ export const processFeed = async (type, id) => {
|
||||
const link = `https://www.dongqiudi.com/${type}/${id}.html`;
|
||||
const apiUrl = 'https://api.dongqiudi.com/v3/archive/app/channel/feeds';
|
||||
const { data: response } = await got(link);
|
||||
const $ = load(response);
|
||||
|
||||
const nuxtData = readNuxtData(response);
|
||||
if (!nuxtData) {
|
||||
throw new Error('Unable to extract Dongqiudi page data');
|
||||
}
|
||||
let name;
|
||||
let image;
|
||||
if (type === 'team') {
|
||||
name = nuxtData.teamInfo.name;
|
||||
image = nuxtData.teamInfo.logo;
|
||||
name = $('h1.tp-hero__name').text();
|
||||
image = $('.tp-hero__logo').attr('src');
|
||||
} else {
|
||||
name = nuxtData.detail.base_info.person_name;
|
||||
image = nuxtData.detail.base_info.person_logo;
|
||||
name = $('h1.pp-hero__name').text().trim();
|
||||
image = $('.pp-hero__avatar').attr('src');
|
||||
}
|
||||
|
||||
const { data } = await got(apiUrl, {
|
||||
@@ -110,12 +95,17 @@ export const processFeed = async (type, id) => {
|
||||
}));
|
||||
|
||||
if (type === 'team' && list.length === 0) {
|
||||
list = nuxtData.newsList.map((news) => ({
|
||||
title: news.title,
|
||||
link: `https://www.dongqiudi.com/articles/${news.id}.html`,
|
||||
category: [news.category],
|
||||
pubDate: timezone(parseDate(news.time), 8),
|
||||
}));
|
||||
list = $('.tp-news-item')
|
||||
.toArray()
|
||||
.map((element) => {
|
||||
const news = $(element);
|
||||
return {
|
||||
title: news.find('.tp-news-item__title').text(),
|
||||
link: new URL(news.attr('href')!, link).href,
|
||||
category: [news.find('.tp-news-item__tag').text()],
|
||||
pubDate: timezone(parseDate(news.find('.tp-news-item__time').text(), 'YYYY-MM-DD HH:mm'), 8),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const out = await Promise.all(
|
||||
@@ -123,7 +113,7 @@ export const processFeed = async (type, id) => {
|
||||
cache.tryGet(item.link, async () => {
|
||||
const { data: response } = await got(item.link);
|
||||
|
||||
processFeedType2(item, response);
|
||||
await processFeedType2(item, response);
|
||||
|
||||
return item;
|
||||
})
|
||||
@@ -139,16 +129,18 @@ export const processFeed = async (type, id) => {
|
||||
};
|
||||
|
||||
export const processFeedType2 = (item, response) => {
|
||||
const data = readNuxtData(response)?.article;
|
||||
|
||||
// filter out undefined item
|
||||
if (!data) {
|
||||
const $ = load(response);
|
||||
const articleBody = $('.article-body');
|
||||
if (!articleBody.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const body = processVideo(load(data.rawBody, null, false));
|
||||
processHref(body('a'));
|
||||
processImg(body('img'));
|
||||
item.description = body.html();
|
||||
item.author = data.author;
|
||||
processVideo($, articleBody);
|
||||
processHref(articleBody.find('a'));
|
||||
processImg(articleBody.find('img'));
|
||||
item.description = articleBody.html();
|
||||
const author = $('.article-head__author-name').text();
|
||||
if (author) {
|
||||
item.author = author;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { load } from 'cheerio';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import { evaluateScriptData } from '@/utils/evaluate-script';
|
||||
import got from '@/utils/got';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/hottest',
|
||||
@@ -37,7 +37,7 @@ async function handler() {
|
||||
.map((element) => $(element).text())
|
||||
.filter((source) => source.includes('__INITIAL_STATE__'))
|
||||
.join('\n');
|
||||
const { widgets } = parseScriptData<{
|
||||
const { widgets } = await evaluateScriptData<{
|
||||
widgets: {
|
||||
hottestWidget: {
|
||||
threads: Array<{ title: string; mainImage: { path: string; name: string }; temperature: number; displayPrice: string; url: string }>;
|
||||
|
||||
@@ -3,9 +3,9 @@ import { load } from 'cheerio';
|
||||
import { config } from '@/config';
|
||||
import ConfigNotFoundError from '@/errors/types/config-not-found';
|
||||
import type { Route } from '@/types';
|
||||
import { evaluateScriptCallback } from '@/utils/evaluate-script';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseScriptCallback } from '@/utils/parse-script-data';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/tag/:name?/:type?',
|
||||
@@ -83,7 +83,7 @@ async function handler(ctx) {
|
||||
},
|
||||
});
|
||||
|
||||
const data = parseScriptCallback<any[]>(response.data, 'dwr.engine._remoteHandleCallback');
|
||||
const data = await evaluateScriptCallback<any[]>(response.data, 'dwr.engine._remoteHandleCallback');
|
||||
|
||||
const title =
|
||||
{
|
||||
|
||||
@@ -4,15 +4,15 @@ import { load } from 'cheerio';
|
||||
import dayjs from 'dayjs';
|
||||
import localizedFormat from 'dayjs/plugin/localizedFormat.js';
|
||||
|
||||
import { evaluateScriptData } from '@/utils/evaluate-script';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
|
||||
import { renderEshopCnDescription } from './templates/eshop-cn';
|
||||
|
||||
dayjs.extend(localizedFormat);
|
||||
|
||||
function nuxtReader(data) {
|
||||
async function nuxtReader(data) {
|
||||
let nuxt;
|
||||
try {
|
||||
const $ = load(data);
|
||||
@@ -20,7 +20,7 @@ function nuxtReader(data) {
|
||||
.toArray()
|
||||
.map((element) => $(element).text())
|
||||
.find((source) => /\b__NUXT__\s*=/.test(source));
|
||||
nuxt = parseScriptData<{ data: any[] }>(script ?? '', '__NUXT__').data[0];
|
||||
nuxt = (await evaluateScriptData<{ data: any[] }>(script ?? '', '__NUXT__')).data[0];
|
||||
} catch {
|
||||
throw new Error('Nuxt 框架信息提取失败,请报告这个问题');
|
||||
}
|
||||
|
||||
@@ -2,8 +2,8 @@ import { load } from 'cheerio';
|
||||
import queryString from 'query-string';
|
||||
|
||||
import cache from '@/utils/cache';
|
||||
import { evaluateScriptData } from '@/utils/evaluate-script';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
|
||||
import { maskHeader } from '../../constants';
|
||||
import got from '../../pixiv-got';
|
||||
@@ -32,7 +32,7 @@ export async function getNSFWNovelContent(novelId: string, token: string): Promi
|
||||
if (!script) {
|
||||
throw new Error('No novel data found');
|
||||
}
|
||||
const novelDetail = parseScriptData<NSFWNovelDetail | undefined>(script, 'window.pixiv.novel');
|
||||
const novelDetail = await evaluateScriptData<NSFWNovelDetail | undefined>(script, 'window.pixiv.novel');
|
||||
|
||||
if (!novelDetail) {
|
||||
throw new Error('No novel data found');
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { load } from 'cheerio';
|
||||
|
||||
import cache from '@/utils/cache';
|
||||
import { evaluateScriptData } from '@/utils/evaluate-script';
|
||||
import got from '@/utils/got';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
|
||||
export default {
|
||||
getPlayInfo: async (ctx, shareId, ksong_mid = '') => {
|
||||
@@ -16,7 +16,7 @@ export default {
|
||||
.map((element) => $(element).text())
|
||||
.filter((source) => source.includes('__DATA__'))
|
||||
.join('\n');
|
||||
const data = parseScriptData<{
|
||||
const data = await evaluateScriptData<{
|
||||
detail: {
|
||||
song_name: string;
|
||||
content: string;
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { load } from 'cheerio';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import { evaluateScriptData } from '@/utils/evaluate-script';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
|
||||
import cache from './cache';
|
||||
|
||||
@@ -36,7 +36,7 @@ async function handler(ctx) {
|
||||
.map((element) => $(element).text())
|
||||
.filter((source) => source.includes('__DATA__'))
|
||||
.join('\n');
|
||||
const data = parseScriptData<{
|
||||
const data = await evaluateScriptData<{
|
||||
data: {
|
||||
nickname: string;
|
||||
head_img_url: string;
|
||||
|
||||
@@ -2,9 +2,9 @@ import { load } from 'cheerio';
|
||||
import { CookieJar } from 'tough-cookie';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import { evaluateScriptData } from '@/utils/evaluate-script';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseScriptData } from '@/utils/parse-script-data';
|
||||
|
||||
const cookieJar = new CookieJar();
|
||||
const baseUrl = 'https://xueqiu.com';
|
||||
@@ -50,7 +50,7 @@ async function handler(ctx) {
|
||||
.map((element) => $(element).text())
|
||||
.filter((source) => source.includes('SNOWMAN_TARGET'))
|
||||
.join('\n');
|
||||
const snowmanTarget = parseScriptData<{ screen_name: string; description: string }>(script, 'SNOWMAN_TARGET');
|
||||
const snowmanTarget = await evaluateScriptData<{ screen_name: string; description: string }>(script, 'SNOWMAN_TARGET');
|
||||
|
||||
const { data } = await got(`${baseUrl}/statuses/original/timeline.json`, {
|
||||
cookieJar,
|
||||
|
||||
@@ -2,7 +2,6 @@ import sanitizeHtml from 'sanitize-html';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import { ViewType } from '@/types';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processImage, withZhihuClient } from './utils';
|
||||
@@ -21,7 +20,7 @@ export const route: Route = {
|
||||
optional: true,
|
||||
},
|
||||
],
|
||||
requirePuppeteer: isWorker || process.env.WORKER_BUILD === 'true',
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { Route } from '@/types';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processImage, withZhihuClient } from './utils';
|
||||
@@ -17,7 +16,7 @@ export const route: Route = {
|
||||
optional: true,
|
||||
},
|
||||
],
|
||||
requirePuppeteer: isWorker || process.env.WORKER_BUILD === 'true',
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
import { JSDOM, VirtualConsole } from 'jsdom';
|
||||
import { VM } from 'vm2';
|
||||
|
||||
import { config } from '@/config';
|
||||
import cache from '@/utils/cache';
|
||||
import { generateHeaders } from '@/utils/header-generator';
|
||||
import md5 from '@/utils/md5';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
|
||||
import { getSignedHeaders } from './sign';
|
||||
import type { ZhihuClient } from './utils';
|
||||
|
||||
let isUnreachableRuntimeErrorGuarded = false;
|
||||
const pendingZseCredentials = new Map<string, Promise<{ dc0: string; zseCk: string; ua: string }>>();
|
||||
|
||||
const preventUnreachableRuntimeError = () => {
|
||||
if (isUnreachableRuntimeErrorGuarded) {
|
||||
return;
|
||||
}
|
||||
isUnreachableRuntimeErrorGuarded = true;
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
const error = reason as { name?: string; message?: string } | undefined;
|
||||
if (error?.name === 'RuntimeError' && error.message === 'unreachable') {
|
||||
return;
|
||||
}
|
||||
throw reason;
|
||||
});
|
||||
};
|
||||
|
||||
const generateZseCk = async (url: string, apiPath: string, configuredDc0: string) => {
|
||||
preventUnreachableRuntimeError();
|
||||
|
||||
// `__zse_ck` is checked against the user-agent that generated it.
|
||||
const ua = generateHeaders()['user-agent'];
|
||||
const headers = { 'user-agent': ua };
|
||||
|
||||
let dc0 = configuredDc0;
|
||||
if (!dc0) {
|
||||
const seed = await ofetch.raw('https://www.zhihu.com/explore', {
|
||||
headers,
|
||||
redirect: 'manual',
|
||||
ignoreResponseError: true,
|
||||
});
|
||||
dc0 =
|
||||
(seed.headers.getSetCookie?.() ?? [])
|
||||
.find((line) => line.startsWith('d_c0='))
|
||||
?.split(';', 1)[0]
|
||||
.slice('d_c0='.length)
|
||||
.split('|', 1)[0]
|
||||
.replace(/=+$/, '') || '';
|
||||
}
|
||||
if (!dc0) {
|
||||
throw new Error('zhihu: failed to obtain a guest d_c0 cookie');
|
||||
}
|
||||
|
||||
const challenge = await ofetch.raw<string>(`https://www.zhihu.com${apiPath}`, {
|
||||
headers: {
|
||||
...headers,
|
||||
cookie: `d_c0=${dc0}; __zse_ck=005_x-x`,
|
||||
referer: url,
|
||||
'x-requested-with': 'fetch',
|
||||
},
|
||||
ignoreResponseError: true,
|
||||
});
|
||||
const html = challenge._data ?? '';
|
||||
const meta = html.match(/id="zh-zse-ck"[^>]*content="([^"]*)"/)?.[1];
|
||||
const hash = html.match(/zse-ck\/v4\/([a-f0-9]+)\.js/)?.[1];
|
||||
if (!meta || !hash) {
|
||||
throw new Error('zhihu: challenge page did not contain an URL to __zse_ck meta/script');
|
||||
}
|
||||
|
||||
const vmScript = await ofetch<string>(`https://static.zhihu.com/zse-ck/v4/${hash}.js`, {
|
||||
headers,
|
||||
parseResponse: (text) => text,
|
||||
});
|
||||
const dom = new JSDOM(`<!doctype html><html><head><meta id="zh-zse-ck" content="${meta}"><script data-assets-tracker-config='{"appName":"zse_ck"}'></script></head><body></body></html>`, {
|
||||
url,
|
||||
referrer: 'https://www.zhihu.com/',
|
||||
runScripts: 'outside-only',
|
||||
pretendToBeVisual: true,
|
||||
virtualConsole: new VirtualConsole(),
|
||||
});
|
||||
const { window } = dom;
|
||||
Object.defineProperties(window.navigator, {
|
||||
userAgent: { value: ua, configurable: true },
|
||||
webdriver: { value: false, configurable: true },
|
||||
});
|
||||
window.TextEncoder = TextEncoder;
|
||||
window.TextDecoder = TextDecoder as typeof window.TextDecoder;
|
||||
window.atob = (value: string) => Buffer.from(value, 'base64').toString('binary');
|
||||
window.btoa = (value: string) => Buffer.from(value, 'binary').toString('base64');
|
||||
Object.assign(window, { __g: {} });
|
||||
|
||||
const cookieDescriptor = Object.getOwnPropertyDescriptor(window.Document.prototype, 'cookie');
|
||||
const setCookie = cookieDescriptor?.set;
|
||||
if (!cookieDescriptor?.get || !setCookie) {
|
||||
window.close();
|
||||
throw new Error('zhihu: JSDOM did not provide document.cookie accessors');
|
||||
}
|
||||
const tokenPromise = new Promise<string>((resolve) => {
|
||||
Object.defineProperty(window.document, 'cookie', {
|
||||
configurable: true,
|
||||
get: cookieDescriptor.get,
|
||||
set(value: string) {
|
||||
setCookie.call(window.document, value);
|
||||
const token = value.match(/__zse_ck=([^;]+)/)?.[1];
|
||||
if (token?.includes('-')) {
|
||||
resolve(token);
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
let zseCk: string | undefined;
|
||||
let timeoutId: ReturnType<typeof setTimeout> | number | undefined;
|
||||
try {
|
||||
// Zhihu's challenge is intentionally delivered as executable JavaScript.
|
||||
new VM({ timeout: 3000, sandbox: window }).run(vmScript);
|
||||
const timeout = new Promise<undefined>((resolve) => {
|
||||
timeoutId = setTimeout(resolve, 3000);
|
||||
});
|
||||
zseCk = await Promise.race([tokenPromise, timeout]);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
window.close();
|
||||
}
|
||||
if (!zseCk) {
|
||||
throw new Error('zhihu: WASM VM did not produce a __zse_ck');
|
||||
}
|
||||
return { dc0, zseCk, ua };
|
||||
};
|
||||
|
||||
const getGeneratedZseCredentials = (url: string, apiPath: string, configuredDc0: string) => {
|
||||
const cacheKey = `zhihu:zse-ck:v4:${configuredDc0 ? md5(configuredDc0) : 'guest'}`;
|
||||
const pending = pendingZseCredentials.get(cacheKey);
|
||||
if (pending) {
|
||||
return pending;
|
||||
}
|
||||
|
||||
const created = (async () => {
|
||||
try {
|
||||
return await cache.tryGet(cacheKey, () => generateZseCk(url, apiPath, configuredDc0), config.cache.contentExpire, false);
|
||||
} finally {
|
||||
pendingZseCredentials.delete(cacheKey);
|
||||
}
|
||||
})();
|
||||
pendingZseCredentials.set(cacheKey, created);
|
||||
return created;
|
||||
};
|
||||
|
||||
const mergeGeneratedCookies = (configured: string, dc0: string, zseCk: string) => {
|
||||
const remaining = configured
|
||||
.split(';')
|
||||
.map((pair) => pair.trim())
|
||||
.filter((pair) => {
|
||||
const name = pair.split('=', 1)[0];
|
||||
return name && name !== 'd_c0' && name !== '__zse_ck';
|
||||
});
|
||||
return [`__zse_ck=${zseCk}`, `d_c0=${dc0}`, ...remaining].join('; ');
|
||||
};
|
||||
|
||||
export const createJSDOMClient = (pageUrl: string, configured: string, configuredDc0: string): ZhihuClient => {
|
||||
let sessionPromise: Promise<{ dc0: string; headers: { cookie: string; 'user-agent': string } }> | undefined;
|
||||
const startSession = async (apiPath: string) => {
|
||||
try {
|
||||
const { dc0, zseCk, ua } = await getGeneratedZseCredentials(pageUrl, apiPath, configuredDc0);
|
||||
return {
|
||||
dc0,
|
||||
headers: {
|
||||
// An isolated login cookie must not be mixed with a new guest identity.
|
||||
cookie: configuredDc0 ? mergeGeneratedCookies(configured, dc0, zseCk) : `__zse_ck=${zseCk}; d_c0=${dc0}`,
|
||||
'user-agent': ua,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
sessionPromise = undefined;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const getSession = (apiPath = new URL(pageUrl).pathname + new URL(pageUrl).search) => (sessionPromise ??= startSession(apiPath));
|
||||
|
||||
return {
|
||||
get: async <Result = any>(apiPath: string): Promise<Result> => {
|
||||
if (!apiPath.startsWith('/api/')) {
|
||||
throw new Error('zhihu: expected an API path');
|
||||
}
|
||||
const { dc0, headers } = await getSession(apiPath);
|
||||
return ofetch<Result>(`https://www.zhihu.com${apiPath}`, {
|
||||
headers: { ...getSignedHeaders(apiPath, dc0), ...headers, Referer: pageUrl },
|
||||
});
|
||||
},
|
||||
getPage: async () => {
|
||||
const { headers } = await getSession();
|
||||
return ofetch<string>(pageUrl, { headers: { ...headers, Referer: pageUrl }, parseResponse: (text) => text });
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,6 +0,0 @@
|
||||
import type { createJSDOMClient as createNodeJSDOMClient } from './jsdom';
|
||||
|
||||
// Exclude JSDOM from Worker bundles, including unreachable dynamic import chunks.
|
||||
export const createJSDOMClient: typeof createNodeJSDOMClient = () => {
|
||||
throw new Error('zhihu: JSDOM session initialization is unavailable on Workers; use a browser session');
|
||||
};
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import type { Articles } from './types';
|
||||
@@ -19,7 +18,7 @@ export const route: Route = {
|
||||
optional: true,
|
||||
},
|
||||
],
|
||||
requirePuppeteer: isWorker || process.env.WORKER_BUILD === 'true',
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { Route } from '@/types';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processImage, withZhihuClient } from './utils';
|
||||
@@ -17,7 +16,7 @@ export const route: Route = {
|
||||
optional: true,
|
||||
},
|
||||
],
|
||||
requirePuppeteer: isWorker || process.env.WORKER_BUILD === 'true',
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processImage, withZhihuClient } from './utils';
|
||||
@@ -17,7 +16,7 @@ export const route: Route = {
|
||||
description: 'A complete d_c0 and __zse_ck cookie pair skips session initialization. Otherwise Workers use a Playwright browser session; Docker and Vercel generate credentials with JSDOM.',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: isWorker || process.env.WORKER_BUILD === 'true',
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
|
||||
+194
-8
@@ -1,10 +1,15 @@
|
||||
import { load } from 'cheerio';
|
||||
import { JSDOM, VirtualConsole } from 'jsdom';
|
||||
import { VM } from 'vm2';
|
||||
|
||||
import { config } from '@/config';
|
||||
import cache from '@/utils/cache';
|
||||
import { generateHeaders } from '@/utils/header-generator';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import md5 from '@/utils/md5';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
|
||||
import type { createBrowserClient } from './browser';
|
||||
import { createBrowserClient } from './browser';
|
||||
import { getSignedHeaders } from './sign';
|
||||
|
||||
export const header = {
|
||||
@@ -72,22 +77,203 @@ export type ZhihuClient = {
|
||||
getPage(): Promise<string>;
|
||||
};
|
||||
|
||||
let isUnreachableRuntimeErrorGuarded = false;
|
||||
const pendingZseCredentials = new Map<string, Promise<{ dc0: string; zseCk: string; ua: string }>>();
|
||||
|
||||
const preventUnreachableRuntimeError = () => {
|
||||
if (isUnreachableRuntimeErrorGuarded) {
|
||||
return;
|
||||
}
|
||||
isUnreachableRuntimeErrorGuarded = true;
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
const error = reason as { name?: string; message?: string } | undefined;
|
||||
if (error?.name === 'RuntimeError' && error.message === 'unreachable') {
|
||||
return;
|
||||
}
|
||||
throw reason;
|
||||
});
|
||||
};
|
||||
|
||||
const generateZseCk = async (url: string, apiPath: string, configuredDc0: string) => {
|
||||
preventUnreachableRuntimeError();
|
||||
|
||||
// `__zse_ck` is checked against the user-agent that generated it.
|
||||
const ua = generateHeaders()['user-agent'];
|
||||
const headers = { 'user-agent': ua };
|
||||
|
||||
let dc0 = configuredDc0;
|
||||
if (!dc0) {
|
||||
const seed = await ofetch.raw('https://www.zhihu.com/explore', {
|
||||
headers,
|
||||
redirect: 'manual',
|
||||
ignoreResponseError: true,
|
||||
});
|
||||
dc0 =
|
||||
(seed.headers.getSetCookie?.() ?? [])
|
||||
.find((line) => line.startsWith('d_c0='))
|
||||
?.split(';', 1)[0]
|
||||
.slice('d_c0='.length)
|
||||
.split('|', 1)[0]
|
||||
.replace(/=+$/, '') || '';
|
||||
}
|
||||
if (!dc0) {
|
||||
throw new Error('zhihu: failed to obtain a guest d_c0 cookie');
|
||||
}
|
||||
|
||||
const challenge = await ofetch.raw<string>(`https://www.zhihu.com${apiPath}`, {
|
||||
headers: {
|
||||
...headers,
|
||||
cookie: `d_c0=${dc0}; __zse_ck=005_x-x`,
|
||||
referer: url,
|
||||
'x-requested-with': 'fetch',
|
||||
},
|
||||
ignoreResponseError: true,
|
||||
});
|
||||
const html = challenge._data ?? '';
|
||||
const meta = html.match(/id="zh-zse-ck"[^>]*content="([^"]*)"/)?.[1];
|
||||
const hash = html.match(/zse-ck\/v4\/([a-f0-9]+)\.js/)?.[1];
|
||||
if (!meta || !hash) {
|
||||
throw new Error('zhihu: challenge page did not contain an URL to __zse_ck meta/script');
|
||||
}
|
||||
|
||||
const vmScript = await ofetch<string>(`https://static.zhihu.com/zse-ck/v4/${hash}.js`, {
|
||||
headers,
|
||||
parseResponse: (text) => text,
|
||||
});
|
||||
const dom = new JSDOM(`<!doctype html><html><head><meta id="zh-zse-ck" content="${meta}"><script data-assets-tracker-config='{"appName":"zse_ck"}'></script></head><body></body></html>`, {
|
||||
url,
|
||||
referrer: 'https://www.zhihu.com/',
|
||||
runScripts: 'outside-only',
|
||||
pretendToBeVisual: true,
|
||||
virtualConsole: new VirtualConsole(),
|
||||
});
|
||||
const { window } = dom;
|
||||
Object.defineProperties(window.navigator, {
|
||||
userAgent: { value: ua, configurable: true },
|
||||
webdriver: { value: false, configurable: true },
|
||||
});
|
||||
window.TextEncoder = TextEncoder;
|
||||
window.TextDecoder = TextDecoder as typeof window.TextDecoder;
|
||||
window.atob = (value: string) => Buffer.from(value, 'base64').toString('binary');
|
||||
window.btoa = (value: string) => Buffer.from(value, 'binary').toString('base64');
|
||||
Object.assign(window, { __g: {} });
|
||||
|
||||
const cookieDescriptor = Object.getOwnPropertyDescriptor(window.Document.prototype, 'cookie');
|
||||
const setCookie = cookieDescriptor?.set;
|
||||
if (!cookieDescriptor?.get || !setCookie) {
|
||||
window.close();
|
||||
throw new Error('zhihu: JSDOM did not provide document.cookie accessors');
|
||||
}
|
||||
const tokenPromise = new Promise<string>((resolve) => {
|
||||
Object.defineProperty(window.document, 'cookie', {
|
||||
configurable: true,
|
||||
get: cookieDescriptor.get,
|
||||
set(value: string) {
|
||||
setCookie.call(window.document, value);
|
||||
const token = value.match(/__zse_ck=([^;]+)/)?.[1];
|
||||
if (token?.includes('-')) {
|
||||
resolve(token);
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
let zseCk: string | undefined;
|
||||
let timeoutId: ReturnType<typeof setTimeout> | number | undefined;
|
||||
try {
|
||||
// Zhihu's challenge is intentionally delivered as executable JavaScript.
|
||||
new VM({ timeout: 3000, sandbox: window }).run(vmScript);
|
||||
const timeout = new Promise<undefined>((resolve) => {
|
||||
timeoutId = setTimeout(resolve, 3000);
|
||||
});
|
||||
zseCk = await Promise.race([tokenPromise, timeout]);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
window.close();
|
||||
}
|
||||
if (!zseCk) {
|
||||
throw new Error('zhihu: WASM VM did not produce a __zse_ck');
|
||||
}
|
||||
return { dc0, zseCk, ua };
|
||||
};
|
||||
|
||||
const getGeneratedZseCredentials = (url: string, apiPath: string, configuredDc0: string) => {
|
||||
const cacheKey = `zhihu:zse-ck:v4:${configuredDc0 ? md5(configuredDc0) : 'guest'}`;
|
||||
const pending = pendingZseCredentials.get(cacheKey);
|
||||
if (pending) {
|
||||
return pending;
|
||||
}
|
||||
|
||||
const created = (async () => {
|
||||
try {
|
||||
return await cache.tryGet(cacheKey, () => generateZseCk(url, apiPath, configuredDc0), config.cache.contentExpire, false);
|
||||
} finally {
|
||||
pendingZseCredentials.delete(cacheKey);
|
||||
}
|
||||
})();
|
||||
pendingZseCredentials.set(cacheKey, created);
|
||||
return created;
|
||||
};
|
||||
|
||||
const mergeGeneratedCookies = (configured: string, dc0: string, zseCk: string) => {
|
||||
const remaining = configured
|
||||
.split(';')
|
||||
.map((pair) => pair.trim())
|
||||
.filter((pair) => {
|
||||
const name = pair.split('=', 1)[0];
|
||||
return name && name !== 'd_c0' && name !== '__zse_ck';
|
||||
});
|
||||
return [`__zse_ck=${zseCk}`, `d_c0=${dc0}`, ...remaining].join('; ');
|
||||
};
|
||||
|
||||
const createJSDOMClient = (pageUrl: string, configured: string, configuredDc0: string): ZhihuClient => {
|
||||
let sessionPromise: Promise<{ dc0: string; headers: { cookie: string; 'user-agent': string } }> | undefined;
|
||||
const startSession = async (apiPath: string) => {
|
||||
try {
|
||||
const { dc0, zseCk, ua } = await getGeneratedZseCredentials(pageUrl, apiPath, configuredDc0);
|
||||
return {
|
||||
dc0,
|
||||
headers: {
|
||||
// An isolated login cookie must not be mixed with a new guest identity.
|
||||
cookie: configuredDc0 ? mergeGeneratedCookies(configured, dc0, zseCk) : `__zse_ck=${zseCk}; d_c0=${dc0}`,
|
||||
'user-agent': ua,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
sessionPromise = undefined;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const getSession = (apiPath = new URL(pageUrl).pathname + new URL(pageUrl).search) => (sessionPromise ??= startSession(apiPath));
|
||||
|
||||
return {
|
||||
get: async <Result = any>(apiPath: string): Promise<Result> => {
|
||||
if (!apiPath.startsWith('/api/')) {
|
||||
throw new Error('zhihu: expected an API path');
|
||||
}
|
||||
const { dc0, headers } = await getSession(apiPath);
|
||||
return ofetch<Result>(`https://www.zhihu.com${apiPath}`, {
|
||||
headers: { ...getSignedHeaders(apiPath, dc0), ...headers, Referer: pageUrl },
|
||||
});
|
||||
},
|
||||
getPage: async () => {
|
||||
const { headers } = await getSession();
|
||||
return ofetch<string>(pageUrl, { headers: { ...headers, Referer: pageUrl }, parseResponse: (text) => text });
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const withZhihuClient = async <T>(pageUrl: string, callback: (client: ZhihuClient) => Promise<T>): Promise<T> => {
|
||||
const configured = config.zhihu.cookies || '';
|
||||
const dc0 = getCookieValueFrom(configured, 'd_c0');
|
||||
const hasConfiguredSession = !!dc0 && !!getCookieValueFrom(configured, '__zse_ck');
|
||||
if (!isWorker && !hasConfiguredSession) {
|
||||
const { createJSDOMClient } = await import('./jsdom');
|
||||
return callback(createJSDOMClient(pageUrl, configured, dc0));
|
||||
}
|
||||
|
||||
let browserPromise: ReturnType<typeof createBrowserClient> | undefined;
|
||||
const startBrowser = async (apiPath?: string) => {
|
||||
const { createBrowserClient } = await import('./browser');
|
||||
// Column APIs can initialize directly; other routes need their page session.
|
||||
return createBrowserClient(pageUrl, dc0 ? configured : '', apiPath?.startsWith('/api/v4/columns/') ? apiPath : undefined);
|
||||
};
|
||||
const getBrowser = (apiPath?: string) => (browserPromise ??= startBrowser(apiPath));
|
||||
// Column APIs can initialize directly; other routes need their page session.
|
||||
const getBrowser = (apiPath?: string) => (browserPromise ??= createBrowserClient(pageUrl, dc0 ? configured : '', apiPath?.startsWith('/api/v4/columns/') ? apiPath : undefined));
|
||||
|
||||
try {
|
||||
return await callback({
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { load } from 'cheerio';
|
||||
|
||||
import type { Route } from '@/types';
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processImage, withZhihuClient } from './utils';
|
||||
@@ -18,7 +17,7 @@ export const route: Route = {
|
||||
description: 'A complete d_c0 and __zse_ck cookie pair skips session initialization. Otherwise Workers use a Playwright browser session; Docker and Vercel generate credentials with JSDOM.',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: isWorker || process.env.WORKER_BUILD === 'true',
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
const unavailable = () => {
|
||||
throw new Error('jsdom is unavailable on Workers');
|
||||
};
|
||||
export const JSDOM = unavailable;
|
||||
export const VirtualConsole = unavailable;
|
||||
@@ -0,0 +1,3 @@
|
||||
export const VM = () => {
|
||||
throw new Error('vm2 is unavailable on Workers');
|
||||
};
|
||||
@@ -0,0 +1,133 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { evaluateScriptCallback, evaluateScriptData } from './evaluate-script';
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('serialized script data', () => {
|
||||
it.each(['__DATA__', 'window.__DATA__', 'globalThis.__DATA__', 'self.__DATA__'])('normalizes the global target %s', async (target) => {
|
||||
await expect(evaluateScriptData(String.raw`window.__DATA__ = { title: "Hello\u0020世界", tags: ["a", "b"] };`, target)).resolves.toEqual({ title: 'Hello 世界', tags: ['a', 'b'] });
|
||||
});
|
||||
|
||||
it('resolves variables, nested assignments, and data-only unary and logical expressions', async () => {
|
||||
const source = 'var title="A & B", unused=analytics(); window.pixiv=window.pixiv||{}; pixiv.novel={title, count:-2, available:!0, absent:void 0};';
|
||||
await expect(evaluateScriptData(source, 'window.pixiv.novel')).resolves.toEqual({ title: 'A & B', count: -2, available: true, absent: undefined });
|
||||
});
|
||||
|
||||
it('reads parent-object assignments and static bracket targets', async () => {
|
||||
await expect(evaluateScriptData('window.pixiv={novel:{title:"Book"}};', 'self["pixiv"].novel')).resolves.toEqual({ title: 'Book' });
|
||||
});
|
||||
|
||||
it('reads Nuxt IIFE parameters and object mutations without changing outer bindings', async () => {
|
||||
const source = 'var a="outer";window.__NUXT__=(function(a,b,c){a.title=b;a.tags=c;return {data:[a,a],error:null}}({},"News",["tag"]));';
|
||||
const value = await evaluateScriptData<{ data: Array<{ title: string; tags: string[] }>; error: null }>(source, '__NUXT__');
|
||||
expect(value).toEqual({
|
||||
data: [
|
||||
{ title: 'News', tags: ['tag'] },
|
||||
{ title: 'News', tags: ['tag'] },
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
expect(value.data[0]).toBe(value.data[1]);
|
||||
await expect(evaluateScriptData(source, 'a')).resolves.toBe('outer');
|
||||
});
|
||||
|
||||
it('preserves repeated var declarations and evaluates unused IIFE arguments in order', async () => {
|
||||
const source = 'var a={value:1};var a;window.__DATA__=(function(x){return x})(a,a.value=2);';
|
||||
await expect(evaluateScriptData(source, '__DATA__')).resolves.toEqual({ value: 2 });
|
||||
});
|
||||
|
||||
it('does not call unrelated application code or access the host global object', async () => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
|
||||
const value = await evaluateScriptData('globalThis.__scriptDataSideEffect="changed";fetch("https://example.invalid/");window.__DATA__={ok:true};', '__DATA__');
|
||||
|
||||
expect(value).toEqual({ ok: true });
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect('__scriptDataSideEffect' in globalThis).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'window.__DATA__ = fetch("https://example.invalid/");',
|
||||
'window.__DATA__ = Function("return 42")();',
|
||||
'window.__DATA__ = { get secret() { return fetch("https://example.invalid/"); } };',
|
||||
'window.__DATA__ = (function(){ while(true){} return {}; })();',
|
||||
'window.__DATA__ = (function(){ return {}; })(fetch("https://example.invalid/"));',
|
||||
])('rejects unsupported expressions required by the target without executing them', async (source) => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
await expect(evaluateScriptData(source, '__DATA__')).rejects.toThrow(/Unsupported|ordinary/);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['__proto__', 'constructor', 'prototype'])('rejects the unsafe property %s', async (key) => {
|
||||
await expect(evaluateScriptData(`window.__DATA__={"${key}":{polluted:true}};`, '__DATA__')).rejects.toThrow('Unsafe script data property');
|
||||
await expect(evaluateScriptData(`var value={};value["${key}"]={polluted:true};window.__DATA__=value;`, '__DATA__')).rejects.toThrow('Unsafe script data property');
|
||||
expect(Object.hasOwn(Object.prototype, 'polluted')).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['window.__DATA__()', 'window[computed]', 'window.__DATA__; unrelated()', 'window.__proto__'])('rejects unsupported target paths: %s', async (target) => {
|
||||
await expect(evaluateScriptData('window.__DATA__={};', target)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('reports missing targets and unresolved variable dependencies', async () => {
|
||||
await expect(evaluateScriptData('unrelated();', '__DATA__')).rejects.toThrow('target was not found');
|
||||
await expect(evaluateScriptData('var data=unrelated();window.__DATA__=data;', '__DATA__')).rejects.toThrow('Unsupported script data expression');
|
||||
});
|
||||
|
||||
it.each(['if(true){window.__DATA__.value=2;}', 'Object.assign(window.__DATA__,{value:2});', 'window.__DATA__.value++;', 'window.__DATA__[unknown()]={};'])(
|
||||
'rejects unsupported code that could change extracted data: %s',
|
||||
async (suffix) => {
|
||||
await expect(evaluateScriptData(`window.__DATA__={value:1};${suffix}`, '__DATA__')).rejects.toThrow('Unsupported script data');
|
||||
}
|
||||
);
|
||||
|
||||
it.each(['if(true){window.__DATA__.newField=2;}', 'var ignored=Object.assign(window.__DATA__,{value:2});', 'window.ignored=Object.assign(window.__DATA__,{value:2});'])(
|
||||
'rejects mutations hidden in new properties and unused assignments: %s',
|
||||
async (suffix) => {
|
||||
await expect(evaluateScriptData(`window.__DATA__={};${suffix}`, '__DATA__')).rejects.toThrow('Unsupported script data');
|
||||
}
|
||||
);
|
||||
|
||||
it('rejects cyclic data and excessively large sources or sparse arrays', async () => {
|
||||
await expect(evaluateScriptData('var a={};a.self=a;window.__DATA__=a;', '__DATA__')).rejects.toThrow('Cyclic script data');
|
||||
await expect(evaluateScriptData(' '.repeat(2_000_001), '__DATA__')).rejects.toThrow('size limit');
|
||||
await expect(evaluateScriptData('window.__DATA__=(function(){var a=[];a[100000]={};return a})();', '__DATA__')).rejects.toThrow('bounded numeric indexes');
|
||||
});
|
||||
});
|
||||
|
||||
describe('serialized callback data', () => {
|
||||
it('extracts a DWR callback argument without invoking the callback', async () => {
|
||||
const callback = vi.fn();
|
||||
vi.stubGlobal('dwr', { engine: { _remoteHandleCallback: callback } });
|
||||
const source = '(function(){if(!window.dwr)return;var s0=[],s1={};s1.post={title:"Entry"};s0[0]=s1;dwr.engine._remoteHandleCallback("493053","0",s0);})();';
|
||||
|
||||
await expect(evaluateScriptCallback(source, 'dwr.engine._remoteHandleCallback')).resolves.toEqual([{ post: { title: 'Entry' } }]);
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('supports empty public DWR responses and explicit argument positions', async () => {
|
||||
const source = '//#DWR-INSERT\n//#DWR-REPLY\ndwr.engine._remoteHandleCallback("493053","0",[]);';
|
||||
await expect(evaluateScriptCallback(source, 'window.dwr.engine._remoteHandleCallback')).resolves.toEqual([]);
|
||||
await expect(evaluateScriptCallback(source, 'dwr.engine._remoteHandleCallback', 0)).resolves.toBe('493053');
|
||||
});
|
||||
|
||||
it('evaluates all callback arguments in order and does not keep an earlier successful capture after an invalid one', async () => {
|
||||
const source = 'var a={value:1};dwr.engine._remoteHandleCallback(a.value=2,"0",a);';
|
||||
await expect(evaluateScriptCallback(source, 'dwr.engine._remoteHandleCallback')).resolves.toEqual({ value: 2 });
|
||||
await expect(evaluateScriptCallback(`${source}dwr.engine._remoteHandleCallback("0","0",unknown());`, 'dwr.engine._remoteHandleCallback')).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing callbacks and external calls used as the data argument', async () => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
await expect(evaluateScriptCallback('unrelated();', 'dwr.engine._remoteHandleCallback')).rejects.toThrow('callback was not found');
|
||||
await expect(evaluateScriptCallback('dwr.engine._remoteHandleCallback("0","0",fetch("https://example.invalid/"));', 'dwr.engine._remoteHandleCallback')).rejects.toThrow();
|
||||
await expect(evaluateScriptCallback('', 'callback', -1)).rejects.toThrow('non-negative integer');
|
||||
await expect(evaluateScriptCallback('dwr["engine._remoteHandleCallback"]("0","0",[]);', 'dwr.engine._remoteHandleCallback')).rejects.toThrow('callback was not found');
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,21 +1,27 @@
|
||||
import { type AnyNode, parse } from 'acorn';
|
||||
import { type Node } from 'oxc-parser';
|
||||
|
||||
import { parseScriptSource } from './parse-js';
|
||||
|
||||
type DataValue = string | number | boolean | null | undefined | DataValue[] | DataObject | ScriptDataError;
|
||||
type DataObject = { [key: string]: DataValue };
|
||||
type Scope = { values: DataObject; parent?: Scope };
|
||||
type Reference = { object: DataObject | DataValue[]; key: string };
|
||||
type Reference = { object: DataObject; key: string };
|
||||
|
||||
const globalNames = new Set(['window', 'globalThis', 'self']);
|
||||
const unsafeKeys = new Set(['__proto__', 'prototype', 'constructor']);
|
||||
const maxScriptLength = 2_000_000;
|
||||
const maxSteps = 1e5;
|
||||
const maxSteps = 10000;
|
||||
const maxDepth = 100;
|
||||
|
||||
class ScriptDataError extends Error {}
|
||||
class UnsafePropertyError extends ScriptDataError {}
|
||||
|
||||
const propertyKey = (value: unknown): string => {
|
||||
if (typeof value !== 'string' && typeof value !== 'number') {
|
||||
const isKey = (value: DataValue): value is string | number => typeof value === 'string' || typeof value === 'number';
|
||||
const isNumber = (value: DataValue): value is number => typeof value === 'number';
|
||||
const isObject = (value: DataValue): value is DataObject | DataValue[] => typeof value === 'object' && value !== null && !(value instanceof ScriptDataError);
|
||||
|
||||
const propertyKey = (value: DataValue): string => {
|
||||
if (!isKey(value)) {
|
||||
throw new ScriptDataError('Script data property keys must be strings or numbers');
|
||||
}
|
||||
const key = String(value);
|
||||
@@ -25,12 +31,12 @@ const propertyKey = (value: unknown): string => {
|
||||
return key;
|
||||
};
|
||||
|
||||
const staticPath = (node: AnyNode): string[] => {
|
||||
const staticPath = (node: Node): string[] => {
|
||||
if (node.type === 'Identifier') {
|
||||
return [propertyKey(node.name)];
|
||||
}
|
||||
if (node.type === 'MemberExpression' && !node.optional) {
|
||||
const key = !node.computed && node.property.type === 'Identifier' ? node.property.name : node.property.type === 'Literal' ? node.property.value : undefined;
|
||||
const key = !node.computed && node.property.type === 'Identifier' ? node.property.name : node.property.type === 'Literal' && !('regex' in node.property) && !('bigint' in node.property) ? node.property.value : undefined;
|
||||
return [...staticPath(node.object), propertyKey(key)];
|
||||
}
|
||||
throw new ScriptDataError('Script data targets must be static property paths');
|
||||
@@ -38,8 +44,16 @@ const staticPath = (node: AnyNode): string[] => {
|
||||
|
||||
const normalizePath = (path: string[]) => (globalNames.has(path[0]) ? path.slice(1) : path);
|
||||
|
||||
const targetPath = (target: string): string[] => {
|
||||
const program = parse(target, { ecmaVersion: 'latest' });
|
||||
const parseProgram = async (source: string, errorMessage: string) => {
|
||||
const { program, errors } = await parseScriptSource(source);
|
||||
if (errors.length) {
|
||||
throw new ScriptDataError(errorMessage);
|
||||
}
|
||||
return program;
|
||||
};
|
||||
|
||||
const targetPath = async (target: string): Promise<string[]> => {
|
||||
const program = await parseProgram(target, 'Script data targets must be a single static property path');
|
||||
if (program.body.length !== 1 || program.body[0].type !== 'ExpressionStatement') {
|
||||
throw new ScriptDataError('Script data targets must be a single static property path');
|
||||
}
|
||||
@@ -50,8 +64,6 @@ const targetPath = (target: string): string[] => {
|
||||
return path;
|
||||
};
|
||||
|
||||
const isObject = (value: DataValue): value is DataObject | DataValue[] => typeof value === 'object' && value !== null && !(value instanceof ScriptDataError);
|
||||
|
||||
const readProperty = (object: DataValue, key: string): DataValue => {
|
||||
if (object instanceof ScriptDataError) {
|
||||
throw object;
|
||||
@@ -63,25 +75,20 @@ const readProperty = (object: DataValue, key: string): DataValue => {
|
||||
};
|
||||
|
||||
/** Reads serialized data with a limited AST vocabulary; it never executes JavaScript. */
|
||||
class ScriptDataReader {
|
||||
private root: Scope = { values: Object.create(null) };
|
||||
private steps = 0;
|
||||
private poisonedAssignments = new WeakSet<AnyNode>();
|
||||
private captured = false;
|
||||
private callbackValue: DataValue;
|
||||
const readScriptData = async <T>(source: string, target: string[], argumentIndex?: number): Promise<T> => {
|
||||
const root: Scope = { values: Object.create(null) };
|
||||
const poisonedAssignments = new WeakSet<Node>();
|
||||
let steps = 0;
|
||||
let captured = false;
|
||||
let callbackValue: DataValue;
|
||||
|
||||
constructor(
|
||||
private target: string[],
|
||||
private argumentIndex?: number
|
||||
) {}
|
||||
|
||||
private step(depth: number) {
|
||||
if (++this.steps > maxSteps || depth > maxDepth) {
|
||||
const step = (depth: number) => {
|
||||
if (++steps > maxSteps || depth > maxDepth) {
|
||||
throw new UnsafePropertyError('Script data exceeds the parsing complexity limit');
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private identifier(name: string, scope: Scope): DataValue {
|
||||
const identifier = (name: string, scope: Scope): DataValue => {
|
||||
propertyKey(name);
|
||||
for (let current: Scope | undefined = scope; current; current = current.parent) {
|
||||
if (Object.hasOwn(current.values, name)) {
|
||||
@@ -93,15 +100,15 @@ class ScriptDataReader {
|
||||
}
|
||||
}
|
||||
if (globalNames.has(name)) {
|
||||
return this.root.values;
|
||||
return root.values;
|
||||
}
|
||||
if (name === 'undefined') {
|
||||
return undefined;
|
||||
}
|
||||
throw new ScriptDataError(`Unknown script data variable: ${name}`);
|
||||
}
|
||||
};
|
||||
|
||||
private reference(node: AnyNode, scope: Scope, depth: number): Reference {
|
||||
const reference = (node: Node, scope: Scope, depth: number): Reference => {
|
||||
if (node.type === 'Identifier') {
|
||||
const key = propertyKey(node.name);
|
||||
if (globalNames.has(key)) {
|
||||
@@ -116,77 +123,77 @@ class ScriptDataReader {
|
||||
if (node.type !== 'MemberExpression' || node.optional) {
|
||||
throw new ScriptDataError('Unsupported script data assignment target');
|
||||
}
|
||||
const object = this.evaluate(node.object, scope, depth + 1);
|
||||
const key = propertyKey(!node.computed && node.property.type === 'Identifier' ? node.property.name : this.evaluate(node.property, scope, depth + 1));
|
||||
const object = evaluate(node.object, scope, depth + 1);
|
||||
const key = propertyKey(!node.computed && node.property.type === 'Identifier' ? node.property.name : evaluate(node.property, scope, depth + 1));
|
||||
if (!isObject(object)) {
|
||||
throw new ScriptDataError('Cannot assign a property of non-object script data');
|
||||
}
|
||||
if (Array.isArray(object) && (!/^(?:0|[1-9]\d*)$/.test(key) || Number(key) >= maxSteps)) {
|
||||
throw new ScriptDataError('Script data arrays require bounded numeric indexes');
|
||||
}
|
||||
return { object, key };
|
||||
}
|
||||
return { object: object as DataObject, key };
|
||||
};
|
||||
|
||||
private evaluate(node: AnyNode, scope: Scope, depth: number): DataValue {
|
||||
this.step(depth);
|
||||
const evaluate = (node: Node, scope: Scope, depth: number): DataValue => {
|
||||
step(depth);
|
||||
switch (node.type) {
|
||||
case 'Literal':
|
||||
if (node.regex || node.bigint) {
|
||||
if ('regex' in node || 'bigint' in node) {
|
||||
break;
|
||||
}
|
||||
return node.value as string | number | boolean | null;
|
||||
return node.value;
|
||||
case 'Identifier':
|
||||
return this.identifier(node.name, scope);
|
||||
return identifier(node.name, scope);
|
||||
case 'ArrayExpression':
|
||||
return node.elements.map((element) => (element ? this.evaluate(element, scope, depth + 1) : undefined));
|
||||
return node.elements.map((element) => (element ? evaluate(element, scope, depth + 1) : undefined));
|
||||
case 'ObjectExpression': {
|
||||
const value: DataObject = Object.create(null);
|
||||
for (const property of node.properties) {
|
||||
if (property.type !== 'Property' || property.kind !== 'init' || property.method) {
|
||||
throw new ScriptDataError('Only ordinary script data object properties are supported');
|
||||
}
|
||||
const key = propertyKey(!property.computed && property.key.type === 'Identifier' ? property.key.name : this.evaluate(property.key, scope, depth + 1));
|
||||
value[key] = this.evaluate(property.value, scope, depth + 1);
|
||||
const key = propertyKey(!property.computed && property.key.type === 'Identifier' ? property.key.name : evaluate(property.key, scope, depth + 1));
|
||||
value[key] = evaluate(property.value, scope, depth + 1);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
case 'MemberExpression': {
|
||||
const key = propertyKey(!node.computed && node.property.type === 'Identifier' ? node.property.name : this.evaluate(node.property, scope, depth + 1));
|
||||
return readProperty(this.evaluate(node.object, scope, depth + 1), key);
|
||||
const key = propertyKey(!node.computed && node.property.type === 'Identifier' ? node.property.name : evaluate(node.property, scope, depth + 1));
|
||||
return readProperty(evaluate(node.object, scope, depth + 1), key);
|
||||
}
|
||||
case 'UnaryExpression': {
|
||||
const value = this.evaluate(node.argument, scope, depth + 1);
|
||||
const value = evaluate(node.argument, scope, depth + 1);
|
||||
if (node.operator === 'void') {
|
||||
return undefined;
|
||||
}
|
||||
if (node.operator === '!') {
|
||||
return !value;
|
||||
}
|
||||
if (typeof value === 'number' && (node.operator === '-' || node.operator === '+')) {
|
||||
if (isNumber(value) && (node.operator === '-' || node.operator === '+')) {
|
||||
return node.operator === '-' ? -value : value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'LogicalExpression': {
|
||||
const left = this.evaluate(node.left, scope, depth + 1);
|
||||
const left = evaluate(node.left, scope, depth + 1);
|
||||
if ((node.operator === '||' && left) || (node.operator === '&&' && !left) || (node.operator === '??' && left !== undefined && left !== null)) {
|
||||
return left;
|
||||
}
|
||||
return this.evaluate(node.right, scope, depth + 1);
|
||||
return evaluate(node.right, scope, depth + 1);
|
||||
}
|
||||
case 'AssignmentExpression': {
|
||||
const { object, key } = this.reference(node.left, scope, depth + 1);
|
||||
const { object, key } = reference(node.left, scope, depth + 1);
|
||||
try {
|
||||
if (node.operator !== '=') {
|
||||
throw new ScriptDataError('Only simple script data assignments are supported');
|
||||
}
|
||||
const value = this.evaluate(node.right, scope, depth + 1);
|
||||
(object as DataObject)[key] = value;
|
||||
const value = evaluate(node.right, scope, depth + 1);
|
||||
object[key] = value;
|
||||
return value;
|
||||
} catch (error) {
|
||||
if (error instanceof ScriptDataError) {
|
||||
(object as DataObject)[key] = error;
|
||||
this.poisonedAssignments.add(node);
|
||||
object[key] = error;
|
||||
poisonedAssignments.add(node);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -194,27 +201,27 @@ class ScriptDataReader {
|
||||
case 'SequenceExpression': {
|
||||
let value: DataValue;
|
||||
for (const expression of node.expressions) {
|
||||
value = this.evaluate(expression, scope, depth + 1);
|
||||
value = evaluate(expression, scope, depth + 1);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
case 'CallExpression': {
|
||||
if (this.argumentIndex !== undefined && this.matchesCallback(node.callee)) {
|
||||
this.captured = false;
|
||||
this.callbackValue = undefined;
|
||||
const args = node.arguments.map((argument) => this.evaluate(argument, scope, depth + 1));
|
||||
if (this.argumentIndex >= args.length) {
|
||||
if (argumentIndex !== undefined && matchesCallback(node.callee)) {
|
||||
captured = false;
|
||||
callbackValue = undefined;
|
||||
const args = node.arguments.map((argument) => evaluate(argument, scope, depth + 1));
|
||||
if (argumentIndex >= args.length) {
|
||||
throw new ScriptDataError('Script data callback is missing its data argument');
|
||||
}
|
||||
this.callbackValue = args[this.argumentIndex];
|
||||
this.captured = true;
|
||||
callbackValue = args[argumentIndex];
|
||||
captured = true;
|
||||
return undefined;
|
||||
}
|
||||
const fn = node.callee;
|
||||
if ((fn.type !== 'FunctionExpression' && fn.type !== 'ArrowFunctionExpression') || fn.async || fn.generator) {
|
||||
break;
|
||||
}
|
||||
const args = node.arguments.map((argument) => this.evaluate(argument, scope, depth + 1));
|
||||
const args = node.arguments.map((argument) => evaluate(argument, scope, depth + 1));
|
||||
const local: Scope = { values: Object.create(null), parent: scope };
|
||||
for (const [index, parameter] of fn.params.entries()) {
|
||||
if (parameter.type !== 'Identifier' || globalNames.has(parameter.name)) {
|
||||
@@ -223,28 +230,31 @@ class ScriptDataReader {
|
||||
const key = propertyKey(parameter.name);
|
||||
local.values[key] = args[index];
|
||||
}
|
||||
return fn.body.type === 'BlockStatement' ? this.statements(fn.body.body, local, depth + 1, true)?.value : this.evaluate(fn.body, local, depth + 1);
|
||||
if (!fn.body) {
|
||||
break;
|
||||
}
|
||||
return fn.body.type === 'BlockStatement' ? statements(fn.body.body, local, depth + 1, true)?.value : evaluate(fn.body, local, depth + 1);
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
throw new ScriptDataError(`Unsupported script data expression: ${node.type}`);
|
||||
}
|
||||
};
|
||||
|
||||
private matchesCallback(node: AnyNode): boolean {
|
||||
const matchesCallback = (node: Node): boolean => {
|
||||
try {
|
||||
const path = normalizePath(staticPath(node));
|
||||
return path.length === this.target.length && path.every((key, index) => key === this.target[index]);
|
||||
return path.length === target.length && path.every((key, index) => key === target[index]);
|
||||
} catch (error) {
|
||||
if (error instanceof UnsafePropertyError) {
|
||||
throw error;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private isCallbackGuard(node: AnyNode): boolean {
|
||||
if (this.argumentIndex === undefined || node.type !== 'IfStatement' || node.alternate || node.test.type !== 'UnaryExpression' || node.test.operator !== '!') {
|
||||
const isCallbackGuard = (node: Node): boolean => {
|
||||
if (argumentIndex === undefined || node.type !== 'IfStatement' || node.alternate || node.test.type !== 'UnaryExpression' || node.test.operator !== '!') {
|
||||
return false;
|
||||
}
|
||||
const consequent = node.consequent.type === 'BlockStatement' && node.consequent.body.length === 1 ? node.consequent.body[0] : node.consequent;
|
||||
@@ -252,18 +262,18 @@ class ScriptDataReader {
|
||||
return false;
|
||||
}
|
||||
const path = normalizePath(staticPath(node.test.argument));
|
||||
return path.length > 0 && path.length < this.target.length && path.every((key, index) => key === this.target[index]);
|
||||
}
|
||||
return path.length > 0 && path.length < target.length && path.every((key, index) => key === target[index]);
|
||||
};
|
||||
|
||||
private referencesKnownData(node: AnyNode, scope: Scope, depth: number): boolean {
|
||||
this.step(depth);
|
||||
const referencesKnownData = (node: Node, scope: Scope, depth: number): boolean => {
|
||||
step(depth);
|
||||
if (node.type === 'Identifier' || node.type === 'MemberExpression') {
|
||||
try {
|
||||
const path = staticPath(node);
|
||||
let value = this.identifier(path[0], scope);
|
||||
let value = identifier(path[0], scope);
|
||||
for (const key of path.slice(1)) {
|
||||
if (!isObject(value) || !Object.hasOwn(value, key)) {
|
||||
return value !== this.root.values;
|
||||
return value !== root.values;
|
||||
}
|
||||
value = readProperty(value, key);
|
||||
}
|
||||
@@ -277,15 +287,18 @@ class ScriptDataReader {
|
||||
}
|
||||
}
|
||||
}
|
||||
return Object.values(node).some((value) => {
|
||||
return Object.entries(node).some(([key, value]) => {
|
||||
if (key === 'parent') {
|
||||
return false;
|
||||
}
|
||||
const children = Array.isArray(value) ? value : [value];
|
||||
return children.some((child) => child && typeof child === 'object' && typeof child.type === 'string' && this.referencesKnownData(child, scope, depth + 1));
|
||||
return children.some((child) => child?.type && referencesKnownData(child, scope, depth + 1));
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
private statements(nodes: AnyNode[], scope: Scope, depth: number, strict: boolean): { value: DataValue } | undefined {
|
||||
const statements = (nodes: Node[], scope: Scope, depth: number, strict: boolean): { value: DataValue } | undefined => {
|
||||
for (const node of nodes) {
|
||||
this.step(depth);
|
||||
step(depth);
|
||||
try {
|
||||
switch (node.type) {
|
||||
case 'VariableDeclaration':
|
||||
@@ -296,10 +309,10 @@ class ScriptDataReader {
|
||||
const key = propertyKey(declaration.id.name);
|
||||
try {
|
||||
if (declaration.init || !Object.hasOwn(scope.values, key)) {
|
||||
scope.values[key] = declaration.init ? this.evaluate(declaration.init, scope, depth + 1) : undefined;
|
||||
scope.values[key] = declaration.init ? evaluate(declaration.init, scope, depth + 1) : undefined;
|
||||
}
|
||||
} catch (error) {
|
||||
if (strict || !(error instanceof ScriptDataError) || error instanceof UnsafePropertyError || (declaration.init && this.referencesKnownData(declaration.init, scope, depth + 1))) {
|
||||
if (strict || !(error instanceof ScriptDataError) || error instanceof UnsafePropertyError || (declaration.init && referencesKnownData(declaration.init, scope, depth + 1))) {
|
||||
throw error;
|
||||
}
|
||||
scope.values[key] = error;
|
||||
@@ -307,14 +320,14 @@ class ScriptDataReader {
|
||||
}
|
||||
break;
|
||||
case 'ExpressionStatement':
|
||||
this.evaluate(node.expression, scope, depth + 1);
|
||||
evaluate(node.expression, scope, depth + 1);
|
||||
break;
|
||||
case 'ReturnStatement':
|
||||
return { value: node.argument ? this.evaluate(node.argument, scope, depth + 1) : undefined };
|
||||
return { value: node.argument ? evaluate(node.argument, scope, depth + 1) : undefined };
|
||||
case 'EmptyStatement':
|
||||
break;
|
||||
default:
|
||||
if (!this.isCallbackGuard(node)) {
|
||||
if (!isCallbackGuard(node)) {
|
||||
throw new ScriptDataError(`Unsupported script data statement: ${node.type}`);
|
||||
}
|
||||
}
|
||||
@@ -326,17 +339,17 @@ class ScriptDataReader {
|
||||
// Failed simple assignments already poison their destination. Other unsupported code
|
||||
// must not silently leave stale data when it references a known serialized value.
|
||||
const isPoisonedAssignment =
|
||||
node.type === 'ExpressionStatement' && node.expression.type === 'AssignmentExpression' && this.poisonedAssignments.has(node.expression) && !this.referencesKnownData(node.expression.right, scope, depth + 1);
|
||||
if (!isPoisonedAssignment && this.referencesKnownData(node, scope, depth + 1)) {
|
||||
node.type === 'ExpressionStatement' && node.expression.type === 'AssignmentExpression' && poisonedAssignments.has(node.expression) && !referencesKnownData(node.expression.right, scope, depth + 1);
|
||||
if (!isPoisonedAssignment && referencesKnownData(node, scope, depth + 1)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
private validate(value: DataValue, ancestors = new Set<DataValue>(), depth = 0): void {
|
||||
this.step(depth);
|
||||
const validate = (value: DataValue, ancestors = new Set<DataValue>(), depth = 0): void => {
|
||||
step(depth);
|
||||
if (value instanceof ScriptDataError) {
|
||||
throw value;
|
||||
}
|
||||
@@ -346,45 +359,47 @@ class ScriptDataReader {
|
||||
}
|
||||
ancestors.add(value);
|
||||
for (const child of Object.values(value)) {
|
||||
this.validate(child, ancestors, depth + 1);
|
||||
validate(child, ancestors, depth + 1);
|
||||
}
|
||||
ancestors.delete(value);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
read(source: string): DataValue {
|
||||
if (source.length > maxScriptLength) {
|
||||
throw new ScriptDataError('Script data source exceeds the size limit');
|
||||
}
|
||||
this.statements(parse(source, { ecmaVersion: 'latest' }).body, this.root, 0, false);
|
||||
let value: DataValue = this.root.values;
|
||||
if (this.argumentIndex === undefined) {
|
||||
for (const key of this.target) {
|
||||
if (!isObject(value) || !Object.hasOwn(value, key)) {
|
||||
if (value instanceof ScriptDataError) {
|
||||
throw value;
|
||||
}
|
||||
throw new ScriptDataError('Script data target was not found');
|
||||
if (source.length > maxScriptLength) {
|
||||
throw new ScriptDataError('Script data source exceeds the size limit');
|
||||
}
|
||||
statements((await parseProgram(source, 'Script data source could not be parsed')).body, root, 0, false);
|
||||
let value: DataValue = root.values;
|
||||
if (argumentIndex === undefined) {
|
||||
for (const key of target) {
|
||||
if (!isObject(value) || !Object.hasOwn(value, key)) {
|
||||
if (value instanceof ScriptDataError) {
|
||||
throw value;
|
||||
}
|
||||
value = readProperty(value, key);
|
||||
throw new ScriptDataError('Script data target was not found');
|
||||
}
|
||||
} else {
|
||||
if (!this.captured) {
|
||||
throw new ScriptDataError('Script data callback was not found or could not be parsed');
|
||||
}
|
||||
value = this.callbackValue;
|
||||
value = readProperty(value, key);
|
||||
}
|
||||
this.validate(value);
|
||||
return value;
|
||||
} else {
|
||||
if (!captured) {
|
||||
throw new ScriptDataError('Script data callback was not found or could not be parsed');
|
||||
}
|
||||
value = callbackValue;
|
||||
}
|
||||
}
|
||||
validate(value);
|
||||
return value as T;
|
||||
};
|
||||
|
||||
export const parseScriptData = <T = unknown>(source: string, target: string): T => new ScriptDataReader(targetPath(target)).read(source) as T;
|
||||
export const evaluateScriptData = async <T = unknown>(source: string, target: string): Promise<T> => {
|
||||
const path = await targetPath(target);
|
||||
return readScriptData<T>(source, path);
|
||||
};
|
||||
|
||||
/** Extracts a serialized callback argument without invoking the callback or any external function. */
|
||||
export const parseScriptCallback = <T = unknown>(source: string, callbackPath: string, argumentIndex = 2): T => {
|
||||
export const evaluateScriptCallback = async <T = unknown>(source: string, callbackPath: string, argumentIndex = 2): Promise<T> => {
|
||||
if (!Number.isSafeInteger(argumentIndex) || argumentIndex < 0) {
|
||||
throw new ScriptDataError('Script data callback argument index must be a non-negative integer');
|
||||
}
|
||||
return new ScriptDataReader(targetPath(callbackPath), argumentIndex).read(source) as T;
|
||||
const path = await targetPath(callbackPath);
|
||||
return readScriptData<T>(source, path, argumentIndex);
|
||||
};
|
||||
@@ -0,0 +1 @@
|
||||
import './evaluate-script.test';
|
||||
@@ -0,0 +1 @@
|
||||
import './parse-date-in-timezone.test';
|
||||
@@ -0,0 +1,11 @@
|
||||
import { type OxcError, parseSync, type Program, rawTransferSupported } from 'oxc-parser';
|
||||
|
||||
export const parseScriptSource = (source: string): { program: Program; errors: OxcError[] } => {
|
||||
const { program, errors } = parseSync('script.js', source, {
|
||||
lang: 'js',
|
||||
sourceType: 'script',
|
||||
preserveParens: false,
|
||||
experimentalRawTransfer: rawTransferSupported(),
|
||||
} as never);
|
||||
return { program, errors };
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import parserWasm from '@oxc-parser/binding-wasm32-wasip1/wasm.wasm';
|
||||
import { instantiate } from '@oxc-parser/binding-wasm32-wasip1/workerd';
|
||||
import { type OxcError, type Program } from 'oxc-parser';
|
||||
import { wrap } from 'oxc-parser/src-js/wrap.js';
|
||||
|
||||
let bindingPromise: ReturnType<typeof instantiate> | undefined;
|
||||
/**
|
||||
* @see https://github.com/oxc-project/oxc/pull/25442 still bleeding edge
|
||||
* */
|
||||
export const parseScriptSource = async (source: string): Promise<{ program: Program; errors: OxcError[] }> => {
|
||||
bindingPromise ??= instantiate(parserWasm);
|
||||
const binding = await bindingPromise;
|
||||
const { program, errors } = wrap(
|
||||
binding.parseSync('script.js', source, {
|
||||
lang: 'js',
|
||||
sourceType: 'script',
|
||||
preserveParens: false,
|
||||
}) as never
|
||||
);
|
||||
return { program, errors };
|
||||
};
|
||||
@@ -1,134 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { parseScriptCallback, parseScriptData } from './parse-script-data';
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('serialized script data', () => {
|
||||
it.each(['__DATA__', 'window.__DATA__', 'globalThis.__DATA__', 'self.__DATA__'])('normalizes the global target %s', (target) => {
|
||||
expect(parseScriptData(String.raw`window.__DATA__ = { title: "Hello\u0020世界", tags: ["a", "b"] };`, target)).toEqual({ title: 'Hello 世界', tags: ['a', 'b'] });
|
||||
});
|
||||
|
||||
it('resolves variables, nested assignments, and data-only unary and logical expressions', () => {
|
||||
const source = 'var title="A & B", unused=analytics(); window.pixiv=window.pixiv||{}; pixiv.novel={title, count:-2, available:!0, absent:void 0};';
|
||||
expect(parseScriptData(source, 'window.pixiv.novel')).toEqual({ title: 'A & B', count: -2, available: true, absent: undefined });
|
||||
});
|
||||
|
||||
it('reads parent-object assignments and static bracket targets', () => {
|
||||
expect(parseScriptData('window.pixiv={novel:{title:"Book"}};', 'self["pixiv"].novel')).toEqual({ title: 'Book' });
|
||||
});
|
||||
|
||||
it('reads Nuxt IIFE parameters and object mutations without changing outer bindings', () => {
|
||||
const source = 'var a="outer";window.__NUXT__=(function(a,b,c){a.title=b;a.tags=c;return {data:[a,a],error:null}}({},"News",["tag"]));';
|
||||
const value = parseScriptData<{ data: Array<{ title: string; tags: string[] }>; error: null }>(source, '__NUXT__');
|
||||
expect(value).toEqual({
|
||||
data: [
|
||||
{ title: 'News', tags: ['tag'] },
|
||||
{ title: 'News', tags: ['tag'] },
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
expect(value.data[0]).toBe(value.data[1]);
|
||||
expect(parseScriptData(source, 'a')).toBe('outer');
|
||||
});
|
||||
|
||||
it('preserves repeated var declarations and evaluates unused IIFE arguments in order', () => {
|
||||
const source = 'var a={value:1};var a;window.__DATA__=(function(x){return x})(a,a.value=2);';
|
||||
expect(parseScriptData(source, '__DATA__')).toEqual({ value: 2 });
|
||||
});
|
||||
|
||||
it('does not call unrelated application code or access the host global object', () => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
vi.stubGlobal('__scriptDataSideEffect', 'unchanged');
|
||||
|
||||
const value = parseScriptData('globalThis.__scriptDataSideEffect="changed";fetch("https://example.invalid/");window.__DATA__={ok:true};', '__DATA__');
|
||||
|
||||
expect(value).toEqual({ ok: true });
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(Reflect.get(globalThis, '__scriptDataSideEffect')).toBe('unchanged');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'window.__DATA__ = fetch("https://example.invalid/");',
|
||||
'window.__DATA__ = Function("return 42")();',
|
||||
'window.__DATA__ = { get secret() { return fetch("https://example.invalid/"); } };',
|
||||
'window.__DATA__ = (function(){ while(true){} return {}; })();',
|
||||
'window.__DATA__ = (function(){ return {}; })(fetch("https://example.invalid/"));',
|
||||
])('rejects unsupported expressions required by the target without executing them', (source) => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
expect(() => parseScriptData(source, '__DATA__')).toThrow(/Unsupported|ordinary/);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['__proto__', 'constructor', 'prototype'])('rejects the unsafe property %s', (key) => {
|
||||
expect(() => parseScriptData(`window.__DATA__={"${key}":{polluted:true}};`, '__DATA__')).toThrow('Unsafe script data property');
|
||||
expect(() => parseScriptData(`var value={};value["${key}"]={polluted:true};window.__DATA__=value;`, '__DATA__')).toThrow('Unsafe script data property');
|
||||
expect(Object.hasOwn(Object.prototype, 'polluted')).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['window.__DATA__()', 'window[computed]', 'window.__DATA__; unrelated()', 'window.__proto__'])('rejects unsupported target paths: %s', (target) => {
|
||||
expect(() => parseScriptData('window.__DATA__={};', target)).toThrow();
|
||||
});
|
||||
|
||||
it('reports missing targets and unresolved variable dependencies', () => {
|
||||
expect(() => parseScriptData('unrelated();', '__DATA__')).toThrow('target was not found');
|
||||
expect(() => parseScriptData('var data=unrelated();window.__DATA__=data;', '__DATA__')).toThrow('Unsupported script data expression');
|
||||
});
|
||||
|
||||
it.each(['if(true){window.__DATA__.value=2;}', 'Object.assign(window.__DATA__,{value:2});', 'window.__DATA__.value++;', 'window.__DATA__[unknown()]={};'])(
|
||||
'rejects unsupported code that could change extracted data: %s',
|
||||
(suffix) => {
|
||||
expect(() => parseScriptData(`window.__DATA__={value:1};${suffix}`, '__DATA__')).toThrow('Unsupported script data');
|
||||
}
|
||||
);
|
||||
|
||||
it.each(['if(true){window.__DATA__.newField=2;}', 'var ignored=Object.assign(window.__DATA__,{value:2});', 'window.ignored=Object.assign(window.__DATA__,{value:2});'])(
|
||||
'rejects mutations hidden in new properties and unused assignments: %s',
|
||||
(suffix) => {
|
||||
expect(() => parseScriptData(`window.__DATA__={};${suffix}`, '__DATA__')).toThrow('Unsupported script data');
|
||||
}
|
||||
);
|
||||
|
||||
it('rejects cyclic data and excessively large sources or sparse arrays', () => {
|
||||
expect(() => parseScriptData('var a={};a.self=a;window.__DATA__=a;', '__DATA__')).toThrow('Cyclic script data');
|
||||
expect(() => parseScriptData(' '.repeat(2_000_001), '__DATA__')).toThrow('size limit');
|
||||
expect(() => parseScriptData('window.__DATA__=(function(){var a=[];a[100000]={};return a})();', '__DATA__')).toThrow('bounded numeric indexes');
|
||||
});
|
||||
});
|
||||
|
||||
describe('serialized callback data', () => {
|
||||
it('extracts a DWR callback argument without invoking the callback', () => {
|
||||
const callback = vi.fn();
|
||||
vi.stubGlobal('dwr', { engine: { _remoteHandleCallback: callback } });
|
||||
const source = '(function(){if(!window.dwr)return;var s0=[],s1={};s1.post={title:"Entry"};s0[0]=s1;dwr.engine._remoteHandleCallback("493053","0",s0);})();';
|
||||
|
||||
expect(parseScriptCallback(source, 'dwr.engine._remoteHandleCallback')).toEqual([{ post: { title: 'Entry' } }]);
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('supports empty public DWR responses and explicit argument positions', () => {
|
||||
const source = '//#DWR-INSERT\n//#DWR-REPLY\ndwr.engine._remoteHandleCallback("493053","0",[]);';
|
||||
expect(parseScriptCallback(source, 'window.dwr.engine._remoteHandleCallback')).toEqual([]);
|
||||
expect(parseScriptCallback(source, 'dwr.engine._remoteHandleCallback', 0)).toBe('493053');
|
||||
});
|
||||
|
||||
it('evaluates all callback arguments in order and does not keep an earlier successful capture after an invalid one', () => {
|
||||
const source = 'var a={value:1};dwr.engine._remoteHandleCallback(a.value=2,"0",a);';
|
||||
expect(parseScriptCallback(source, 'dwr.engine._remoteHandleCallback')).toEqual({ value: 2 });
|
||||
expect(() => parseScriptCallback(`${source}dwr.engine._remoteHandleCallback("0","0",unknown());`, 'dwr.engine._remoteHandleCallback')).toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing callbacks and external calls used as the data argument', () => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
expect(() => parseScriptCallback('unrelated();', 'dwr.engine._remoteHandleCallback')).toThrow('callback was not found');
|
||||
expect(() => parseScriptCallback('dwr.engine._remoteHandleCallback("0","0",fetch("https://example.invalid/"));', 'dwr.engine._remoteHandleCallback')).toThrow();
|
||||
expect(() => parseScriptCallback('', 'callback', -1)).toThrow('non-negative integer');
|
||||
expect(() => parseScriptCallback('dwr["engine._remoteHandleCallback"]("0","0",[]);', 'dwr.engine._remoteHandleCallback')).toThrow('callback was not found');
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,18 +1,18 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { getPlaywrightPage, setBrowserBinding } from '../lib/utils/playwright.worker';
|
||||
import { getPlaywrightPage, setBrowserBinding } from './playwright.worker';
|
||||
|
||||
const mocks = vi.hoisted(() => ({ endpoint: undefined as string | undefined, launch: vi.fn(), newContext: vi.fn(), newPage: vi.fn(), goto: vi.fn(), close: vi.fn(), contextClose: vi.fn(), connect: vi.fn() }));
|
||||
vi.mock('@cloudflare/playwright', () => ({ launch: mocks.launch }));
|
||||
vi.mock('../lib/utils/playwright-remote.worker', () => ({ connectRemotePlaywright: mocks.connect, setPlaywrightServiceBinding: vi.fn() }));
|
||||
vi.mock('../lib/config', () => ({
|
||||
vi.mock('./playwright-remote.worker', () => ({ connectRemotePlaywright: mocks.connect, setPlaywrightServiceBinding: vi.fn() }));
|
||||
vi.mock('@/config', () => ({
|
||||
config: {
|
||||
get playwrightWSEndpoint() {
|
||||
return mocks.endpoint;
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock('../lib/utils/logger', () => ({ default: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() } }));
|
||||
vi.mock('./logger', () => ({ default: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() } }));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
+2
-2
@@ -76,10 +76,10 @@
|
||||
"@opentelemetry/sdk-metrics": "2.11.0",
|
||||
"@opentelemetry/sdk-trace-base": "2.11.0",
|
||||
"@opentelemetry/semantic-conventions": "1.43.0",
|
||||
"@oxc-parser/binding-wasm32-wasip1": "0.0.1",
|
||||
"@rss3/sdk": "0.0.25",
|
||||
"@scalar/hono-api-reference": "0.12.1",
|
||||
"@sentry/node": "10.74.0",
|
||||
"acorn": "8.18.0",
|
||||
"cheerio": "1.2.0",
|
||||
"city-timezones": "1.3.4",
|
||||
"cross-env": "10.1.0",
|
||||
@@ -114,6 +114,7 @@
|
||||
"oauth-1.0a": "2.2.6",
|
||||
"ofetch": "1.5.1",
|
||||
"otplib": "13.5.0",
|
||||
"oxc-parser": "0.149.0",
|
||||
"p-map": "7.0.7",
|
||||
"pac-proxy-agent": "9.1.0",
|
||||
"patchright": "1.63.0",
|
||||
@@ -190,7 +191,6 @@
|
||||
"mockdate": "3.0.5",
|
||||
"msw": "2.15.0",
|
||||
"node-network-devtools": "1.0.30",
|
||||
"oxc-parser": "0.149.0",
|
||||
"oxfmt": "0.67.0",
|
||||
"oxlint": "1.82.0",
|
||||
"oxlint-plugin-eslint": "1.82.0",
|
||||
|
||||
Generated
+79
-61
@@ -82,6 +82,9 @@ importers:
|
||||
'@opentelemetry/semantic-conventions':
|
||||
specifier: 1.43.0
|
||||
version: 1.43.0
|
||||
'@oxc-parser/binding-wasm32-wasip1':
|
||||
specifier: 0.0.1
|
||||
version: 0.0.1
|
||||
'@rss3/sdk':
|
||||
specifier: 0.0.25
|
||||
version: 0.0.25
|
||||
@@ -91,9 +94,6 @@ importers:
|
||||
'@sentry/node':
|
||||
specifier: 10.74.0
|
||||
version: 10.74.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.222.0(@opentelemetry/api@1.9.1))
|
||||
acorn:
|
||||
specifier: 8.18.0
|
||||
version: 8.18.0
|
||||
cheerio:
|
||||
specifier: 1.2.0
|
||||
version: 1.2.0
|
||||
@@ -196,6 +196,9 @@ importers:
|
||||
otplib:
|
||||
specifier: 13.5.0
|
||||
version: 13.5.0
|
||||
oxc-parser:
|
||||
specifier: 0.149.0
|
||||
version: 0.149.0
|
||||
p-map:
|
||||
specifier: 7.0.7
|
||||
version: 7.0.7
|
||||
@@ -419,9 +422,6 @@ importers:
|
||||
node-network-devtools:
|
||||
specifier: 1.0.30
|
||||
version: 1.0.30(undici@8.10.2)(utf-8-validate@5.0.10)
|
||||
oxc-parser:
|
||||
specifier: 0.149.0
|
||||
version: 0.149.0
|
||||
oxfmt:
|
||||
specifier: 0.67.0
|
||||
version: 0.67.0
|
||||
@@ -687,15 +687,24 @@ packages:
|
||||
'@emnapi/core@1.11.2':
|
||||
resolution: {integrity: sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==}
|
||||
|
||||
'@emnapi/core@2.0.0-alpha.3':
|
||||
resolution: {integrity: sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==}
|
||||
|
||||
'@emnapi/runtime@1.11.2':
|
||||
resolution: {integrity: sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==}
|
||||
|
||||
'@emnapi/runtime@1.11.3':
|
||||
resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==}
|
||||
|
||||
'@emnapi/runtime@2.0.0-alpha.3':
|
||||
resolution: {integrity: sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==}
|
||||
|
||||
'@emnapi/wasi-threads@1.2.2':
|
||||
resolution: {integrity: sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==}
|
||||
|
||||
'@emnapi/wasi-threads@2.0.1':
|
||||
resolution: {integrity: sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==}
|
||||
|
||||
'@epic-web/invariant@1.0.0':
|
||||
resolution: {integrity: sha512-lrTPqgvfFQtR/eY/qkIzp98OGdNJu0m5ji3q/nJI8v3SXkRKEnWiOxMmbvcSoAIzv/cGiuvRy57k4suKQSAdwA==}
|
||||
|
||||
@@ -1601,6 +1610,13 @@ packages:
|
||||
'@emnapi/core': ^2.0.0-alpha.3
|
||||
'@emnapi/runtime': ^2.0.0-alpha.3
|
||||
|
||||
'@napi-rs/wasm-runtime@1.2.3':
|
||||
resolution: {integrity: sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q==}
|
||||
engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0}
|
||||
peerDependencies:
|
||||
'@emnapi/core': ^1.7.1 || ^2.0.0-alpha.4
|
||||
'@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.4
|
||||
|
||||
'@noble/ciphers@1.3.0':
|
||||
resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==}
|
||||
engines: {node: ^14.21.3 || >=16}
|
||||
@@ -1906,6 +1922,10 @@ packages:
|
||||
cpu: [arm64]
|
||||
os: [openharmony]
|
||||
|
||||
'@oxc-parser/binding-wasm32-wasip1@0.0.1':
|
||||
resolution: {integrity: sha512-p2X0okYbIkav6eB5zwwVc6eFRycAO4GbGpBCf9IJCI2fUzwoxysWaMXIyZzrq6Pg4/LsEqFb0Vp6TKovK541vA==}
|
||||
engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0}
|
||||
|
||||
'@oxc-parser/binding-win32-arm64-msvc@0.149.0':
|
||||
resolution: {integrity: sha512-9hMvs3GZESAcbUGU3mqXFWRLgBd8TLeosNYfHusHp/hrBnk5lIP2nJtark3hZ6qT+lqSdS79sH8NoWHj1x9iUw==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
@@ -2724,8 +2744,8 @@ packages:
|
||||
'@tybys/wasm-util@0.10.3':
|
||||
resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==}
|
||||
|
||||
'@types/aws-lambda@8.10.163':
|
||||
resolution: {integrity: sha512-+4zuoEB3S8RIhimtOFT7zAEk2SbpwrKjjGl9CyYnQR6k08uynxfauIIB0pDU1ssr05F8oyGiETwpn+8eXZMqPw==}
|
||||
'@types/aws-lambda@8.10.162':
|
||||
resolution: {integrity: sha512-Fn658grtLOci1oxi1391vvDWJRKNGWRSqfxRkmN/Iy3c0tQH1USMKEXcPYHLvope+ZgTFocx9FRQJx1muBL6qw==}
|
||||
|
||||
'@types/bluebird@3.5.42':
|
||||
resolution: {integrity: sha512-Jhy+MWRlro6UjVi578V/4ZGNfeCOcNCp0YaFNIUGFKlImowqwb1O/22wDVk3FDGMLqxdpOV3qQHD5fPEH4hK6A==}
|
||||
@@ -3274,8 +3294,8 @@ packages:
|
||||
arg@5.0.2:
|
||||
resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==}
|
||||
|
||||
argparse@3.0.2:
|
||||
resolution: {integrity: sha512-mFdDM6WqWKraGLsVb+C9CahPnzTXOefAOLq3jYcca2YZ8bEWpr++Tzj+zSaKW9+X9L5uSxcm1AZ3Y6aZJ09OhQ==}
|
||||
argparse@3.0.0:
|
||||
resolution: {integrity: sha512-BOp5NMrHqKxmq/OLr+clzzrRxgOKSLkcjmkWuChp7Irqwn4s74WjOBPIgWfA/HMcBnVkZ5XEuf9uUqzlpfCQ6A==}
|
||||
|
||||
asn1@0.2.6:
|
||||
resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==}
|
||||
@@ -3818,9 +3838,6 @@ packages:
|
||||
es-module-lexer@2.3.2:
|
||||
resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==}
|
||||
|
||||
es-module-lexer@3.0.2:
|
||||
resolution: {integrity: sha512-BuIB67FngDSyQ/dpQNOZybwdEBDUGJQvOqwWr4ha/ufYiqzuEwPkKO2zLhRAgay28tStRIHUeWmszZAJo3GCOg==}
|
||||
|
||||
es5-ext@0.10.64:
|
||||
resolution: {integrity: sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==}
|
||||
engines: {node: '>=0.10'}
|
||||
@@ -4344,8 +4361,8 @@ packages:
|
||||
resolution: {integrity: sha512-jrWWecWFifN3Feum2MqaTUupNeYSAdPEltKuwGdr8IPdxW72NZXf7ObVUfTWoUH2MPrli0q7SZm9pRt8ob/9XQ==}
|
||||
engines: {node: '>=20.0.0'}
|
||||
|
||||
import-in-the-middle@3.5.0:
|
||||
resolution: {integrity: sha512-O6wo8l0lhBuytKz3x8S65TvacFtoopji+gCVeEBbjCaRy75gkk4bCBQoVmY7OEZ63cbFK+yf77pDCTWaadV7hQ==}
|
||||
import-in-the-middle@3.4.0:
|
||||
resolution: {integrity: sha512-Xfjwfarhe+LGmoaof+sexeNo3sGRysb5x56WrZLwHtmqT0OilSKtVWkv0lrCcMgSKyKP9oBogBAisHZvtJH0aw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
import-without-cache@0.4.0:
|
||||
@@ -4970,11 +4987,6 @@ packages:
|
||||
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
|
||||
hasBin: true
|
||||
|
||||
nanoid@3.3.19:
|
||||
resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==}
|
||||
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
|
||||
hasBin: true
|
||||
|
||||
nanoid@5.1.16:
|
||||
resolution: {integrity: sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==}
|
||||
engines: {node: ^18 || >=20}
|
||||
@@ -5775,8 +5787,8 @@ packages:
|
||||
resolution: {integrity: sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA==}
|
||||
hasBin: true
|
||||
|
||||
tldts-core@7.4.12:
|
||||
resolution: {integrity: sha512-nYNzS2WRf4QJmjzFFgAxLOBjyBxAGRbCy9PVBPaglcYyYajh40VBn+v5Ngr96ZMc7oM0+aCJdtQnNejvdBnXMQ==}
|
||||
tldts-core@7.4.13:
|
||||
resolution: {integrity: sha512-mbYsrih5FRtGxs3Usvl/PqwJsNpp+jsmrdFviiK02teHDG0/HebBG/pqCylje3kzgXYzuLoHJF/0mz9W53t8Xg==}
|
||||
|
||||
tldts@7.4.12:
|
||||
resolution: {integrity: sha512-WylhSDKVeYnWXL3a+vKTaOxjnOeEGw938hImY8zoRWJjRRK/Jp1K+IihBzIONpUmW4e3WmXT6q5FW6vlESVZCA==}
|
||||
@@ -5920,10 +5932,6 @@ packages:
|
||||
resolution: {integrity: sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
type-fest@5.9.0:
|
||||
resolution: {integrity: sha512-yANm3Jr3GiJ1qgJlxGAVxTOIcEOk1rhQHamlXtnrCK7EHP4HeM9OGxtMg/W7HFdrVzw/ZWJKGVIJusVH85sLtw==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
type@2.7.3:
|
||||
resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==}
|
||||
|
||||
@@ -5955,18 +5963,14 @@ packages:
|
||||
undici-types@8.9.0:
|
||||
resolution: {integrity: sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==}
|
||||
|
||||
undici@6.28.1:
|
||||
resolution: {integrity: sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==}
|
||||
undici@6.28.0:
|
||||
resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==}
|
||||
engines: {node: '>=18.17'}
|
||||
|
||||
undici@7.29.0:
|
||||
resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
undici@7.29.1:
|
||||
resolution: {integrity: sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
undici@8.10.2:
|
||||
resolution: {integrity: sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==}
|
||||
engines: {node: '>=22.19.0'}
|
||||
@@ -6413,17 +6417,17 @@ snapshots:
|
||||
'@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6)
|
||||
'@octokit/request': 10.0.11
|
||||
'@octokit/request-error': 7.1.0
|
||||
undici: 6.28.1
|
||||
undici: 6.28.0
|
||||
|
||||
'@actions/http-client@3.0.2':
|
||||
dependencies:
|
||||
tunnel: 0.0.6
|
||||
undici: 6.28.1
|
||||
undici: 6.28.0
|
||||
|
||||
'@actions/http-client@4.0.1':
|
||||
dependencies:
|
||||
tunnel: 0.0.6
|
||||
undici: 6.28.1
|
||||
undici: 6.28.0
|
||||
|
||||
'@actions/io@3.0.2': {}
|
||||
|
||||
@@ -6610,6 +6614,11 @@ snapshots:
|
||||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@emnapi/core@2.0.0-alpha.3':
|
||||
dependencies:
|
||||
'@emnapi/wasi-threads': 2.0.1
|
||||
tslib: 2.8.1
|
||||
|
||||
'@emnapi/runtime@1.11.2':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
@@ -6620,11 +6629,19 @@ snapshots:
|
||||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@emnapi/runtime@2.0.0-alpha.3':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@emnapi/wasi-threads@1.2.2':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@emnapi/wasi-threads@2.0.1':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@epic-web/invariant@1.0.0': {}
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.7':
|
||||
@@ -6923,7 +6940,7 @@ snapshots:
|
||||
'@honeybadger-io/js@6.16.3':
|
||||
dependencies:
|
||||
'@honeybadger-io/core': 6.11.1
|
||||
'@types/aws-lambda': 8.10.163
|
||||
'@types/aws-lambda': 8.10.162
|
||||
'@types/express': 5.0.6
|
||||
|
||||
'@hono/node-server@2.1.1(hono@4.13.7)':
|
||||
@@ -7221,6 +7238,12 @@ snapshots:
|
||||
'@tybys/wasm-util': 0.10.3
|
||||
optional: true
|
||||
|
||||
'@napi-rs/wasm-runtime@1.2.3(@emnapi/core@2.0.0-alpha.3)(@emnapi/runtime@2.0.0-alpha.3)':
|
||||
dependencies:
|
||||
'@emnapi/core': 2.0.0-alpha.3
|
||||
'@emnapi/runtime': 2.0.0-alpha.3
|
||||
'@tybys/wasm-util': 0.10.3
|
||||
|
||||
'@noble/ciphers@1.3.0': {}
|
||||
|
||||
'@noble/curves@1.9.7':
|
||||
@@ -7342,7 +7365,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/api-logs': 0.220.0
|
||||
import-in-the-middle: 3.5.0
|
||||
import-in-the-middle: 3.4.0
|
||||
require-in-the-middle: 8.0.1
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@@ -7477,6 +7500,12 @@ snapshots:
|
||||
'@oxc-parser/binding-openharmony-arm64@0.149.0':
|
||||
optional: true
|
||||
|
||||
'@oxc-parser/binding-wasm32-wasip1@0.0.1':
|
||||
dependencies:
|
||||
'@emnapi/core': 2.0.0-alpha.3
|
||||
'@emnapi/runtime': 2.0.0-alpha.3
|
||||
'@napi-rs/wasm-runtime': 1.2.3(@emnapi/core@2.0.0-alpha.3)(@emnapi/runtime@2.0.0-alpha.3)
|
||||
|
||||
'@oxc-parser/binding-win32-arm64-msvc@0.149.0':
|
||||
optional: true
|
||||
|
||||
@@ -7905,7 +7934,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@scalar/helpers': 0.11.3
|
||||
nanoid: 5.1.16
|
||||
type-fest: 5.9.0
|
||||
type-fest: 5.8.0
|
||||
zod: 4.6.2
|
||||
|
||||
'@scalar/validation@0.6.3': {}
|
||||
@@ -7931,7 +7960,7 @@ snapshots:
|
||||
'@sentry/conventions': 0.16.0
|
||||
'@sentry/core': 10.74.0
|
||||
'@sentry/opentelemetry': 10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.11.0(@opentelemetry/api@1.9.1))
|
||||
import-in-the-middle: 3.5.0
|
||||
import-in-the-middle: 3.4.0
|
||||
optionalDependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1)
|
||||
@@ -7949,7 +7978,7 @@ snapshots:
|
||||
'@sentry/node-core': 10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.222.0(@opentelemetry/api@1.9.1))(@opentelemetry/instrumentation@0.220.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.11.0(@opentelemetry/api@1.9.1))
|
||||
'@sentry/opentelemetry': 10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.11.0(@opentelemetry/api@1.9.1))
|
||||
'@sentry/server-utils': 10.74.0
|
||||
import-in-the-middle: 3.5.0
|
||||
import-in-the-middle: 3.4.0
|
||||
transitivePeerDependencies:
|
||||
- '@opentelemetry/core'
|
||||
- '@opentelemetry/exporter-trace-otlp-http'
|
||||
@@ -7994,9 +8023,8 @@ snapshots:
|
||||
'@tybys/wasm-util@0.10.3':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@types/aws-lambda@8.10.163': {}
|
||||
'@types/aws-lambda@8.10.162': {}
|
||||
|
||||
'@types/bluebird@3.5.42': {}
|
||||
|
||||
@@ -8488,7 +8516,7 @@ snapshots:
|
||||
|
||||
arg@5.0.2: {}
|
||||
|
||||
argparse@3.0.2: {}
|
||||
argparse@3.0.0: {}
|
||||
|
||||
asn1@0.2.6:
|
||||
dependencies:
|
||||
@@ -8656,7 +8684,7 @@ snapshots:
|
||||
parse5: 7.3.0
|
||||
parse5-htmlparser2-tree-adapter: 7.1.0
|
||||
parse5-parser-stream: 7.1.2
|
||||
undici: 7.29.1
|
||||
undici: 7.29.0
|
||||
whatwg-mimetype: 4.0.0
|
||||
|
||||
chownr@3.0.0: {}
|
||||
@@ -8961,8 +8989,6 @@ snapshots:
|
||||
|
||||
es-module-lexer@2.3.2: {}
|
||||
|
||||
es-module-lexer@3.0.2: {}
|
||||
|
||||
es5-ext@0.10.64:
|
||||
dependencies:
|
||||
es6-iterator: 2.0.3
|
||||
@@ -9680,10 +9706,10 @@ snapshots:
|
||||
pino: 10.3.1
|
||||
socks: 2.8.10
|
||||
|
||||
import-in-the-middle@3.5.0:
|
||||
import-in-the-middle@3.4.0:
|
||||
dependencies:
|
||||
cjs-module-lexer: 2.2.1
|
||||
es-module-lexer: 3.0.2
|
||||
es-module-lexer: 2.3.2
|
||||
module-details-from-path: 1.0.4
|
||||
|
||||
import-without-cache@0.4.0: {}
|
||||
@@ -10070,7 +10096,7 @@ snapshots:
|
||||
|
||||
markdown-it@15.0.2:
|
||||
dependencies:
|
||||
argparse: 3.0.2
|
||||
argparse: 3.0.0
|
||||
entities: 8.1.0
|
||||
linkify-it: 6.1.0
|
||||
mdurl: 2.1.0
|
||||
@@ -10475,8 +10501,6 @@ snapshots:
|
||||
|
||||
nanoid@3.3.18: {}
|
||||
|
||||
nanoid@3.3.19: {}
|
||||
|
||||
nanoid@5.1.16: {}
|
||||
|
||||
narou@2.0.1: {}
|
||||
@@ -10844,7 +10868,7 @@ snapshots:
|
||||
|
||||
postcss@8.5.28:
|
||||
dependencies:
|
||||
nanoid: 3.3.19
|
||||
nanoid: 3.3.18
|
||||
picocolors: 1.1.1
|
||||
source-map-js: 1.2.1
|
||||
|
||||
@@ -11423,11 +11447,11 @@ snapshots:
|
||||
|
||||
tlds@1.261.0: {}
|
||||
|
||||
tldts-core@7.4.12: {}
|
||||
tldts-core@7.4.13: {}
|
||||
|
||||
tldts@7.4.12:
|
||||
dependencies:
|
||||
tldts-core: 7.4.12
|
||||
tldts-core: 7.4.13
|
||||
|
||||
to-no-case@1.0.2: {}
|
||||
|
||||
@@ -11536,10 +11560,6 @@ snapshots:
|
||||
dependencies:
|
||||
tagged-tag: 1.0.0
|
||||
|
||||
type-fest@5.9.0:
|
||||
dependencies:
|
||||
tagged-tag: 1.0.0
|
||||
|
||||
type@2.7.3: {}
|
||||
|
||||
typescript@6.0.3: {}
|
||||
@@ -11582,12 +11602,10 @@ snapshots:
|
||||
|
||||
undici-types@8.9.0: {}
|
||||
|
||||
undici@6.28.1: {}
|
||||
undici@6.28.0: {}
|
||||
|
||||
undici@7.29.0: {}
|
||||
|
||||
undici@7.29.1: {}
|
||||
|
||||
undici@8.10.2: {}
|
||||
|
||||
unenv@2.0.0-rc.24:
|
||||
|
||||
@@ -22,7 +22,7 @@ export const findOrphanFiles = async (): Promise<string[]> => {
|
||||
return relative;
|
||||
}
|
||||
const dir = path.dirname(absolute);
|
||||
const base = path.basename(absolute).replace(/\.(?:spec|test)\.[cm]?[jt]sx?$/, '');
|
||||
const base = path.basename(absolute).replace(/(?:\.worker)?\.(?:spec|test)\.[cm]?[jt]sx?$/, '');
|
||||
const exists = entries.some((entry) => entry.parentPath === dir && entry.isFile() && new RegExp(`^${base.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)}\\.[cm]?[jt]sx?$`).test(entry.name));
|
||||
return exists ? null : relative;
|
||||
})
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { route } from '../lib/routes/lofter/tag';
|
||||
import got from '../lib/utils/got';
|
||||
|
||||
vi.mock('../lib/config', () => ({ config: { lofter: { cookies: 'test-cookie' } } }));
|
||||
vi.mock('../lib/utils/got', () => ({ default: vi.fn() }));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
const context = { req: { param: (name: string) => ({ name: 'Photography', type: 'date' })[name] } };
|
||||
|
||||
describe('Lofter tag DWR response', () => {
|
||||
it('reads DWR object references and preserves feed content without executing the response', async () => {
|
||||
// A representative DWR protocol response; no private feed data is included.
|
||||
vi.mocked(got).mockResolvedValue({
|
||||
data: `//#DWR-INSERT
|
||||
//#DWR-REPLY
|
||||
var s0=[],s1={},s2={},s3={};
|
||||
s0[0]=s1;s1.post=s2;s2.blogInfo=s3;
|
||||
s3.blogNickName="Photographer";
|
||||
s2.blogPageUrl="https://example.lofter.com/post/1";
|
||||
s2.title="";
|
||||
s2.digest="<p>A & B</p>";
|
||||
s2.publishTime=1704067200000;
|
||||
s2.tagList=["Photography"];
|
||||
s2.photoLinks='[{"orign":"https://example.com/photo.jpg"}]';
|
||||
s2.embed='{"h256Url":"https://example.com/video.mp4","video_img_url":"https://example.com/poster.jpg"}';
|
||||
dwr.engine._remoteHandleCallback('493053','0',s0);`,
|
||||
});
|
||||
|
||||
const feed = await (route.handler as (ctx: typeof context) => Promise<any>)(context);
|
||||
|
||||
expect(feed.title).toBe('Photography - 日榜 | LOFTER');
|
||||
expect(feed.item).toEqual([
|
||||
{
|
||||
author: 'Photographer',
|
||||
link: 'https://example.lofter.com/post/1',
|
||||
title: 'Photographer:A & B',
|
||||
pubDate: new Date('2024-01-01T00:00:00.000Z'),
|
||||
description:
|
||||
'<video src="https://example.com/video.mp4" poster="https://example.com/poster.jpg" controls="controls"></video><img src="https://example.com/photo.jpg"/><html><head></head><body><p>A & B</p></body></html>',
|
||||
category: ['Photography'],
|
||||
},
|
||||
]);
|
||||
expect(got).toHaveBeenCalledWith(expect.objectContaining({ headers: { Referer: 'https://www.lofter.com/tag/Photography', Cookie: 'test-cookie' } }));
|
||||
});
|
||||
|
||||
it('reads the empty callback response returned by the public endpoint', async () => {
|
||||
vi.mocked(got).mockResolvedValue({ data: "//#DWR-INSERT\n//#DWR-REPLY\ndwr.engine._remoteHandleCallback('493053','0',[]);" });
|
||||
|
||||
const feed = await (route.handler as (ctx: typeof context) => Promise<any>)(context);
|
||||
expect(feed.item).toEqual([]);
|
||||
});
|
||||
|
||||
it('rejects executable callback arguments', async () => {
|
||||
vi.mocked(got).mockResolvedValue({ data: "dwr.engine._remoteHandleCallback('493053','0',fetch('https://example.com'));" });
|
||||
|
||||
await expect((route.handler as (ctx: typeof context) => Promise<any>)(context)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -1,92 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { processFeed, processFeedType2 } from '../lib/routes/dongqiudi/utils';
|
||||
import nintendo from '../lib/routes/nintendo/utils';
|
||||
import cache from '../lib/utils/cache';
|
||||
import got from '../lib/utils/got';
|
||||
|
||||
vi.mock('../lib/utils/cache', () => ({
|
||||
default: { tryGet: vi.fn((_key, getValue) => getValue()) },
|
||||
}));
|
||||
vi.mock('../lib/utils/got', () => ({ default: vi.fn() }));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('Nintendo Nuxt data', () => {
|
||||
it('reads serialized object references and escaped content without executing page scripts', () => {
|
||||
const html = `<script>console.log(window.__NUXT__);throw new Error('Unrelated page code');</script>
|
||||
<script>window.__NUXT__=(function(a,b){a.title=b;a.content="<p>A&B</p>";return {data:[{newsList:[a],newsData:a}]}}({},"News \\u4e2d\\u6587"));</script>`;
|
||||
|
||||
expect(nintendo.nuxtReader(html)).toEqual({
|
||||
newsList: [{ title: 'News 中文', content: '<p>A&B</p>' }],
|
||||
newsData: { title: 'News 中文', content: '<p>A&B</p>' },
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves cached article content, category, and publication time', async () => {
|
||||
const link = 'https://www.nintendoswitch.com.cn/topics/test';
|
||||
vi.mocked(got).mockResolvedValue({
|
||||
data: '<script>window.__NUXT__={data:[{newsData:{content:"<p>Article</p>",category:["News"],releaseTime:1704067200000}}]};</script>',
|
||||
});
|
||||
|
||||
const items = await nintendo.ProcessNewsChina([{ link, description: '<img src="cover.png">' }], cache);
|
||||
|
||||
expect(cache.tryGet).toHaveBeenCalledWith(link, expect.any(Function));
|
||||
expect(items[0]).toEqual({
|
||||
link,
|
||||
description: '<img src="cover.png"><p>Article</p>',
|
||||
category: ['News'],
|
||||
pubDate: new Date('2024-01-01T00:00:00.000Z'),
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['<html>No page data</html>', '<script>window.__NUXT__=fetch("https://example.com");</script>'])('rejects missing or executable page data', (html) => {
|
||||
expect(() => nintendo.nuxtReader(html)).toThrow('Nuxt 框架信息提取失败');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Dongqiudi Nuxt data', () => {
|
||||
const articleHtml = `<script>throw new Error('Unrelated page code');</script>
|
||||
<script>window.__NUXT__=(function(a){return {data:[{article:{rawBody:a,author:"Reporter"}}]}}('<p>Match report</p><a href="dongqiudi:///news/42">Related</a><img src="https://example.com/image.jpg?watermark=1">'));</script>`;
|
||||
|
||||
it('preserves article content cleanup and the author', () => {
|
||||
const item: { title: string; description?: string; author?: string } = { title: 'Match' };
|
||||
|
||||
processFeedType2(item, articleHtml);
|
||||
|
||||
expect(item).toEqual({
|
||||
title: 'Match',
|
||||
description: '<p>Match report</p><a href="https://www.dongqiudi.com/article/42">Related</a><img src="https://example.com/image.jpg">',
|
||||
author: 'Reporter',
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['<html>No Nuxt data</html>', '<script>window.__NUXT__={data:[{}]};</script>', '<script>window.__NUXT__=null;</script>'])('leaves an item unchanged when article data is absent', (html) => {
|
||||
const item = { title: 'Match' };
|
||||
processFeedType2(item, html);
|
||||
expect(item).toEqual({ title: 'Match' });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['team', 'teamInfo:{name:"Club",logo:"club.png"}', 'Club', 'club.png'],
|
||||
['player', 'detail:{base_info:{person_name:"Player",person_logo:"player.png"}}', 'Player', 'player.png'],
|
||||
])('keeps the %s metadata, API request, and article cache', async (type, metadata, name, image) => {
|
||||
vi.mocked(got)
|
||||
.mockResolvedValueOnce({ data: `<script>window.__NUXT__={data:[{${metadata}}]};</script>` })
|
||||
.mockResolvedValueOnce({ data: { data: { articles: [{ id: 42, title: 'Match', category: 'Football', show_time: 1_704_067_200 }] } } })
|
||||
.mockResolvedValueOnce({ data: articleHtml });
|
||||
|
||||
const feed = await processFeed(type, '123');
|
||||
|
||||
expect(feed.title).toBe(`${name} - 相关新闻`);
|
||||
expect(feed.image).toBe(image);
|
||||
expect(feed.item).toHaveLength(1);
|
||||
expect(feed.item[0]).toMatchObject({ author: 'Reporter', category: ['Football'], pubDate: new Date('2024-01-01T00:00:00.000Z') });
|
||||
expect(cache.tryGet).toHaveBeenCalledWith('https://www.dongqiudi.com/articles/42.html', expect.any(Function));
|
||||
expect(got).toHaveBeenNthCalledWith(2, 'https://api.dongqiudi.com/v3/archive/app/channel/feeds', {
|
||||
searchParams: { id: '123', type, size: 20, platform: 'web', version: '' },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,208 +0,0 @@
|
||||
import type { Context } from 'hono';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { route as hotukdealsRoute } from '../lib/routes/hotukdeals/hottest';
|
||||
import { getNSFWNovelContent } from '../lib/routes/pixiv/novel-api/content/nsfw';
|
||||
import { parseNovelContent } from '../lib/routes/pixiv/novel-api/content/utils';
|
||||
import playCache from '../lib/routes/qq/kg/cache';
|
||||
import { route as qqRoute } from '../lib/routes/qq/kg/user';
|
||||
import { route as xueqiuRoute } from '../lib/routes/xueqiu/column';
|
||||
import type { Data, Route } from '../lib/types';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
request: Object.assign(vi.fn(), { get: vi.fn() }),
|
||||
pixivRequest: vi.fn(),
|
||||
tryGet: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../lib/utils/got', () => ({ default: mocks.request }));
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: mocks.tryGet } }));
|
||||
vi.mock('../lib/routes/pixiv/pixiv-got', () => ({ default: mocks.pixivRequest }));
|
||||
vi.mock('../lib/routes/pixiv/novel-api/content/utils', () => ({ parseNovelContent: vi.fn() }));
|
||||
|
||||
const invoke = async (route: Route, params: Record<string, string> = {}) => (await route.handler({ req: { param: (name: string) => params[name] } } as unknown as Context)) as Data;
|
||||
const page = (source: string) => `<script>throw new Error('Browser scripts must not execute');</script><script>${source}</script>`;
|
||||
|
||||
// QQ and Xueqiu use the JSON-assignment format observed in public responses on
|
||||
// 2026-09-06. Keep only the consumed fields and replace public account data.
|
||||
const playDetail = {
|
||||
song_name: 'Song & title',
|
||||
content: 'Line one<br>Line two & three',
|
||||
nick: 'Singer',
|
||||
cover: 'https://example.com/cover.jpg',
|
||||
playurl: 'https://example.com/song.m4a',
|
||||
ksong_mid: 'song-1',
|
||||
ctime: 1_700_000_000,
|
||||
comments: [{ nick: 'Listener', content: 'Nice song', ctime: 1_700_000_001, comment_id: 'comment-1' }],
|
||||
};
|
||||
|
||||
// Pixiv requires authentication and HotUKDeals returned 403 during inspection.
|
||||
// Their fixtures cover the existing consumed-data contract, not live success.
|
||||
const novelDetail = {
|
||||
id: '42',
|
||||
title: 'Novel title',
|
||||
caption: '<p>Caption & summary</p>',
|
||||
text: 'A line\n[uploadedimage:7]',
|
||||
userId: 'author-1',
|
||||
rating: { bookmark: 3, view: 12, like: 2 },
|
||||
cdate: '2026-09-01T10:00:00+09:00',
|
||||
isOriginal: true,
|
||||
aiType: 0,
|
||||
tags: ['fiction'],
|
||||
coverUrl: 'https://example.com/novel.jpg',
|
||||
images: { '7': { urls: { original: 'https://example.com/image.jpg' } }, '8': {} },
|
||||
seriesId: 'series-1',
|
||||
seriesTitle: 'Series title',
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
const entries = new Map<string, unknown>();
|
||||
mocks.tryGet.mockImplementation(async (key, getValue) => {
|
||||
if (!entries.has(key)) {
|
||||
entries.set(key, await getValue());
|
||||
}
|
||||
return entries.get(key);
|
||||
});
|
||||
mocks.request.mockRejectedValue(new Error('Unexpected upstream request'));
|
||||
mocks.request.get.mockRejectedValue(new Error('Unexpected upstream request'));
|
||||
mocks.pixivRequest.mockRejectedValue(new Error('Unexpected Pixiv request'));
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('External network access is disabled')));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('routes that parse embedded page data', () => {
|
||||
it('preserves QQ user feed metadata, audio attachments and play-detail cache keys', async () => {
|
||||
const userData = {
|
||||
data: {
|
||||
nickname: 'Singer',
|
||||
head_img_url: 'https://example.com/avatar.jpg',
|
||||
ugclist: [{ shareid: 'share-1', ksong_mid: 'song-1', title: 'Song & title', ctime: 1_600_000_000, avatar: 'https://example.com/fallback.jpg' }],
|
||||
},
|
||||
share: { content: 'Singer profile' },
|
||||
};
|
||||
mocks.request.mockResolvedValueOnce({ data: page(`window.__DATA__ = ${JSON.stringify(userData)};`) }).mockResolvedValueOnce({ data: page(`window.__DATA__ = ${JSON.stringify({ detail: playDetail })};`) });
|
||||
|
||||
const result = await invoke(qqRoute, { userId: 'singer-1' });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: 'Singer - 全民K歌',
|
||||
link: 'https://node.kg.qq.com/personal?uid=singer-1',
|
||||
description: 'Singer profile',
|
||||
image: 'https://example.com/avatar.jpg',
|
||||
allowEmpty: true,
|
||||
itunes_author: 'Singer',
|
||||
itunes_category: '全民K歌',
|
||||
item: [
|
||||
{
|
||||
title: 'Song & title',
|
||||
description: playDetail.content,
|
||||
link: 'https://node.kg.qq.com/play?s=share-1',
|
||||
guid: 'ksong:song-1',
|
||||
author: 'Singer',
|
||||
pubDate: new Date(1_700_000_000_000),
|
||||
itunes_item_image: playDetail.cover,
|
||||
enclosure_url: playDetail.playurl,
|
||||
enclosure_type: 'audio/x-m4a',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(mocks.tryGet).toHaveBeenCalledWith('ksong:song-1', expect.any(Function));
|
||||
expect(mocks.request.mock.calls.map(([url]) => url)).toEqual(['https://node.kg.qq.com/personal?uid=singer-1', 'https://node.kg.qq.com/play?s=share-1']);
|
||||
});
|
||||
|
||||
it('preserves QQ comments and reuses a play-detail cache entry', async () => {
|
||||
mocks.request.mockResolvedValueOnce({ data: page(`window.__DATA__ = ${JSON.stringify({ detail: playDetail })};`) });
|
||||
|
||||
const first = await playCache.getPlayInfo(undefined, 'share-1', 'song-1');
|
||||
const second = await playCache.getPlayInfo(undefined, 'share-1', 'song-1');
|
||||
|
||||
expect(first).toMatchObject({ name: playDetail.song_name, description: playDetail.content, author: 'Singer', ksong_mid: 'song-1', comments: playDetail.comments });
|
||||
expect(second).toBe(first);
|
||||
expect(mocks.request).toHaveBeenCalledExactlyOnceWith('https://node.kg.qq.com/play?s=share-1');
|
||||
});
|
||||
|
||||
it('preserves the Xueqiu cookie jar, first-page request and author data', async () => {
|
||||
mocks.request
|
||||
.mockResolvedValueOnce({ data: '' })
|
||||
.mockResolvedValueOnce({ data: page('window.SNOWMAN_TARGET = {"screen_name":"Author & name","description":"A profile & text"};') })
|
||||
.mockResolvedValueOnce({ data: { list: [{ title: 'Article', description: '<p>Body</p>', created_at: 1_700_000_000_000, target: '/author/123' }] } });
|
||||
|
||||
const result = await invoke(xueqiuRoute, { id: 'author-1' });
|
||||
|
||||
expect(result).toEqual({
|
||||
title: 'Author & name - 雪球',
|
||||
link: 'https://xueqiu.com/author-1/column',
|
||||
description: 'A profile & text',
|
||||
item: [{ title: 'Article', description: '<p>Body</p>', pubDate: new Date(1_700_000_000_000), link: 'https://xueqiu.com/author/123', author: 'Author & name' }],
|
||||
});
|
||||
const cookieJar = mocks.request.mock.calls[0][1].cookieJar;
|
||||
expect(mocks.request).toHaveBeenNthCalledWith(2, 'https://xueqiu.com/author-1/column', { cookieJar });
|
||||
expect(mocks.request).toHaveBeenNthCalledWith(3, 'https://xueqiu.com/statuses/original/timeline.json', { cookieJar, searchParams: { user_id: 'author-1', page: 1 } });
|
||||
});
|
||||
|
||||
it('preserves HotUKDeals titles, image URLs, prices and temperatures', async () => {
|
||||
const state = { widgets: { hottestWidget: { threads: [{ title: 'A & B deal', mainImage: { path: 'images', name: 'deal' }, temperature: 123, displayPrice: '£12.99', url: 'https://example.com/deal' }] } } };
|
||||
mocks.request.get.mockResolvedValueOnce({ data: page(`window.__INITIAL_STATE__ = ${JSON.stringify(state)};`) });
|
||||
|
||||
expect(await invoke(hotukdealsRoute)).toEqual({
|
||||
title: 'hotukdeals hottest',
|
||||
link: 'https://www.hotukdeals.com/',
|
||||
item: [{ title: 'A & B deal', description: '<img src="https://images.hotukdeals.com/images/deal/re/768x768/qt/60/deal.jpg"><br>123° A & B deal<br>£12.99', link: 'https://example.com/deal' }],
|
||||
});
|
||||
expect(mocks.request.get).toHaveBeenCalledExactlyOnceWith('https://www.hotukdeals.com/');
|
||||
});
|
||||
|
||||
it.each(['parent object', 'nested assignment'])('preserves authenticated Pixiv novel data from a %s', async (form) => {
|
||||
const source = form === 'parent object' ? `window.pixiv = {novel: ${JSON.stringify(novelDetail)}};` : `window.pixiv = {};</script><script>pixiv.novel = ${JSON.stringify(novelDetail)};`;
|
||||
mocks.pixivRequest.mockResolvedValueOnce({ data: page(source) });
|
||||
vi.mocked(parseNovelContent).mockResolvedValueOnce('<p>Rendered novel</p>');
|
||||
|
||||
const result = await getNSFWNovelContent('42', 'test-token');
|
||||
|
||||
expect(result).toEqual({
|
||||
id: '42',
|
||||
title: 'Novel title',
|
||||
description: novelDetail.caption,
|
||||
content: '<p>Rendered novel</p>',
|
||||
userId: 'author-1',
|
||||
userName: null,
|
||||
bookmarkCount: 3,
|
||||
viewCount: 12,
|
||||
likeCount: 2,
|
||||
createDate: new Date('2026-09-01T01:00:00Z'),
|
||||
updateDate: null,
|
||||
isOriginal: true,
|
||||
aiType: 0,
|
||||
tags: ['fiction'],
|
||||
coverUrl: novelDetail.coverUrl,
|
||||
images: { '7': 'https://example.com/image.jpg' },
|
||||
seriesId: 'series-1',
|
||||
seriesTitle: 'Series title',
|
||||
});
|
||||
expect(parseNovelContent).toHaveBeenCalledExactlyOnceWith(novelDetail.text, { '7': 'https://example.com/image.jpg' }, 'test-token');
|
||||
expect(mocks.pixivRequest).toHaveBeenCalledWith(
|
||||
'https://app-api.pixiv.net/webview/v2/novel',
|
||||
expect.objectContaining({ headers: expect.objectContaining({ Authorization: 'Bearer test-token' }), searchParams: 'id=42&viewer_version=20221031_ai' })
|
||||
);
|
||||
expect(mocks.tryGet).toHaveBeenCalledWith('https://app-api.pixiv.net/webview/v2/novel:42', expect.any(Function));
|
||||
expect(await getNSFWNovelContent('42', 'test-token')).toBe(result);
|
||||
expect(mocks.pixivRequest).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('rejects missing Pixiv novel data instead of returning an empty novel', async () => {
|
||||
mocks.pixivRequest.mockResolvedValueOnce({ data: '<html><body>Login required</body></html>' });
|
||||
|
||||
await expect(getNSFWNovelContent('42', 'test-token')).rejects.toThrow('No novel data found');
|
||||
expect(parseNovelContent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects missing page state instead of accepting a blocked HotUKDeals page', async () => {
|
||||
mocks.request.get.mockResolvedValueOnce({ data: '<html><body>Access denied</body></html>' });
|
||||
|
||||
await expect(invoke(hotukdealsRoute)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -1,72 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { fetchThread, ThreadFetcher } from '../lib/routes/yamibo/utils';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
ofetch: vi.fn(),
|
||||
getPlaywrightPage: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
addCookies: vi.fn(),
|
||||
goto: vi.fn(),
|
||||
route: vi.fn(),
|
||||
waitForSelector: vi.fn(),
|
||||
content: vi.fn(),
|
||||
}));
|
||||
vi.mock('../lib/config', () => ({ config: { yamibo: { auth: 'test-auth', salt: 'test-salt' } } }));
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: mocks.ofetch }));
|
||||
vi.mock('../lib/utils/playwright', () => ({ getPlaywrightPage: mocks.getPlaywrightPage }));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.goto.mockResolvedValue(undefined);
|
||||
mocks.waitForSelector.mockResolvedValue(undefined);
|
||||
mocks.ofetch.mockResolvedValue('<script type="text/javascript">/* fixture */</script>');
|
||||
mocks.content.mockResolvedValue('<div id="postlist">Thread content</div>');
|
||||
mocks.getPlaywrightPage.mockResolvedValue({
|
||||
page: { goto: mocks.goto, route: mocks.route, waitForSelector: mocks.waitForSelector, content: mocks.content },
|
||||
context: { addCookies: mocks.addCookies },
|
||||
destroy: mocks.destroy,
|
||||
});
|
||||
});
|
||||
|
||||
describe('Yamibo browser fallback', () => {
|
||||
it('keeps ordinary HTML requests free of browser work', async () => {
|
||||
mocks.ofetch.mockResolvedValue('<div id="postlist">Ordinary HTML</div>');
|
||||
const result = await fetchThread('123', { ordertype: '1' });
|
||||
expect(result).toEqual({ link: 'https://bbs.yamibo.com/forum.php?mod=viewthread&tid=123&ordertype=1', data: '<div id="postlist">Ordinary HTML</div>' });
|
||||
expect(mocks.ofetch).toHaveBeenCalledWith(result.link, { headers: { cookie: 'EeqY_2132_saltkey=test-salt; EeqY_2132_auth=test-auth' } });
|
||||
expect(mocks.getPlaywrightPage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reuses one browser and serializes navigation for concurrent article requests', async () => {
|
||||
const fetcher = new ThreadFetcher();
|
||||
let active = 0;
|
||||
let maximum = 0;
|
||||
mocks.goto.mockImplementation(async () => {
|
||||
active++;
|
||||
maximum = Math.max(maximum, active);
|
||||
await Promise.resolve();
|
||||
active--;
|
||||
});
|
||||
try {
|
||||
const results = await Promise.all([fetcher.fetchThread('123'), fetcher.fetchThread('456')]);
|
||||
expect(results.map((result) => result.data)).toEqual(Array.from({ length: 2 }, () => '<div id="postlist">Thread content</div>'));
|
||||
expect(mocks.getPlaywrightPage).toHaveBeenCalledOnce();
|
||||
expect(mocks.goto).toHaveBeenCalledTimes(2);
|
||||
expect(maximum).toBe(1);
|
||||
expect(mocks.addCookies).toHaveBeenCalledWith([
|
||||
{ name: 'EeqY_2132_saltkey', value: 'test-salt', url: 'https://bbs.yamibo.com' },
|
||||
{ name: 'EeqY_2132_auth', value: 'test-auth', url: 'https://bbs.yamibo.com' },
|
||||
]);
|
||||
} finally {
|
||||
await fetcher.close();
|
||||
}
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('closes the browser when the expected thread content never appears', async () => {
|
||||
mocks.waitForSelector.mockRejectedValue(new Error('Page timeout'));
|
||||
await expect(fetchThread('123')).rejects.toThrow('browser access did not return thread content');
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,382 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
cookies: '',
|
||||
ofetch: vi.fn(),
|
||||
getPlaywrightPage: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
addCookies: vi.fn(),
|
||||
browserCookies: vi.fn(),
|
||||
goto: vi.fn(),
|
||||
route: vi.fn(),
|
||||
waitForFunction: vi.fn(),
|
||||
evaluate: vi.fn(),
|
||||
content: vi.fn(),
|
||||
url: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../lib/config', () => ({
|
||||
config: {
|
||||
zhihu: {
|
||||
get cookies() {
|
||||
return mocks.cookies;
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: mocks.ofetch }));
|
||||
vi.mock('../lib/utils/is-worker', () => ({ isWorker: true }));
|
||||
vi.mock('../lib/utils/playwright', () => ({ getPlaywrightPage: mocks.getPlaywrightPage }));
|
||||
|
||||
const origin = 'https://www.zhihu.com';
|
||||
const pageUrl = `${origin}/people/example`;
|
||||
const apiPath = '/api/v4/members/example/activities?limit=20';
|
||||
const columnPageUrl = 'https://zhuanlan.zhihu.com/example';
|
||||
const columnApiPath = '/api/v4/columns/example/items';
|
||||
const feedData = { data: [{ id: 'api-item' }] };
|
||||
const pageHtml = '<html><body>Profile content</body></html>';
|
||||
const nativeFetch = vi.fn<typeof fetch>();
|
||||
const navigationResponse = (contentType = 'text/html', status = 200, body = pageHtml) => ({
|
||||
headers: () => ({ 'content-type': contentType }),
|
||||
status: () => status,
|
||||
text: () => Promise.resolve(body),
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.resetAllMocks();
|
||||
mocks.cookies = '';
|
||||
mocks.goto.mockImplementation((url) => {
|
||||
mocks.url.mockReturnValue(url);
|
||||
return Promise.resolve(navigationResponse());
|
||||
});
|
||||
mocks.waitForFunction.mockResolvedValue(undefined);
|
||||
mocks.browserCookies.mockResolvedValue([
|
||||
{ name: 'd_c0', value: 'browser-dc0' },
|
||||
{ name: '__zse_ck', value: 'browser-zse' },
|
||||
]);
|
||||
mocks.evaluate.mockImplementation((callback, argument) => callback(argument));
|
||||
mocks.content.mockResolvedValue(pageHtml);
|
||||
mocks.url.mockReturnValue(pageUrl);
|
||||
mocks.getPlaywrightPage.mockResolvedValue({
|
||||
page: { goto: mocks.goto, route: mocks.route, waitForFunction: mocks.waitForFunction, evaluate: mocks.evaluate, content: mocks.content, url: mocks.url },
|
||||
context: { addCookies: mocks.addCookies, cookies: mocks.browserCookies },
|
||||
destroy: mocks.destroy,
|
||||
});
|
||||
nativeFetch.mockImplementation(() => Promise.resolve(Response.json(feedData)));
|
||||
vi.stubGlobal('fetch', nativeFetch);
|
||||
vi.stubGlobal('window', { AbortSignal });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('Zhihu client session', () => {
|
||||
it('uses complete configured credentials for API and HTML requests without starting a browser', async () => {
|
||||
mocks.cookies = 'd_c0=configured==; __zse_ck=existing; z_c0=login=value';
|
||||
mocks.ofetch.mockResolvedValueOnce(feedData).mockResolvedValueOnce(pageHtml);
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
const result = await withZhihuClient(pageUrl, async (client) => ({ data: await client.get(apiPath), html: await client.getPage() }));
|
||||
|
||||
expect(result).toEqual({ data: feedData, html: pageHtml });
|
||||
expect(mocks.ofetch).toHaveBeenNthCalledWith(1, `${origin}${apiPath}`, {
|
||||
headers: expect.objectContaining({ cookie: mocks.cookies, Referer: pageUrl, 'x-zse-96': expect.stringMatching(/^2\.0_/) }),
|
||||
});
|
||||
expect(mocks.ofetch).toHaveBeenNthCalledWith(2, pageUrl, {
|
||||
headers: { cookie: mocks.cookies, Referer: pageUrl },
|
||||
parseResponse: expect.any(Function),
|
||||
});
|
||||
expect(mocks.ofetch.mock.calls[1][1].parseResponse(pageHtml)).toBe(pageHtml);
|
||||
expect(mocks.getPlaywrightPage).not.toHaveBeenCalled();
|
||||
expect(nativeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['', 'z_c0=isolated-login', 'd_c0=; z_c0=isolated-login'])('initializes a guest session before the target without transferring an unrelated login: %s', async (cookies) => {
|
||||
mocks.cookies = cookies;
|
||||
mocks.browserCookies.mockResolvedValueOnce([{ name: 'd_c0', value: 'browser-dc0' }]);
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(pageUrl, (client) => client.get(apiPath))).toEqual(feedData);
|
||||
|
||||
expect(mocks.addCookies).not.toHaveBeenCalled();
|
||||
expect(mocks.goto.mock.calls).toEqual([
|
||||
[`${origin}/explore`, { waitUntil: 'domcontentloaded' }],
|
||||
[pageUrl, { waitUntil: 'domcontentloaded' }],
|
||||
]);
|
||||
expect(mocks.browserCookies).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.browserCookies).toHaveBeenCalledWith(origin);
|
||||
expect(mocks.getPlaywrightPage).toHaveBeenCalledExactlyOnceWith(pageUrl, { noGoto: true, closeTimeout: 0 });
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps a configured identity and login intact and requests the API through the browser native fetch', async () => {
|
||||
mocks.cookies = 'd_c0=seed==;z_c0=login=value';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
|
||||
expect(mocks.addCookies).toHaveBeenCalledWith([
|
||||
{ name: 'd_c0', value: 'seed==', domain: '.zhihu.com', path: '/' },
|
||||
{ name: 'z_c0', value: 'login=value', domain: '.zhihu.com', path: '/' },
|
||||
]);
|
||||
expect(mocks.goto).toHaveBeenCalledExactlyOnceWith(pageUrl, { waitUntil: 'domcontentloaded' });
|
||||
expect(mocks.evaluate).toHaveBeenCalledOnce();
|
||||
expect(nativeFetch).toHaveBeenCalledExactlyOnceWith(`${origin}${apiPath}`, {
|
||||
headers: expect.objectContaining({ 'x-zse-96': expect.stringMatching(/^2\.0_/) }),
|
||||
credentials: 'include',
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
const headers = new Headers(nativeFetch.mock.calls[0][1]?.headers);
|
||||
expect(headers.has('cookie')).toBe(false);
|
||||
expect(headers.has('user-agent')).toBe(false);
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
const handler = mocks.route.mock.calls[0][1];
|
||||
await Promise.all(
|
||||
['document', 'script', 'xhr', 'fetch', 'image'].map(async (resourceType) => {
|
||||
const continueRequest = vi.fn();
|
||||
const abortRequest = vi.fn();
|
||||
await handler({ request: () => ({ resourceType: () => resourceType }), continue: continueRequest, abort: abortRequest });
|
||||
expect(continueRequest).toHaveBeenCalledTimes(resourceType === 'image' ? 0 : 1);
|
||||
expect(abortRequest).toHaveBeenCalledTimes(resourceType === 'image' ? 1 : 0);
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('reuses one browser for API and HTML reads and signs each API request with its current identity', async () => {
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
const signature = vi.spyOn(await import('../lib/routes/zhihu/sign'), 'getSignedHeaders');
|
||||
mocks.browserCookies
|
||||
.mockResolvedValueOnce([{ name: 'd_c0', value: 'guest-dc0' }])
|
||||
.mockResolvedValueOnce([
|
||||
{ name: 'd_c0', value: 'guest-dc0' },
|
||||
{ name: '__zse_ck', value: 'browser-zse' },
|
||||
])
|
||||
.mockResolvedValueOnce([{ name: 'd_c0', value: 'first-dc0' }])
|
||||
.mockResolvedValueOnce([{ name: 'd_c0', value: 'updated-dc0' }]);
|
||||
|
||||
const html = await withZhihuClient(pageUrl, async (client) => {
|
||||
expect(await client.get(apiPath)).toEqual(feedData);
|
||||
expect(await client.get('/api/v4/members/example')).toEqual(feedData);
|
||||
expect(mocks.destroy).not.toHaveBeenCalled();
|
||||
return client.getPage();
|
||||
});
|
||||
|
||||
expect(html).toBe(pageHtml);
|
||||
expect(signature).toHaveBeenNthCalledWith(1, apiPath, 'first-dc0');
|
||||
expect(signature).toHaveBeenNthCalledWith(2, '/api/v4/members/example', 'updated-dc0');
|
||||
expect(mocks.getPlaywrightPage).toHaveBeenCalledOnce();
|
||||
expect(mocks.goto).toHaveBeenLastCalledWith(pageUrl, { waitUntil: 'domcontentloaded' });
|
||||
expect(mocks.content).toHaveBeenCalledOnce();
|
||||
expect(nativeFetch).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('requests the signed API after an initial JSON error without waiting for an unnecessary session cookie', async () => {
|
||||
mocks.browserCookies.mockResolvedValue([{ name: 'd_c0', value: 'browser-dc0' }]);
|
||||
mocks.goto.mockImplementation((url) => {
|
||||
mocks.url.mockReturnValue(url);
|
||||
return Promise.resolve(url === `${origin}${columnApiPath}` ? navigationResponse('application/json; charset=utf-8', 403, '{"error":{"code":403}}') : navigationResponse());
|
||||
});
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(columnPageUrl, (client) => client.get(columnApiPath))).toEqual(feedData);
|
||||
|
||||
expect(mocks.goto.mock.calls).toEqual([
|
||||
[`${origin}/explore`, { waitUntil: 'domcontentloaded' }],
|
||||
[`${origin}${columnApiPath}`, { waitUntil: 'domcontentloaded' }],
|
||||
]);
|
||||
expect(mocks.waitForFunction).not.toHaveBeenCalled();
|
||||
expect(mocks.browserCookies).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.evaluate).toHaveBeenCalledOnce();
|
||||
expect(nativeFetch).toHaveBeenCalledExactlyOnceWith(`${origin}${columnApiPath}`, expect.objectContaining({ credentials: 'include' }));
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('consumes a successful initial JSON response once without requesting the first API twice', async () => {
|
||||
mocks.browserCookies.mockResolvedValue([{ name: 'd_c0', value: 'browser-dc0' }]);
|
||||
mocks.goto.mockImplementation((url) => {
|
||||
mocks.url.mockReturnValue(url);
|
||||
return Promise.resolve(url === `${origin}${columnApiPath}` ? navigationResponse('application/json', 200, JSON.stringify(feedData)) : navigationResponse());
|
||||
});
|
||||
const refreshedData = { data: [{ id: 'refreshed-item' }] };
|
||||
nativeFetch.mockResolvedValue(Response.json(refreshedData));
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(columnPageUrl, async (client) => {
|
||||
expect(await client.get(columnApiPath)).toEqual(feedData);
|
||||
expect(nativeFetch).not.toHaveBeenCalled();
|
||||
expect(mocks.evaluate).not.toHaveBeenCalled();
|
||||
expect(await client.get(columnApiPath)).toEqual(refreshedData);
|
||||
});
|
||||
|
||||
expect(mocks.goto).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.waitForFunction).not.toHaveBeenCalled();
|
||||
expect(nativeFetch).toHaveBeenCalledOnce();
|
||||
expect(mocks.evaluate).toHaveBeenCalledOnce();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('initializes a column on the API origin and restores that origin after reading the column HTML', async () => {
|
||||
const columnUrl = 'https://zhuanlan.zhihu.com/example';
|
||||
const columnApi = '/api/v4/columns/example/items';
|
||||
mocks.url.mockReturnValue(columnUrl);
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(columnUrl, async (client) => {
|
||||
expect(await client.get(columnApi)).toEqual(feedData);
|
||||
expect(await client.getPage()).toBe(pageHtml);
|
||||
expect(await client.get(columnApi)).toEqual(feedData);
|
||||
});
|
||||
|
||||
expect(mocks.goto.mock.calls).toEqual([
|
||||
[`${origin}/explore`, { waitUntil: 'domcontentloaded' }],
|
||||
[`${origin}${columnApi}`, { waitUntil: 'domcontentloaded' }],
|
||||
[columnUrl, { waitUntil: 'domcontentloaded' }],
|
||||
[`${origin}/explore`, { waitUntil: 'domcontentloaded' }],
|
||||
]);
|
||||
expect(mocks.getPlaywrightPage).toHaveBeenCalledExactlyOnceWith(columnUrl, { noGoto: true, closeTimeout: 0 });
|
||||
expect(nativeFetch).toHaveBeenCalledTimes(2);
|
||||
expect(nativeFetch).toHaveBeenCalledWith(`${origin}${columnApi}`, expect.objectContaining({ credentials: 'include' }));
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('initializes a c_-style column through its API even when the page already uses the API origin', async () => {
|
||||
const columnUrl = `${origin}/column/c_123`;
|
||||
const columnApi = '/api/v4/columns/c_123/items';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(columnUrl, (client) => client.get(columnApi))).toEqual(feedData);
|
||||
|
||||
expect(mocks.goto.mock.calls).toEqual([
|
||||
[`${origin}/explore`, { waitUntil: 'domcontentloaded' }],
|
||||
[`${origin}${columnApi}`, { waitUntil: 'domcontentloaded' }],
|
||||
]);
|
||||
expect(mocks.getPlaywrightPage).toHaveBeenCalledExactlyOnceWith(columnUrl, { noGoto: true, closeTimeout: 0 });
|
||||
expect(nativeFetch).toHaveBeenCalledExactlyOnceWith(`${origin}${columnApi}`, expect.objectContaining({ credentials: 'include' }));
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[`${origin}/people/example`, '/api/v3/moments/example/activities?limit=5'],
|
||||
[`${origin}/topic/123/newest`, '/api/v5.1/topics/123/feeds/timeline_activity'],
|
||||
[`${origin}/question/123`, '/api/v4/questions/123/answers?limit=20'],
|
||||
])('initializes the human page for a non-column API: %s', async (targetUrl, targetApi) => {
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(targetUrl, (client) => client.get(targetApi))).toEqual(feedData);
|
||||
|
||||
expect(mocks.goto.mock.calls).toEqual([
|
||||
[`${origin}/explore`, { waitUntil: 'domcontentloaded' }],
|
||||
[targetUrl, { waitUntil: 'domcontentloaded' }],
|
||||
]);
|
||||
expect(nativeFetch).toHaveBeenCalledExactlyOnceWith(`${origin}${targetApi}`, expect.objectContaining({ credentials: 'include' }));
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('does not initialize a browser when the callback performs no requests', async () => {
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(pageUrl, () => Promise.resolve('cached result'))).toBe('cached result');
|
||||
|
||||
expect(mocks.getPlaywrightPage).not.toHaveBeenCalled();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(nativeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('closes the browser when the route callback fails after reading data', async () => {
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await expect(
|
||||
withZhihuClient(pageUrl, async (client) => {
|
||||
await client.get(apiPath);
|
||||
throw new Error('Route parsing failed');
|
||||
})
|
||||
).rejects.toThrow('Route parsing failed');
|
||||
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ status: 403, body: 'test-private-response', error: 'HTTP 403' },
|
||||
{ status: 200, body: '<html>test-private-response</html>', error: 'did not return JSON' },
|
||||
])('rejects an unusable API response and closes the browser: $error', async ({ status, body, error }) => {
|
||||
nativeFetch.mockResolvedValue(new Response(body, { status }));
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
const result = withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
|
||||
await expect(result).rejects.toThrow(error);
|
||||
await expect(result).rejects.not.toThrow('test-private-response');
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('closes the browser when its native API fetch fails', async () => {
|
||||
nativeFetch.mockRejectedValue(new Error('Browser fetch failed'));
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await expect(withZhihuClient(pageUrl, (client) => client.get(apiPath))).rejects.toThrow('Browser fetch failed');
|
||||
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('closes the browser and gives an actionable phase when session cookies are unavailable', async () => {
|
||||
mocks.waitForFunction.mockRejectedValue(new Error('Page timeout'));
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
const result = withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
|
||||
await expect(result).rejects.toThrow('session cookie wait');
|
||||
await expect(result).rejects.toThrow('Configure ZHIHU_COOKIES');
|
||||
expect(nativeFetch).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('stops before the target and closes the browser when guest navigation fails', async () => {
|
||||
mocks.goto.mockRejectedValueOnce(new Error('Navigation failed with test-private-diagnostic'));
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
const result = withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
|
||||
await expect(result).rejects.toThrow('guest cookie initialization');
|
||||
await expect(result).rejects.toThrow('Configure ZHIHU_COOKIES');
|
||||
await expect(result).rejects.not.toThrow('test-private-diagnostic');
|
||||
expect(mocks.goto).toHaveBeenCalledExactlyOnceWith(`${origin}/explore`, { waitUntil: 'domcontentloaded' });
|
||||
expect(mocks.browserCookies).not.toHaveBeenCalled();
|
||||
expect(mocks.waitForFunction).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('stops before the target and closes the browser when the guest page does not set d_c0', async () => {
|
||||
mocks.browserCookies.mockResolvedValueOnce([{ name: '__zse_ck', value: 'browser-zse' }]);
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await expect(withZhihuClient(pageUrl, (client) => client.get(apiPath))).rejects.toThrow('guest cookie initialization');
|
||||
|
||||
expect(mocks.goto).toHaveBeenCalledExactlyOnceWith(`${origin}/explore`, { waitUntil: 'domcontentloaded' });
|
||||
expect(mocks.browserCookies).toHaveBeenCalledExactlyOnceWith(origin);
|
||||
expect(mocks.waitForFunction).not.toHaveBeenCalled();
|
||||
expect(mocks.evaluate).not.toHaveBeenCalled();
|
||||
expect(mocks.destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('rejects non-API paths before initializing a session', async () => {
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await expect(withZhihuClient(pageUrl, (client) => client.get('//example.invalid/api/'))).rejects.toThrow('expected an API path');
|
||||
|
||||
expect(mocks.getPlaywrightPage).not.toHaveBeenCalled();
|
||||
expect(nativeFetch).not.toHaveBeenCalled();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,241 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
cookies: '',
|
||||
ofetch: vi.fn(),
|
||||
raw: vi.fn(),
|
||||
tryGet: vi.fn(),
|
||||
generateHeaders: vi.fn(),
|
||||
createBrowserClient: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../lib/config', () => ({
|
||||
config: {
|
||||
zhihu: {
|
||||
get cookies() {
|
||||
return mocks.cookies;
|
||||
},
|
||||
},
|
||||
cache: { contentExpire: 3600 },
|
||||
},
|
||||
}));
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: Object.assign(mocks.ofetch, { raw: mocks.raw }) }));
|
||||
vi.mock('../lib/utils/cache', () => ({ default: { tryGet: mocks.tryGet } }));
|
||||
vi.mock('../lib/utils/header-generator', () => ({ generateHeaders: mocks.generateHeaders }));
|
||||
vi.mock('../lib/utils/is-worker', () => ({ isWorker: false }));
|
||||
vi.mock('../lib/routes/zhihu/browser', () => ({ createBrowserClient: mocks.createBrowserClient }));
|
||||
|
||||
const origin = 'https://www.zhihu.com';
|
||||
const pageUrl = `${origin}/people/example`;
|
||||
const apiPath = '/api/v4/members/example/activities?limit=20';
|
||||
const otherApiPath = '/api/v4/members/example?include=description';
|
||||
const scriptUrl = 'https://static.zhihu.com/zse-ck/v4/0123456789abcdef.js';
|
||||
const challengeHtml = '<meta id="zh-zse-ck" content="challenge-fixture"><script src="https://static.zhihu.com/zse-ck/v4/0123456789abcdef.js"></script>';
|
||||
const challengeScript = `
|
||||
if (navigator.userAgent !== 'RSSHub JSDOM fixture' || navigator.webdriver !== false) {
|
||||
throw new Error('The challenge did not receive the configured browser fingerprint');
|
||||
}
|
||||
if (document.getElementById('zh-zse-ck').getAttribute('content') !== 'challenge-fixture') {
|
||||
throw new Error('The challenge did not receive its page metadata');
|
||||
}
|
||||
document.cookie = '__zse_ck=generated-token; path=/';
|
||||
`;
|
||||
const userAgent = 'RSSHub JSDOM fixture';
|
||||
const feedData = { data: [{ id: 'api-item' }] };
|
||||
const pageHtml = '<html><body>Profile content</body></html>';
|
||||
const cached = new Map<string, unknown>();
|
||||
let initialRejectionListeners: Set<ReturnType<typeof process.listeners>[number]>;
|
||||
|
||||
const requestHeaders = (url: string) => {
|
||||
const request = mocks.ofetch.mock.calls.find(([requestedUrl]) => requestedUrl === url);
|
||||
expect(request).toBeDefined();
|
||||
return new Headers(request?.[1]?.headers);
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.resetAllMocks();
|
||||
cached.clear();
|
||||
mocks.cookies = '';
|
||||
initialRejectionListeners = new Set(process.listeners('unhandledRejection'));
|
||||
vi.spyOn(Math, 'random').mockReturnValue(0.25);
|
||||
mocks.generateHeaders.mockReturnValue({ 'user-agent': userAgent });
|
||||
mocks.tryGet.mockImplementation(async (key: string, fetchValue: () => Promise<unknown>) => {
|
||||
if (!cached.has(key)) {
|
||||
cached.set(key, await fetchValue());
|
||||
}
|
||||
return cached.get(key);
|
||||
});
|
||||
mocks.raw.mockImplementation((url: string) => Promise.resolve(url === `${origin}/explore` ? { headers: new Headers({ 'set-cookie': 'd_c0=guest-identity==|fixture; Path=/; Secure' }) } : { _data: challengeHtml }));
|
||||
mocks.ofetch.mockImplementation((url: string) => {
|
||||
if (url === scriptUrl) {
|
||||
return Promise.resolve(challengeScript);
|
||||
}
|
||||
if (url === pageUrl) {
|
||||
return Promise.resolve(pageHtml);
|
||||
}
|
||||
if (url.startsWith(`${origin}/api/`)) {
|
||||
return Promise.resolve(feedData);
|
||||
}
|
||||
throw new Error(`Unexpected fixture URL: ${url}`);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const listener of process.listeners('unhandledRejection')) {
|
||||
if (!initialRejectionListeners.has(listener)) {
|
||||
process.removeListener('unhandledRejection', listener);
|
||||
}
|
||||
}
|
||||
expect(mocks.createBrowserClient).not.toHaveBeenCalled();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('Zhihu JSDOM client on Node', () => {
|
||||
it.each(['', 'z_c0=isolated-login', 'd_c0=; z_c0=isolated-login'])('executes the challenge in JSDOM and keeps guest credentials isolated: %s', async (cookies) => {
|
||||
mocks.cookies = cookies;
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(pageUrl, (client) => client.get(apiPath))).toEqual(feedData);
|
||||
|
||||
expect(mocks.raw).toHaveBeenNthCalledWith(1, `${origin}/explore`, {
|
||||
headers: { 'user-agent': userAgent },
|
||||
redirect: 'manual',
|
||||
ignoreResponseError: true,
|
||||
});
|
||||
expect(mocks.raw).toHaveBeenNthCalledWith(2, `${origin}${apiPath}`, {
|
||||
headers: {
|
||||
'user-agent': userAgent,
|
||||
cookie: 'd_c0=guest-identity; __zse_ck=005_x-x',
|
||||
referer: pageUrl,
|
||||
'x-requested-with': 'fetch',
|
||||
},
|
||||
ignoreResponseError: true,
|
||||
});
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('cookie')).toBe('__zse_ck=generated-token; d_c0=guest-identity');
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('user-agent')).toBe(userAgent);
|
||||
expect(mocks.ofetch).toHaveBeenCalledWith(scriptUrl, { headers: { 'user-agent': userAgent }, parseResponse: expect.any(Function) });
|
||||
expect(mocks.tryGet).toHaveBeenCalledWith('zhihu:zse-ck:v4:guest', expect.any(Function), 3600, false);
|
||||
});
|
||||
|
||||
it('preserves the configured identity and login cookies without creating a guest session', async () => {
|
||||
mocks.cookies = 'd_c0=configured==; z_c0=login=value; __zse_ck=; extra=retained';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
|
||||
expect(mocks.raw).toHaveBeenCalledExactlyOnceWith(`${origin}${apiPath}`, expect.objectContaining({ headers: expect.objectContaining({ cookie: 'd_c0=configured==; __zse_ck=005_x-x' }) }));
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('cookie')).toBe('__zse_ck=generated-token; d_c0=configured==; z_c0=login=value; extra=retained');
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('user-agent')).toBe(userAgent);
|
||||
});
|
||||
|
||||
it('keeps API and HTML requests in the same generated identity and signs each API path separately', async () => {
|
||||
mocks.cookies = 'd_c0=configured==; z_c0=login';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
const { getSignedHeaders } = await import('../lib/routes/zhihu/sign');
|
||||
|
||||
const result = await withZhihuClient(pageUrl, async (client) => ({
|
||||
first: await client.get(apiPath),
|
||||
second: await client.get(otherApiPath),
|
||||
html: await client.getPage(),
|
||||
}));
|
||||
|
||||
expect(result).toEqual({ first: feedData, second: feedData, html: pageHtml });
|
||||
expect(mocks.raw).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.generateHeaders).toHaveBeenCalledTimes(1);
|
||||
for (const url of [`${origin}${apiPath}`, `${origin}${otherApiPath}`, pageUrl]) {
|
||||
const headers = requestHeaders(url);
|
||||
expect(headers.get('cookie')).toBe('__zse_ck=generated-token; d_c0=configured==; z_c0=login');
|
||||
expect(headers.get('user-agent')).toBe(userAgent);
|
||||
expect(headers.get('referer')).toBe(pageUrl);
|
||||
}
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('x-zse-96')).toBe(getSignedHeaders(apiPath, 'configured==')['x-zse-96']);
|
||||
expect(requestHeaders(`${origin}${otherApiPath}`).get('x-zse-96')).toBe(getSignedHeaders(otherApiPath, 'configured==')['x-zse-96']);
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('x-zse-96')).not.toBe(requestHeaders(`${origin}${otherApiPath}`).get('x-zse-96'));
|
||||
const htmlRequest = mocks.ofetch.mock.calls.find(([url]) => url === pageUrl);
|
||||
expect(htmlRequest?.[1]?.parseResponse(pageHtml)).toBe(pageHtml);
|
||||
});
|
||||
|
||||
it('initializes from the page path when HTML is requested before an API', async () => {
|
||||
mocks.cookies = 'd_c0=configured==';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(pageUrl, async (client) => {
|
||||
expect(await client.getPage()).toBe(pageHtml);
|
||||
expect(await client.get(apiPath)).toEqual(feedData);
|
||||
});
|
||||
|
||||
expect(mocks.raw).toHaveBeenCalledExactlyOnceWith(pageUrl, expect.objectContaining({ ignoreResponseError: true }));
|
||||
expect(requestHeaders(pageUrl).get('cookie')).toBe(requestHeaders(`${origin}${apiPath}`).get('cookie'));
|
||||
expect(requestHeaders(pageUrl).get('user-agent')).toBe(userAgent);
|
||||
});
|
||||
|
||||
it('deduplicates concurrent challenge generation and reuses cached credentials across clients', async () => {
|
||||
mocks.cookies = 'd_c0=configured==';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
const results = await Promise.all([withZhihuClient(pageUrl, (client) => client.get(apiPath)), withZhihuClient(pageUrl, (client) => client.get(otherApiPath))]);
|
||||
expect(results).toEqual([feedData, feedData]);
|
||||
await withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
|
||||
expect(mocks.raw).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.generateHeaders).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.ofetch.mock.calls.filter(([url]) => url === scriptUrl)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('retries generation after a failed challenge without permanently caching the rejection', async () => {
|
||||
mocks.cookies = 'd_c0=configured==';
|
||||
mocks.raw.mockResolvedValueOnce({ _data: '<html>No challenge script</html>' });
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(pageUrl, async (client) => {
|
||||
await expect(client.get(apiPath)).rejects.toThrow('challenge page');
|
||||
expect(await client.get(apiPath)).toEqual(feedData);
|
||||
});
|
||||
|
||||
expect(mocks.raw).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.tryGet).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.ofetch.mock.calls.filter(([url]) => url === `${origin}${apiPath}`)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reuse generated credentials for a different configured identity', async () => {
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
mocks.cookies = 'd_c0=first-identity; z_c0=first-login';
|
||||
await withZhihuClient(pageUrl, (client) => client.get(apiPath));
|
||||
mocks.cookies = 'd_c0=second-identity; z_c0=second-login';
|
||||
await withZhihuClient(pageUrl, (client) => client.get(otherApiPath));
|
||||
|
||||
expect(mocks.raw).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.tryGet.mock.calls[0][0]).not.toBe(mocks.tryGet.mock.calls[1][0]);
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('cookie')).toBe('__zse_ck=generated-token; d_c0=first-identity; z_c0=first-login');
|
||||
expect(requestHeaders(`${origin}${otherApiPath}`).get('cookie')).toBe('__zse_ck=generated-token; d_c0=second-identity; z_c0=second-login');
|
||||
});
|
||||
|
||||
it('uses complete configured credentials without initializing JSDOM or a browser', async () => {
|
||||
mocks.cookies = 'd_c0=configured==; __zse_ck=existing-token; z_c0=login=value';
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient(pageUrl, async (client) => ({ data: await client.get(apiPath), html: await client.getPage() }))).toEqual({ data: feedData, html: pageHtml });
|
||||
|
||||
expect(requestHeaders(`${origin}${apiPath}`).get('cookie')).toBe(mocks.cookies);
|
||||
expect(requestHeaders(pageUrl).get('cookie')).toBe(mocks.cookies);
|
||||
expect(mocks.raw).not.toHaveBeenCalled();
|
||||
expect(mocks.tryGet).not.toHaveBeenCalled();
|
||||
expect(mocks.generateHeaders).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['', 'd_c0=configured==; __zse_ck=existing-token'])('rejects non-API URLs before any I/O: %s', async (cookies) => {
|
||||
mocks.cookies = cookies;
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
await withZhihuClient(pageUrl, async (client) => {
|
||||
await expect(client.get('https://example.com/api/v4/members/example')).rejects.toThrow('expected an API path');
|
||||
await expect(client.get('/people/example')).rejects.toThrow('expected an API path');
|
||||
});
|
||||
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
expect(mocks.raw).not.toHaveBeenCalled();
|
||||
expect(mocks.tryGet).not.toHaveBeenCalled();
|
||||
expect(mocks.generateHeaders).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,64 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { isWorker } from '@/utils/is-worker';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
cookies: '',
|
||||
ofetch: vi.fn(),
|
||||
createBrowserClient: vi.fn(),
|
||||
createJSDOMClient: vi.fn(() => {
|
||||
throw new Error('JSDOM must never run on Workers');
|
||||
}),
|
||||
get: vi.fn(),
|
||||
getPage: vi.fn(),
|
||||
close: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../lib/config', () => ({
|
||||
config: {
|
||||
zhihu: {
|
||||
get cookies() {
|
||||
return mocks.cookies;
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock('../lib/utils/ofetch', () => ({ default: mocks.ofetch }));
|
||||
vi.mock('../lib/routes/zhihu/browser', () => ({ createBrowserClient: mocks.createBrowserClient }));
|
||||
vi.mock('../lib/routes/zhihu/jsdom', () => ({ createJSDOMClient: mocks.createJSDOMClient }));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mocks.cookies = '';
|
||||
mocks.createBrowserClient.mockResolvedValue({ get: mocks.get, getPage: mocks.getPage, close: mocks.close });
|
||||
});
|
||||
|
||||
describe('Zhihu runtime selection in workerd', () => {
|
||||
it('uses a browser session for both API and HTML requests', async () => {
|
||||
expect(isWorker).toBe(true);
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
mocks.get.mockResolvedValue({ data: [1] });
|
||||
mocks.getPage.mockResolvedValue('<html>Column</html>');
|
||||
|
||||
await withZhihuClient('https://zhuanlan.zhihu.com/example', async (client) => {
|
||||
expect(await client.get('/api/v4/columns/example/items')).toEqual({ data: [1] });
|
||||
expect(await client.getPage()).toBe('<html>Column</html>');
|
||||
});
|
||||
|
||||
expect(mocks.createBrowserClient).toHaveBeenCalledExactlyOnceWith('https://zhuanlan.zhihu.com/example', '', '/api/v4/columns/example/items');
|
||||
expect(mocks.close).toHaveBeenCalledOnce();
|
||||
expect(mocks.createJSDOMClient).not.toHaveBeenCalled();
|
||||
expect(mocks.ofetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses complete configured cookies without either initializer', async () => {
|
||||
mocks.cookies = 'd_c0=configured; __zse_ck=existing';
|
||||
mocks.ofetch.mockResolvedValue({ data: [1] });
|
||||
const { withZhihuClient } = await import('../lib/routes/zhihu/utils');
|
||||
|
||||
expect(await withZhihuClient('https://www.zhihu.com/people/example', (client) => client.get('/api/v4/members/example'))).toEqual({ data: [1] });
|
||||
expect(mocks.createBrowserClient).not.toHaveBeenCalled();
|
||||
expect(mocks.createJSDOMClient).not.toHaveBeenCalled();
|
||||
expect(mocks.ofetch).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -85,6 +85,8 @@ export default defineConfig({
|
||||
'@sentry/node': path.resolve('./lib/shims/sentry-node.ts'),
|
||||
'@honeybadger-io/js': path.resolve('./lib/shims/honeybadger.ts'),
|
||||
typescript: path.resolve('./lib/shims/typescript.ts'),
|
||||
jsdom: path.resolve('./lib/shims/jsdom.ts'),
|
||||
vm2: path.resolve('./lib/shims/vm2.ts'),
|
||||
// Routes file with Worker-specific build (match relative import from lib/)
|
||||
'../assets/build/routes.js': path.resolve('./assets/build/routes-worker.js'),
|
||||
// routes.json is only used in test environment, but rolldown still tries to resolve it
|
||||
@@ -95,6 +97,7 @@ export default defineConfig({
|
||||
neverBundle: [
|
||||
// Let Wrangler select the package's static-WASM workerd export.
|
||||
'xxhash-wasm',
|
||||
'@oxc-parser/binding-wasm32-wasip1/wasm.wasm',
|
||||
// Exclude non-code files that might be accidentally imported
|
||||
/\/_README$/,
|
||||
/\.node$/,
|
||||
|
||||
@@ -67,9 +67,11 @@ export default defineConfig({
|
||||
resolve: {
|
||||
alias: {
|
||||
'dotenv/config': path.resolve('./lib/shims/dotenv-config.ts'),
|
||||
jsdom: path.resolve('./lib/shims/jsdom.ts'),
|
||||
vm2: path.resolve('./lib/shims/vm2.ts'),
|
||||
},
|
||||
},
|
||||
test: {
|
||||
include: ['lib/**/*.worker.test.ts', 'tests/**/*.worker.test.ts', 'lib/utils/parse-script-data.test.ts', 'lib/utils/parse-date-in-timezone.test.ts', 'tests/source-date-routes.test.ts', 'tests/cache-coordination.test.ts'],
|
||||
include: ['lib/**/*.worker.test.ts', 'tests/**/*.worker.test.ts'],
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user