From dcd21b8c0ad25cf3dfc4c5e198ae4f1c1f353f3d Mon Sep 17 00:00:00 2001 From: sirius60111 Date: Sun, 12 Jul 2026 19:42:04 +0800 Subject: [PATCH] feat(route/xueqiu): fetch user timeline via API without browser (#22348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /xueqiu/user route drove a headless browser (Patchright) to load the profile page and navigate to each status for full text. The user timeline is public, so the data can be fetched directly from api.xueqiu.com once the WAF challenge cookie is obtained. - replace the browser navigation with direct ofetch calls to api.xueqiu.com/v4/statuses/user_timeline.json and statuses/show.json - requirePuppeteer: false (no longer drives a browser); antiCrawler: true - pass source=买卖 for the 交易 (type 11) tab so it filters correctly - skip the show.json detail request when legal_user_visible is true - inline images and the retweeted status into the description - derive screen_name and the avatar image from the timeline response Co-authored-by: Claude Opus 4.8 (1M context) --- lib/routes/xueqiu/user.ts | 214 ++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 110 deletions(-) diff --git a/lib/routes/xueqiu/user.ts b/lib/routes/xueqiu/user.ts index ccda3ba3c3..97b0bd2446 100644 --- a/lib/routes/xueqiu/user.ts +++ b/lib/routes/xueqiu/user.ts @@ -3,10 +3,11 @@ import sanitizeHtml from 'sanitize-html'; import { parseToken } from '@/routes/xueqiu/cookies'; import type { Route } from '@/types'; import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -import playwright from '@/utils/playwright'; const rootUrl = 'https://xueqiu.com'; +const apiUrl = 'https://api.xueqiu.com'; export const route: Route = { path: '/user/:id/:type?', @@ -15,8 +16,8 @@ export const route: Route = { parameters: { id: '用户 id, 可在用户主页 URL 中找到', type: '动态的类型, 不填则默认全部' }, features: { requireConfig: false, - requirePuppeteer: true, - antiCrawler: false, + requirePuppeteer: false, + antiCrawler: true, supportBT: false, supportPodcast: false, supportScihub: false, @@ -35,9 +36,59 @@ export const route: Route = { | 0 | 2 | 4 | 9 | 11 |`, }; +const stripHtml = (html: string): string => sanitizeHtml(html, { allowedTags: [], allowedAttributes: {} }); + +// Build a feed item from the timeline list data alone (no detail request). +const buildListItem = (item: any) => ({ + title: item.title || stripHtml(item.description || ''), + description: item.description || '', + pubDate: parseDate(item.created_at), + link: rootUrl + item.target, +}); + +const buildTitle = (item: any, detail: any): string => { + if (item.title) { + return item.title; + } + return stripHtml(item.text || item.description || detail.text || ''); +}; + +const buildDescription = (detail: any): string => { + let text = detail.text ?? detail.description; + const images = detail.image_info_list ?? []; + for (const img of images) { + if (img?.filename) { + text += `
`; + } + } + if (detail.retweeted_status) { + text += `
${detail.retweeted_status.user.screen_name}: ${detail.retweeted_status.text}
`; + } + return text; +}; + +const extractProfileImage = (user: any): string | undefined => { + if (!user?.profile_image_url || !user?.photo_domain) { + return undefined; + } + + const imageUrls = user.profile_image_url.split(',').filter(Boolean); + if (imageUrls.length === 0) { + return undefined; + } + + // Priority order for image sizes + const sizePriority = ['!180x180.png', '!50x50.png', '!30x30.png']; + const selectedImageUrl = sizePriority.map((size) => imageUrls.find((url) => url.includes(size))).find(Boolean) || imageUrls[0]; + const baseDomain = user.photo_domain.startsWith('//') ? `https:${user.photo_domain}` : user.photo_domain; + + return `${baseDomain}${selectedImageUrl}`; +}; + async function handler(ctx) { const id = ctx.req.param('id'); const type = ctx.req.param('type') || 10; + const source = type === '11' ? '买卖' : ''; const typename = { 10: '全部', 0: '原发布', @@ -48,119 +99,62 @@ async function handler(ctx) { }; const link = `${rootUrl}/u/${id}`; - const token = await parseToken(link); + const cookie = await parseToken(link); - const context = await playwright(); - try { - const mainPage = await context.newPage(); - - await mainPage.setExtraHTTPHeaders({ - Cookie: token as string, + const response = await ofetch(`${apiUrl}/v4/statuses/user_timeline.json`, { + query: { + user_id: id, + type, + source, + }, + headers: { + Cookie: cookie, Referer: link, - }); + }, + }); - await mainPage.goto(link, { - waitUntil: 'domcontentloaded', - }); - await mainPage.waitForFunction(() => document.readyState === 'complete'); + const data = response.statuses.filter((s) => s.mark !== 1); // 去除置顶动态 - const apiUrl = `${rootUrl}/v4/statuses/user_timeline.json?user_id=${id}&type=${type}`; - const response = await mainPage.evaluate(async (url) => { - const response = await fetch(url); - return response.json(); - }, apiUrl); + const items = await Promise.all( + data.map((item) => + cache.tryGet(item.target, async () => { + // legal_user_visible 为 true 时列表已含完整内容，无需再请求详情 + if (item.legal_user_visible) { + return buildListItem(item); + } - if (!response?.statuses) { - throw new Error('获取用户动态数据失败'); - } + try { + const detail = await ofetch(`${apiUrl}/statuses/show.json`, { + query: { + id: item.id, + }, + headers: { + Cookie: cookie, + Referer: link, + }, + }); - const data = response.statuses.filter((s) => s.mark !== 1); + return { + title: buildTitle(item, detail), + description: buildDescription(detail), + pubDate: parseDate(item.created_at), + link: rootUrl + item.target, + }; + } catch { + return buildListItem(item); + } + }) + ) + ); - if (!data.length) { - throw new Error('未找到有效的动态数据'); - } + const user = data[0]?.user; - const items = await Promise.all( - data.map((item) => - cache.tryGet(item.target, async () => { - const detailUrl = rootUrl + item.target; - try { - await mainPage.goto(detailUrl, { - waitUntil: 'domcontentloaded', - }); - await mainPage.waitForFunction(() => document.readyState === 'complete'); - - const content = await mainPage.evaluate(() => { - const articleContent = document.querySelector('.article__bd')?.innerHTML || ''; - const statusMatch = document.documentElement.innerHTML.match(/SNOWMAN_STATUS = (.*?\});/); - return { - articleContent, - statusData: statusMatch ? statusMatch[1] : null, - }; - }); - - if (content.statusData) { - const data = JSON.parse(content.statusData); - item.text = data.text; - } - - const retweetedStatus = item.retweeted_status ? `
${item.retweeted_status.user.screen_name}: ${item.retweeted_status.description}
` : ''; - const description = content.articleContent || item.description + retweetedStatus; - - return { - title: item.title || sanitizeHtml(description, { allowedTags: [], allowedAttributes: {} }), - description: item.text ? item.text + retweetedStatus : description, - pubDate: parseDate(item.created_at), - link: rootUrl + item.target, - }; - } catch (error: unknown) { - if (error instanceof Error && !error.message?.includes('ERR_ABORTED')) { - throw error; - } - const retweetedStatus = item.retweeted_status ? `
${item.retweeted_status.user.screen_name}: ${item.retweeted_status.description}
` : ''; - const description = item.description + retweetedStatus; - - return { - title: item.title || sanitizeHtml(description, { allowedTags: [], allowedAttributes: {} }), - description: item.description, - pubDate: parseDate(item.created_at), - link: rootUrl + item.target, - }; - } - }) - ) - ); - - const extractProfileImage = (user: any): string | undefined => { - if (!user?.profile_image_url || !user?.photo_domain) { - return undefined; - } - - const imageUrls = user.profile_image_url.split(',').filter(Boolean); - if (imageUrls.length === 0) { - return undefined; - } - - // Priority order for image sizes - const sizePriority = ['!180x180.png', '!50x50.png', '!30x30.png']; - - const selectedImageUrl = sizePriority.map((size) => imageUrls.find((url) => url.includes(size))).find(Boolean) || imageUrls[0]; - - const baseDomain = user.photo_domain.startsWith('//') ? `https:${user.photo_domain}` : user.photo_domain; - - return `${baseDomain}${selectedImageUrl}`; - }; - - const profileImage = extractProfileImage(data[0].user); - - return { - title: `${data[0].user.screen_name} 的雪球${typename[type]}动态`, - link, - description: `${data[0].user.screen_name} 的雪球${typename[type]}动态`, - image: profileImage, - item: items, - }; - } finally { - await context.close(); - } + return { + title: `${user?.screen_name ?? id} 的雪球${typename[type]}动态`, + link, + description: `${user?.screen_name ?? id} 的雪球${typename[type]}动态`, + image: extractProfileImage(user), + item: items, + allowEmpty: true, + }; }