From 3158a469569eea93d54c58ecf019807a54896850 Mon Sep 17 00:00:00 2001 From: Rongrong Date: Tue, 2 Dec 2025 11:59:52 +0800 Subject: [PATCH] fix(route/weibo/utils): tryWithCookies: Cookies expiration not detected (#20583) * feat(route/weibo/utils): getCookies: Add cooldown time Signed-off-by: Rongrong * fix(route/weibo/utils): tryWithCookies: Cookies expiration not detected Signed-off-by: Rongrong --------- Signed-off-by: Rongrong --- lib/routes/weibo/keyword.ts | 3 +- lib/routes/weibo/search/hot.ts | 10 +-- lib/routes/weibo/super-index.ts | 3 +- lib/routes/weibo/user.ts | 6 +- lib/routes/weibo/utils.ts | 120 +++++++++++++++++++------------- 5 files changed, 84 insertions(+), 58 deletions(-) diff --git a/lib/routes/weibo/keyword.ts b/lib/routes/weibo/keyword.ts index a5ec40a7ab..2ba00c5e96 100644 --- a/lib/routes/weibo/keyword.ts +++ b/lib/routes/weibo/keyword.ts @@ -38,7 +38,7 @@ export const route: Route = { async function handler(ctx) { const keyword = ctx.req.param('keyword'); - const data = await weiboUtils.tryWithCookies((cookies) => + const data = await weiboUtils.tryWithCookies((cookies, verifier) => cache.tryGet( `weibo:keyword:${keyword}`, async () => { @@ -51,6 +51,7 @@ async function handler(ctx) { ...weiboUtils.apiHeaders, }, }); + verifier(_r); return _r.data.data.cards; }, config.cache.routeExpire, diff --git a/lib/routes/weibo/search/hot.ts b/lib/routes/weibo/search/hot.ts index cd28c42ab7..b39308b99e 100644 --- a/lib/routes/weibo/search/hot.ts +++ b/lib/routes/weibo/search/hot.ts @@ -59,8 +59,8 @@ async function handler(ctx) { fullpic = ctx.req.query('fullpic') ?? 'false'; const { data: { data }, - } = await weiboUtils.tryWithCookies((cookies) => - got({ + } = await weiboUtils.tryWithCookies(async (cookies, verifier) => { + const _r = await got({ method: 'get', url: 'https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=filter_type%3Drealtimehot%26mi_cid%3D100103%26pos%3D0_0%26c_type%3D30%26display_time%3D1540538388&luicode=10000011&lfid=231583', headers: { @@ -68,8 +68,10 @@ async function handler(ctx) { Cookie: cookies, ...weiboUtils.apiHeaders, }, - }) - ); + }); + verifier(_r); + return _r; + }); let resultItems = null; if (ctx.req.param('fulltext') === 'fulltext') { diff --git a/lib/routes/weibo/super-index.ts b/lib/routes/weibo/super-index.ts index 6aaed19b03..fc2f372230 100644 --- a/lib/routes/weibo/super-index.ts +++ b/lib/routes/weibo/super-index.ts @@ -53,7 +53,7 @@ async function handler(ctx) { const id = ctx.req.param('id'); const type = ctx.req.param('type') ?? 'feed'; - const containerData = (await weiboUtils.tryWithCookies((cookies) => + const containerData = (await weiboUtils.tryWithCookies((cookies, verifier) => cache.tryGet( `weibo:super_index:container:${id}:${type}`, async () => { @@ -69,6 +69,7 @@ async function handler(ctx) { ...weiboUtils.apiHeaders, }, }); + verifier(_r); return _r.data.data; }, config.cache.routeExpire, diff --git a/lib/routes/weibo/user.ts b/lib/routes/weibo/user.ts index cee89df350..2c8b8c6db9 100644 --- a/lib/routes/weibo/user.ts +++ b/lib/routes/weibo/user.ts @@ -73,7 +73,7 @@ async function handler(ctx) { } } - const containerData = await weiboUtils.tryWithCookies((cookies) => + const containerData = await weiboUtils.tryWithCookies((cookies, verifier) => cache.tryGet( `weibo:user:index:${uid}`, async () => { @@ -86,6 +86,7 @@ async function handler(ctx) { ...weiboUtils.apiHeaders, }, }); + verifier(_r); return _r.data; }, config.cache.routeExpire, @@ -98,7 +99,7 @@ async function handler(ctx) { const profileImageUrl = containerData.data.userInfo.profile_image_url; const containerId = containerData.data.tabsInfo.tabs.find((item) => item.tab_type === 'weibo').containerid; - const cards = await weiboUtils.tryWithCookies((cookies) => + const cards = await weiboUtils.tryWithCookies((cookies, verifier) => cache.tryGet( `weibo:user:cards:${uid}:${containerId}`, async () => { @@ -111,6 +112,7 @@ async function handler(ctx) { ...weiboUtils.apiHeaders, }, }); + verifier(_r); return _r.data.data.cards; }, config.cache.routeExpire, diff --git a/lib/routes/weibo/utils.ts b/lib/routes/weibo/utils.ts index 3c553ff8c2..71da1c0f89 100644 --- a/lib/routes/weibo/utils.ts +++ b/lib/routes/weibo/utils.ts @@ -24,62 +24,82 @@ const weiboUtils = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', }, RenewWeiboCookiesError, - getCookies: async (renew = false) => { - if (config.weibo.cookies) { - if (renew) { - throw new Error('Cookies expired. Please update WEIBO_COOKIES'); - } - return config.weibo.cookies; - } + getCookies: (() => { + const url = 'https://m.weibo.cn/'; + const coolingDownMessage = `Cooling down before new visitor Cookies from ${url} may be fetched`; + let coolingDown: boolean = false; - const cacheKey = 'weibo:guest-cookies'; - if (renew) { - cache.set(cacheKey, '', 1); - } - return await cache.tryGet(cacheKey, async () => { - const url = 'https://m.weibo.cn/'; + return async (renew: any = false) => { + if (config.weibo.cookies) { + if (renew) { + throw new Error('Cookies expired. Please update WEIBO_COOKIES'); + } + return config.weibo.cookies; + } + + const cacheKey = 'weibo:visitor-cookies'; if (renew) { - logger.warn(`Renewing visitor Cookies from ${url}`); - } else { - logger.info(`Fetching visitor Cookies from ${url}`); + cache.set(cacheKey, '', 1); } - let times = 0; - const { page, destory } = await getPuppeteerPage(url, { - onBeforeLoad: async (page) => { - const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); - await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']); - await page.setRequestInterception(true); - page.on('request', (request) => { - // 1st: initial request, 302 to visitor.passport.weibo.cn; 2nd: auth ok - if (!expectResourceTypes.has(request.resourceType()) || times >= 2) { - request.abort(); - return; - } - if (request.url().startsWith(url)) { - times++; - } - request.continue(); - }); - }, - // networkidle2 returns too early if the connection is slow - gotoConfig: { waitUntil: 'networkidle0' }, + return await cache.tryGet(cacheKey, async () => { + if (coolingDown) { + if (renew?.message) { + logger.warn(coolingDownMessage); + throw renew; + } else { + throw new Error(coolingDownMessage); + } + } + coolingDown = true; + setTimeout(() => { + coolingDown = false; + }, config.cache.routeExpire * 1000); + + if (renew) { + logger.warn(`Renewing visitor Cookies from ${url}`); + } else { + logger.info(`Fetching visitor Cookies from ${url}`); + } + let times = 0; + const { page, destory } = await getPuppeteerPage(url, { + onBeforeLoad: async (page) => { + const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); + await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']); + await page.setRequestInterception(true); + page.on('request', (request) => { + // 1st: initial request, 302 to visitor.passport.weibo.cn; 2nd: auth ok + if (!expectResourceTypes.has(request.resourceType()) || times >= 2) { + request.abort(); + return; + } + if (request.url().startsWith(url)) { + times++; + } + request.continue(); + }); + }, + // networkidle2 returns too early if the connection is slow + gotoConfig: { waitUntil: 'networkidle0' }, + }); + const cookies: string = await getCookies(page, 'weibo.cn'); + await destory(); + if (times < 2 || !cookies) { + throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Redirection: ${times}, last URL: ${page.url()}`); + } + return cookies; }); - const cookies: string = await getCookies(page, 'weibo.cn'); - await destory(); - if (times < 2) { - throw new Error(`Unexpected redirection. Last URL: ${page.url()}`); - } - if (!cookies) { - throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Last URL: ${page.url()}`); - } - return cookies; - }); - }, + }; + })(), tryWithCookies: (() => { let errors: number = 0; - return async (callback: (cookies: string) => Promise) => { + const verifier = (resp: any): void => { + if (resp?.data?.ok === -100) { + throw new RenewWeiboCookiesError(`Cookies expired. Msg: ${resp?.data?.msg || ''} ${resp?.data?.url || ''}`); + } + }; + return async (callback: (cookies: string, verifier: (resp: any) => void) => Promise): Promise => { try { - return await callback(await weiboUtils.getCookies(false)); + return await callback(await weiboUtils.getCookies(false), verifier); } catch (error: any) { if (error.message?.includes('WEIBO_COOKIES')) { throw error; @@ -96,7 +116,7 @@ const weiboUtils = { throw error; } errors = 0; - return await callback(await weiboUtils.getCookies(true)); + return await callback(await weiboUtils.getCookies(error), verifier); } }; })(),