diff --git a/lib/app.test.ts b/lib/app.test.ts index 185cc6f620..88ba651383 100644 --- a/lib/app.test.ts +++ b/lib/app.test.ts @@ -3,8 +3,6 @@ import { describe, expect, it, vi } from 'vitest'; import app from '@/app'; -const { config } = await import('@/config'); - describe('index', () => { it('serve index', async () => { const res = await app.request('/'); @@ -20,6 +18,6 @@ describe('request-rewriter', () => { // headers const headers: Headers = fetchSpy.mock.lastCall?.[0].headers; - expect(headers.get('user-agent')).toBe(config.ua); + expect(headers.get('user-agent')).toMatch(/Chrome/); }); }); diff --git a/lib/config.test.ts b/lib/config.test.ts index c5f79d68d1..02010cb5d4 100644 --- a/lib/config.test.ts +++ b/lib/config.test.ts @@ -84,9 +84,11 @@ describe('config', () => { delete process.env.NO_RANDOM_UA; }); - it('random ua', async () => { + it('default ua from preset', async () => { const { config } = await import('./config'); - expect(config.ua).not.toBe('RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)'); + expect(config.ua).toContain('Chrome'); + expect(config.ua).toContain('Macintosh'); + expect(config.isDefaultUA).toBe(true); }); it('remote config', async () => { diff --git a/lib/config.ts b/lib/config.ts index 61e5204f88..e03faac7b3 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -263,6 +263,7 @@ export type Config = { requestRetry: number; requestTimeout: number; ua: string; + isDefaultUA: boolean; trueUA: string; allowOrigin?: string; // cache @@ -744,7 +745,8 @@ const calculateValue = () => { listenInaddrAny: toBoolean(envs.LISTEN_INADDR_ANY, true), // 是否允许公网连接，取值 0 1 requestRetry: toInt(envs.REQUEST_RETRY, 2), // 请求失败重试次数 requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request - ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 15_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'), + ua: envs.UA || (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 15_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'), + isDefaultUA: !envs.UA && !toBoolean(envs.NO_RANDOM_UA, false), trueUA: TRUE_UA, allowOrigin: envs.ALLOW_ORIGIN, // cache diff --git a/lib/utils/request-rewriter.test.ts b/lib/utils/request-rewriter.test.ts index 0407fbfef8..794868e969 100644 --- a/lib/utils/request-rewriter.test.ts +++ b/lib/utils/request-rewriter.test.ts @@ -91,7 +91,7 @@ describe('request-rewriter', () => { // headers const headers: Headers = fetchSpy.mock.lastCall?.[0].headers; - expect(headers.get('user-agent')).toBe(config.ua); + expect(headers.get('user-agent')).toMatch(/Chrome/); expect(headers.get('accept')).toBeDefined(); expect(headers.get('referer')).toBe('http://rsshub.test'); expect(headers.get('sec-ch-ua')).toBeDefined(); @@ -138,7 +138,7 @@ describe('request-rewriter', () => { // headers const headers: Headers = fetchSpy.mock.lastCall?.[0].headers; - expect(headers.get('user-agent')).toBe(config.ua); + expect(headers.get('user-agent')).toMatch(/Chrome/); expect(headers.get('accept')).toBeDefined(); expect(headers.get('referer')).toBe('http://rsshub.test'); expect(headers.get('sec-ch-ua')).toBeDefined(); @@ -237,7 +237,7 @@ describe('request-rewriter', () => { // headers const options = httpSpy.mock.lastCall?.[1]; const headers = options?.headers; - expect(headers?.['user-agent']).toBe(config.ua); + expect(headers?.['user-agent']).toMatch(/Chrome/); expect(headers?.accept).toBeDefined(); expect(headers?.referer).toBe('http://rsshub.test'); diff --git a/lib/utils/request-rewriter/fetch.ts b/lib/utils/request-rewriter/fetch.ts index 8acfe06e58..5d36e56f57 100644 --- a/lib/utils/request-rewriter/fetch.ts +++ b/lib/utils/request-rewriter/fetch.ts @@ -35,17 +35,21 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ logger.debug(`Outgoing request: ${request.method} ${request.url}`); - const generatedHeaders = generateHeaders(init?.headerGeneratorOptions); - // ua - if (!request.headers.has('user-agent')) { - request.headers.set('user-agent', config.ua); - } + if (config.isDefaultUA || init?.headerGeneratorOptions) { + const generatedHeaders = generateHeaders(init?.headerGeneratorOptions); - for (const header of HEADER_LIST) { - if (!request.headers.has(header) && generatedHeaders[header]) { - request.headers.set(header, generatedHeaders[header]); + if (!request.headers.get('user-agent')) { + request.headers.set('user-agent', generatedHeaders['user-agent']); } + + for (const header of HEADER_LIST) { + if (!request.headers.has(header) && generatedHeaders[header]) { + request.headers.set(header, generatedHeaders[header]); + } + } + } else if (!request.headers.get('user-agent')) { + request.headers.set('user-agent', config.ua); } // referer diff --git a/lib/utils/request-rewriter/fetch.worker.ts b/lib/utils/request-rewriter/fetch.worker.ts index 022e507e73..62c3846d2b 100644 --- a/lib/utils/request-rewriter/fetch.worker.ts +++ b/lib/utils/request-rewriter/fetch.worker.ts @@ -4,12 +4,13 @@ import { config } from '@/config'; import logger from '@/utils/logger'; // Static browser headers (Chrome-like fingerprint) +const STATIC_BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 15_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'; const STATIC_BROWSER_HEADERS: Record = { accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"', + 'sec-ch-ua': '"Google Chrome";v="139", "Chromium";v="139", "Not_A Brand";v="24"', 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', + 'sec-ch-ua-platform': '"macOS"', 'sec-fetch-dest': 'document', 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none', @@ -25,15 +26,19 @@ const wrappedFetch = (input: RequestInfo | URL, init?: RequestInit): Promise(origin: T) => T = (origin) => options.headers = options.headers || {}; const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase())); - const generatedHeaders = generateHeaders(options.headerGeneratorOptions); - // ua - if (!headersLowerCaseKeys.has('user-agent')) { - options.headers['user-agent'] = config.ua; - } + if (config.isDefaultUA || options.headerGeneratorOptions) { + const generatedHeaders = generateHeaders(options.headerGeneratorOptions); - for (const header of HEADER_LIST) { - if (!headersLowerCaseKeys.has(header) && generatedHeaders[header]) { - options.headers[header] = generatedHeaders[header]; + if (!headersLowerCaseKeys.has('user-agent')) { + options.headers['user-agent'] = generatedHeaders['user-agent']; } + + for (const header of HEADER_LIST) { + if (!headersLowerCaseKeys.has(header) && generatedHeaders[header]) { + options.headers[header] = generatedHeaders[header]; + } + } + } else if (!headersLowerCaseKeys.has('user-agent')) { + options.headers['user-agent'] = config.ua; } // referer