opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-08-30 16:55:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(ua): ensure header-generator and custom ua are effective with each other (#21546)

Browse Source 
* fix(ua): ensure header-generator and custom ua are effective with each other

* fix(ua): review comment
This commit is contained in:
Tony
2026-03-29 22:47:56 +08:00
committed by GitHub
parent cacb6e46c6
commit 7ab0d1eff4
7 changed files with 50 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/app.test.ts
+1 -3
View File
@@ -3,8 +3,6 @@ import { describe, expect, it, vi } from 'vitest';
import app from '@/app';
const { config } = await import('@/config');
describe('index', () => {
it('serve index', async () => {
const res = await app.request('/');
@@ -20,6 +18,6 @@ describe('request-rewriter', () => {
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('user-agent')).toMatch(/Chrome/);
});
});
lib/config.test.ts
+4 -2
View File
@@ -84,9 +84,11 @@ describe('config', () => {
delete process.env.NO_RANDOM_UA;
});
it('random ua', async () => {
it('default ua from preset', async () => {
const { config } = await import('./config');
expect(config.ua).not.toBe('RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)');
expect(config.ua).toContain('Chrome');
expect(config.ua).toContain('Macintosh');
expect(config.isDefaultUA).toBe(true);
});
it('remote config', async () => {
lib/config.ts
+3 -1
View File
@@ -263,6 +263,7 @@ export type Config = {
requestRetry: number;
requestTimeout: number;
ua: string;
isDefaultUA: boolean;
trueUA: string;
allowOrigin?: string;
// cache
@@ -744,7 +745,8 @@ const calculateValue = () => {
listenInaddrAny: toBoolean(envs.LISTEN_INADDR_ANY, true), // 是否允许公网连接，取值 0 1
requestRetry: toInt(envs.REQUEST_RETRY, 2), // 请求失败重试次数
requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request
ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 15_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'),
ua: envs.UA || (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 15_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'),
isDefaultUA: !envs.UA && !toBoolean(envs.NO_RANDOM_UA, false),
trueUA: TRUE_UA,
allowOrigin: envs.ALLOW_ORIGIN,
// cache
lib/utils/request-rewriter.test.ts
+3 -3
View File
@@ -91,7 +91,7 @@ describe('request-rewriter', () => {
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('user-agent')).toMatch(/Chrome/);
expect(headers.get('accept')).toBeDefined();
expect(headers.get('referer')).toBe('http://rsshub.test');
expect(headers.get('sec-ch-ua')).toBeDefined();
@@ -138,7 +138,7 @@ describe('request-rewriter', () => {
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('user-agent')).toMatch(/Chrome/);
expect(headers.get('accept')).toBeDefined();
expect(headers.get('referer')).toBe('http://rsshub.test');
expect(headers.get('sec-ch-ua')).toBeDefined();
@@ -237,7 +237,7 @@ describe('request-rewriter', () => {
// headers
const options = httpSpy.mock.lastCall?.[1];
const headers = options?.headers;
expect(headers?.['user-agent']).toBe(config.ua);
expect(headers?.['user-agent']).toMatch(/Chrome/);
expect(headers?.accept).toBeDefined();
expect(headers?.referer).toBe('http://rsshub.test');
lib/utils/request-rewriter/fetch.ts
+12 -8
View File
@@ -35,17 +35,21 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
const generatedHeaders = generateHeaders(init?.headerGeneratorOptions);
// ua
if (!request.headers.has('user-agent')) {
request.headers.set('user-agent', config.ua);
}
if (config.isDefaultUA || init?.headerGeneratorOptions) {
const generatedHeaders = generateHeaders(init?.headerGeneratorOptions);
for (const header of HEADER_LIST) {
if (!request.headers.has(header) && generatedHeaders[header]) {
request.headers.set(header, generatedHeaders[header]);
if (!request.headers.get('user-agent')) {
request.headers.set('user-agent', generatedHeaders['user-agent']);
}
for (const header of HEADER_LIST) {
if (!request.headers.has(header) && generatedHeaders[header]) {
request.headers.set(header, generatedHeaders[header]);
}
}
} else if (!request.headers.get('user-agent')) {
request.headers.set('user-agent', config.ua);
}
// referer
lib/utils/request-rewriter/fetch.worker.ts
+15 -10
View File
@@ -4,12 +4,13 @@ import { config } from '@/config';
import logger from '@/utils/logger';
// Static browser headers (Chrome-like fingerprint)
const STATIC_BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 15_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36';
const STATIC_BROWSER_HEADERS: Record<string, string> = {
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
'sec-ch-ua': '"Google Chrome";v="139", "Chromium";v="139", "Not_A Brand";v="24"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
@@ -25,15 +26,19 @@ const wrappedFetch = (input: RequestInfo | URL, init?: RequestInit): Promise<Res
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
// Set User-Agent if not provided
if (!request.headers.has('user-agent')) {
request.headers.set('user-agent', config.ua);
}
// Set browser headers if not provided
for (const [header, value] of Object.entries(STATIC_BROWSER_HEADERS)) {
if (!request.headers.has(header)) {
request.headers.set(header, value);
if (config.isDefaultUA) {
if (!request.headers.get('user-agent')) {
request.headers.set('user-agent', STATIC_BROWSER_UA);
}
// Set browser headers if not provided
for (const [header, value] of Object.entries(STATIC_BROWSER_HEADERS)) {
if (!request.headers.has(header)) {
request.headers.set(header, value);
}
}
} else if (!request.headers.get('user-agent')) {
request.headers.set('user-agent', config.ua);
}
// Set Referer if not provided
lib/utils/request-rewriter/get.ts
+12 -8
View File
@@ -48,17 +48,21 @@ const getWrappedGet: <T extends Get>(origin: T) => T = (origin) =>
options.headers = options.headers || {};
const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase()));
const generatedHeaders = generateHeaders(options.headerGeneratorOptions);
// ua
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
if (config.isDefaultUA || options.headerGeneratorOptions) {
const generatedHeaders = generateHeaders(options.headerGeneratorOptions);
for (const header of HEADER_LIST) {
if (!headersLowerCaseKeys.has(header) && generatedHeaders[header]) {
options.headers[header] = generatedHeaders[header];
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = generatedHeaders['user-agent'];
}
for (const header of HEADER_LIST) {
if (!headersLowerCaseKeys.has(header) && generatedHeaders[header]) {
options.headers[header] = generatedHeaders[header];
}
}
} else if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
// referer