opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-09-01 14:57:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14922 from DIYgod/feature/ofetch

Browse Source 
feat: use ofetch
This commit is contained in:
DIYgod
2024-03-31 18:58:48 +01:00
committed by GitHub
parent 614f6dc12f 425a131083
commit ffd5f26d30
37 changed files with 1344 additions and 1109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/app.tsx
+1 -1
View File
@@ -1,4 +1,4 @@
import '@/utils/request-wrapper';
import '@/utils/request-rewriter';
import { Hono } from 'hono';
lib/config.test.ts
-6
View File
@@ -1,5 +1,4 @@
import { describe, expect, it, afterEach, vi } from 'vitest';
import nock from 'nock';
afterEach(() => {
vi.resetModules();
@@ -93,11 +92,6 @@ describe('config', () => {
it('remote config', async () => {
process.env.REMOTE_CONFIG = 'http://rsshub.test/config';
nock(/rsshub\.test/)
.get('/config')
.reply(200, {
UA: 'test',
});
const { config } = await import('./config');
await new Promise((resolve) => setTimeout(resolve, 100));
expect(config.ua).toBe('test');
lib/config.ts
+3 -4
View File
@@ -1,6 +1,6 @@
import 'dotenv/config';
import randUserAgent from '@/utils/rand-user-agent';
import got from 'got';
import { ofetch } from 'ofetch';
let envs = process.env;
@@ -634,9 +634,8 @@ const calculateValue = () => {
calculateValue();
if (envs.REMOTE_CONFIG) {
got.get(envs.REMOTE_CONFIG)
.then(async (response) => {
const data = JSON.parse(response.body);
ofetch(envs.REMOTE_CONFIG)
.then(async (data) => {
if (data) {
envs = Object.assign(envs, data);
calculateValue();
lib/errors/index.test.ts
+1 -1
View File
@@ -22,7 +22,7 @@ describe('httperror', () => {
it(`httperror`, async () => {
const response = await request.get('/test/httperror');
expect(response.status).toBe(503);
expect(response.text).toMatch('Response code 404 (Not Found): target website might be blocking our access, you can host your own RSSHub instance for a better usability.');
expect(response.text).toMatch('404 Not Found: target website might be blocking our access, you can host your own RSSHub instance for a better usability.');
}, 20000);
});
lib/errors/index.tsx
+1 -1
View File
@@ -39,7 +39,7 @@ export const errorHandler: ErrorHandler = (error, ctx) => {
}
let message = '';
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError' || error.name === 'FetchError')) {
ctx.status(503);
message = `${error.message}: target website might be blocking our access, you can host your own RSSHub instance for a better usability.`;
} else if (error instanceof RequestInProgressError) {
lib/middleware/cache.test.ts
+5 -5
View File
@@ -3,7 +3,7 @@ import Parser from 'rss-parser';
import wait from '@/utils/wait';
process.env.CACHE_EXPIRE = '1';
process.env.CACHE_CONTENT_EXPIRE = '3';
process.env.CACHE_CONTENT_EXPIRE = '2';
const parser = new Parser();
@@ -38,7 +38,7 @@ describe('cache', () => {
expect(response3.headers).not.toHaveProperty('rsshub-cache-status');
const parsed3 = await parser.parseString(await response3.text());
await wait(3 * 1000 + 100);
await wait(2 * 1000 + 100);
const response4 = await app.request('/test/cache');
const parsed4 = await parser.parseString(await response4.text());
@@ -51,7 +51,7 @@ describe('cache', () => {
await wait(1 * 1000 + 100);
const response5 = await app.request('/test/refreshCache');
const parsed5 = await parser.parseString(await response5.text());
await wait(2 * 1000 + 100);
await wait(1 * 1000 + 100);
const response6 = await app.request('/test/refreshCache');
const parsed6 = await parser.parseString(await response6.text());
@@ -86,7 +86,7 @@ describe('cache', () => {
expect(response3.headers).not.toHaveProperty('rsshub-cache-status');
const parsed3 = await parser.parseString(await response3.text());
await wait(3 * 1000 + 100);
await wait(2 * 1000 + 100);
const response4 = await app.request('/test/cache');
const parsed4 = await parser.parseString(await response4.text());
@@ -99,7 +99,7 @@ describe('cache', () => {
await wait(1 * 1000 + 100);
const response5 = await app.request('/test/refreshCache');
const parsed5 = await parser.parseString(await response5.text());
await wait(2 * 1000 + 100);
await wait(1 * 1000 + 100);
const response6 = await app.request('/test/refreshCache');
const parsed6 = await parser.parseString(await response6.text());
lib/middleware/parameter.test.ts
+7 -23
View File
@@ -1,8 +1,12 @@
import { describe, expect, it, vi } from 'vitest';
import app from '@/app';
import Parser from 'rss-parser';
import { config } from '@/config';
import nock from 'nock';
process.env.OPENAI_API_KEY = 'sk-1234567890';
process.env.OPENAI_API_ENDPOINT = 'https://api.openai.mock/v1';
vi.mock('@/utils/request-rewriter', () => ({ default: null }));
const { config } = await import('@/config');
const { default: app } = await import('@/app');
const parser = new Parser();
@@ -424,26 +428,6 @@ describe('multi parameter', () => {
describe('openai', () => {
it(`chatgpt`, async () => {
vi.resetModules();
process.env.OPENAI_API_KEY = 'sk-1234567890';
const app = (await import('@/app')).default;
const { config } = await import('@/config');
nock(config.openai.endpoint)
.post('/chat/completions')
.reply(() => [
200,
{
choices: [
{
message: {
content: 'Summary of the article.',
},
},
],
},
]);
const responseWithGpt = await app.request('/test/gpt?chatgpt=true');
const responseNormal = await app.request('/test/gpt');
lib/middleware/parameter.ts
+6 -7
View File
@@ -1,7 +1,7 @@
import * as entities from 'entities';
import { load, type CheerioAPI, type Element } from 'cheerio';
import { simplecc } from 'simplecc-wasm';
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { config } from '@/config';
import { RE2JS } from 're2js';
import markdownit from 'markdown-it';
@@ -34,8 +34,9 @@ const resolveRelativeLink = ($: CheerioAPI, elem: Element, attr: string, baseUrl
const summarizeArticle = async (articleText: string) => {
const apiUrl = `${config.openai.endpoint}/chat/completions`;
const response = await got.post(apiUrl, {
json: {
const response = await ofetch(apiUrl, {
method: 'POST',
body: {
model: config.openai.model,
max_tokens: config.openai.maxTokens,
messages: [
@@ -49,8 +50,7 @@ const summarizeArticle = async (articleText: string) => {
},
});
// @ts-expect-error custom field
return response.data.choices[0].message.content;
return response.choices[0].message.content;
};
const getAuthorString = (item) => {
@@ -305,8 +305,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
if (link) {
// if parser failed, return default description and not report error
try {
// @ts-expect-error custom field
const { data: res } = await got(link);
const res = await ofetch(link);
const $ = load(res);
const result = await Parser.parse(link, {
html: $.html(),
lib/routes/bloomberg/utils.ts
+2 -2
View File
@@ -99,7 +99,7 @@ const parseArticle = (item) =>
res = await got(apiUrl, { headers });
} catch (error) {
// fallback
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError' || error.name === 'FetchError')) {
try {
res = await got(item.link, { headers });
} catch {
@@ -214,7 +214,7 @@ const parseReactRendererPage = async (res, api, item) => {
return await parseStoryJson(res.data, item);
} catch (error) {
// fallback
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError' || error.name === 'FetchError')) {
return {
title: item.title,
link: item.link,
lib/routes/gov/chinatax/latest.ts
+1 -1
View File
@@ -56,7 +56,7 @@ async function handler() {
item.description = content('#fontzoom').html();
return item;
} catch (error) {
if (error.name === 'HTTPError') {
if (error.name === 'HTTPError' || error.name === 'FetchError') {
item.description = error.message;
return item;
}
lib/routes/line/utils.ts
+1 -1
View File
@@ -29,7 +29,7 @@ const parseItems = (list, tryGet) =>
});
data = response.data;
} catch (error) {
if (error instanceof got.HTTPError && error.response.statusCode === 404) {
if ((error.name === 'HTTPError' || error.name === 'FetchError') && error.response.statusCode === 404) {
logger.error(`Error parsing article ${item.link}: ${error.message}`);
return item;
}
lib/routes/phoronix/index.ts
+1 -1
View File
@@ -72,7 +72,7 @@ const webFetch = (url) =>
try {
return webFetchCb(await got(url));
} catch (error) {
if (error.name === 'HTTPError' && error.response.statusCode === 404) {
if ((error.name === 'HTTPError' || error.name === 'FetchError') && error.response.statusCode === 404) {
return '404';
}
throw error;
lib/routes/pixiv/token.ts
+14 -21
View File
@@ -15,34 +15,27 @@ const refreshToken = (tryGet) =>
tryGet(
'pixiv:accessToken',
() =>
got
.post('https://oauth.secure.pixiv.net/auth/token', {
form: {
...authorizationInfo,
get_secure_url: 1,
grant_type: 'refresh_token',
refresh_token: config.pixiv.refreshToken,
},
headers: {
...maskHeader,
},
responseType: 'json',
resolveBodyOnly: true,
})
.catch((error) => {
logger.error('Pixiv refresh token failed.');
logger.error(error);
}),
got.post('https://oauth.secure.pixiv.net/auth/token', {
form: {
...authorizationInfo,
get_secure_url: 1,
grant_type: 'refresh_token',
refresh_token: config.pixiv.refreshToken,
},
headers: {
...maskHeader,
},
}),
3600,
false
);
async function getToken(tryGet) {
const result = await refreshToken(tryGet);
const { data } = await refreshToken(tryGet);
// let expireTime;
if (result && result.access_token) {
if (data && data.access_token) {
logger.debug('Pixiv refresh token success.');
token = result.access_token;
token = data.access_token;
// expireTime = result.expires_in;
}
// } else {
lib/routes/wechat/feeddd.ts
+1 -1
View File
@@ -13,7 +13,7 @@ export default async (ctx) => {
try {
response = await got(apiUrl);
} catch (error) {
if (error.name === 'HTTPError' && error.response.statusCode === 404) {
if ((error.name === 'HTTPError' || error.name === 'FetchError') && error.response.statusCode === 404) {
throw new Error('该公众号不存在，有关如何获取公众号 id，详见 https://docs.rsshub.app/routes/new-media#wei-xin-gong-zhong-hao-feeddd-lai-yuan');
}
throw error;
lib/routes/wfu/news.ts
+1 -1
View File
@@ -32,7 +32,7 @@ async function loadContent(link) {
response = await got.get(link);
} catch (error) {
// 如果网络问题 直接出错
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError' || error.name === 'FetchError')) {
description = 'Page 404 Please Check!';
}
return { description };
lib/setup.test.ts
+77
View File
@@ -0,0 +1,77 @@
import { afterAll, afterEach } from 'vitest';
import { setupServer } from 'msw/node';
import { http, HttpResponse } from 'msw';
const server = setupServer(
http.post(`https://api.openai.mock/v1/chat/completions`, () =>
HttpResponse.json({
choices: [
{
message: {
content: 'Summary of the article.',
},
},
],
})
),
http.get(`http://rsshub.test/config`, () =>
HttpResponse.json({
UA: 'test',
})
),
http.get(`http://rsshub.test/buildData`, () =>
HttpResponse.text(`<div class="content">
<ul>
<li>
<a href="/1">1</a>
<div class="description">RSSHub1</div>
</li>
<li>
<a href="/2">2</a>
<div class="description">RSSHub2</div>
</li>
</ul>
</div>`)
),
http.get(`https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle`, () =>
HttpResponse.text(
'\n' +
'<meta name="description" content="summary" />\n' +
'<meta name="author" content="author" />\n' +
'<meta property="og:title" content="title" />\n' +
'<meta property="twitter:card" content="summary" />\n' +
'<div class="rich_media_content" id="js_content" style="visibility: hidden;">description</div>\n' +
'<div class="profile_inner"><strong class="profile_nickname">mpName</strong></div>\n' +
'<script type="text/javascript" nonce="000000000">\n' +
'var appmsg_type = "9";\n' +
`var ct = "${1_636_626_300}";\n` +
'</script>'
)
),
http.get(`http://rsshub.test/headers`, ({ request }) =>
HttpResponse.json({
...Object.fromEntries(request.headers.entries()),
})
),
http.post(`http://rsshub.test/form-post`, async ({ request }) => {
const formData = await request.formData();
return HttpResponse.json({
test: formData.get('test'),
});
}),
http.post(`http://rsshub.test/json-post`, async ({ request }) => {
const jsonData = (await request.json()) as {
test: string;
};
return HttpResponse.json({
test: jsonData?.test,
});
}),
http.get(`http://rsshub.test/rss`, () => HttpResponse.text('<rss version="2.0"><channel><item></item></channel></rss>'))
);
server.listen();
afterAll(() => server.close());
afterEach(() => server.resetHandlers());
export default server;
lib/utils/common-config.test.ts
-18
View File
@@ -1,6 +1,5 @@
import { describe, expect, it } from 'vitest';
import configUtils, { transElemText, replaceParams, getProp } from '@/utils/common-config';
import nock from 'nock';
describe('index', () => {
it('transElemText', () => {
@@ -40,23 +39,6 @@ describe('index', () => {
});
it('buildData', async () => {
nock('http://rsshub.test')
.get('/buildData')
.reply(() => [
200,
`<div class="content">
<ul>
<li>
<a href="/1">1</a>
<div class="description">RSSHub1</div>
</li>
<li>
<a href="/2">2</a>
<div class="description">RSSHub2</div>
</li>
</ul>
</div>`,
]);
const data = await configUtils({
link: 'http://rsshub.test/buildData',
url: 'http://rsshub.test/buildData',
lib/utils/common-config.ts
+6 -6
View File
@@ -1,5 +1,5 @@
import cheerio from 'cheerio';
import got from '@/utils/got';
import { load } from 'cheerio';
import ofetch from '@/utils/ofetch';
import iconv from 'iconv-lite';
function transElemText($, prop) {
@@ -37,8 +37,8 @@ function getProp(data, prop, $) {
}
async function buildData(data) {
const response = await got.get(data.url);
const contentType = response.headers['content-type'] || '';
const response = await ofetch.raw(data.url);
const contentType = response.headers.get('content-type') || '';
// 若没有指定编码，则默认utf-8
let charset = 'utf-8';
for (const attr of contentType.split(';')) {
@@ -47,8 +47,8 @@ async function buildData(data) {
}
}
// @ts-expect-error custom property
const responseData = charset === 'utf-8' ? response.data : iconv.decode((await got.get({ url: data.url, responseType: 'buffer' })).data, charset);
const $ = cheerio.load(responseData);
const responseData = charset === 'utf-8' ? response._data : iconv.decode(await ofetch(data.url, { responseType: 'buffer' }), charset);
const $ = load(responseData);
const $item = $(data.item.item);
// 这里应该是可以通过参数注入一些代码的，不过应该无伤大雅
return {
lib/utils/got-deprecated.ts
+79
View File
@@ -0,0 +1,79 @@
import logger from '@/utils/logger';
import { config } from '@/config';
import got, { CancelableRequest, Response as GotResponse, OptionsInit, Options, Got } from 'got';
type Response<T> = GotResponse<string> & {
data: T;
status: number;
};
type GotRequestFunction = {
(url: string | URL, options?: Options): CancelableRequest<Response<Record<string, any>>>;
<T>(url: string | URL, options?: Options): CancelableRequest<Response<T>>;
(options: Options): CancelableRequest<Response<Record<string, any>>>;
<T>(options: Options): CancelableRequest<Response<T>>;
};
// @ts-expect-error got instance with custom response type
const custom: {
all?: <T>(list: Array<Promise<T>>) => Promise<Array<T>>;
get: GotRequestFunction;
post: GotRequestFunction;
put: GotRequestFunction;
patch: GotRequestFunction;
head: GotRequestFunction;
delete: GotRequestFunction;
} & GotRequestFunction &
Got = got.extend({
retry: {
limit: config.requestRetry,
statusCodes: [400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 421, 422, 423, 424, 426, 428, 429, 431, 451, 500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 521, 522, 524],
},
hooks: {
beforeRetry: [
(err, count) => {
logger.error(`Request ${err.options.url} fail, retry attempt #${count}: ${err}`);
},
],
beforeRedirect: [
(options, response) => {
logger.http(`Redirecting to ${options.url} for ${response.requestUrl}`);
},
],
afterResponse: [
// @ts-expect-error custom response type
(response: Response<Record<string, any>>) => {
try {
response.data = typeof response.body === 'string' ? JSON.parse(response.body) : response.body;
} catch {
// @ts-expect-error for compatibility
response.data = response.body;
}
response.status = response.statusCode;
return response;
},
],
init: [
(
options: OptionsInit & {
data?: string;
}
) => {
// compatible with axios api
if (options && options.data) {
options.body = options.body || options.data;
}
},
],
},
headers: {
'user-agent': config.ua,
},
timeout: {
request: config.requestTimeout,
},
});
custom.all = (list) => Promise.all(list);
export default custom;
export type { Response, Options } from 'got';
lib/utils/got.test.ts
+27 -64
View File
@@ -1,88 +1,51 @@
import { describe, expect, it, afterEach, vi } from 'vitest';
import nock from 'nock';
afterEach(() => {
vi.resetModules();
});
import { describe, expect, it, vi } from 'vitest';
import { http, HttpResponse } from 'msw';
import got from '@/utils/got';
import { config } from '@/config';
describe('got', () => {
it('headers', async () => {
const { default: got } = await import('@/utils/got');
const { config } = await import('@/config');
nock('http://rsshub.test')
.get('/test')
.reply(function () {
expect(this.req.headers['user-agent']).toBe(config.ua);
return [200, ''];
});
await got.get('http://rsshub.test/test');
const { data } = await got('http://rsshub.test/headers');
expect(data['user-agent']).toBe(config.ua);
});
it('retry', async () => {
const { default: got } = await import('@/utils/got');
const { config } = await import('@/config');
const requestRun = vi.fn();
nock('http://rsshub.test')
.get('/testRerty')
.times(config.requestRetry + 1)
.reply(() => {
const { default: server } = await import('@/setup.test');
server.use(
http.get(`http://rsshub.test/retry-test`, () => {
requestRun();
return [503, '0'];
});
return HttpResponse.error();
})
);
try {
await got.get('http://rsshub.test/testRerty');
await got.get('http://rsshub.test/retry-test');
} catch (error: any) {
expect(error.name).toBe('HTTPError');
expect(error.name).toBe('FetchError');
}
// retries
expect(requestRun).toHaveBeenCalledTimes(config.requestRetry + 1);
});
it('axios', async () => {
const { default: got } = await import('@/utils/got');
nock('http://rsshub.test')
.post('/post')
.reply(() => [200, '{"code": 0}']);
const response1 = await got.post('http://rsshub.test/post', {
it('form-post', async () => {
const response = await got.post('http://rsshub.test/form-post', {
form: {
test: 1,
test: 'rsshub',
},
});
expect(response1.statusCode).toBe(200);
// @ts-expect-error custom property
expect(response1.status).toBe(200);
expect(response1.body).toBe('{"code": 0}');
// @ts-expect-error custom property
expect(response1.data.code).toBe(0);
expect(response.body).toBe('{"test":"rsshub"}');
expect(response.data.test).toBe('rsshub');
});
it('timeout', async () => {
process.env.REQUEST_TIMEOUT = '500';
const { default: got } = await import('@/utils/got');
nock('http://rsshub.test')
.get('/timeout')
.delay(600)
.reply(() => [200, '{"code": 0}']);
const logger = (await import('@/utils/logger')).default;
// @ts-expect-error unused
const loggerSpy = vi.spyOn(logger, 'error').mockReturnValue({});
try {
await got.get('http://rsshub.test/timeout');
throw new Error('Timeout Invalid');
} catch (error: any) {
expect(error.name).toBe('RequestError');
}
expect(loggerSpy).toHaveBeenCalledWith(expect.stringContaining('http://rsshub.test/timeout'));
loggerSpy.mockRestore();
delete process.env.REQUEST_TIMEOUT;
it('json-post', async () => {
const response = await got.post('http://rsshub.test/json-post', {
json: {
test: 'rsshub',
},
});
expect(response.body).toBe('{"test":"rsshub"}');
expect(response.data.test).toBe('rsshub');
});
});
lib/utils/got.ts
+63 -76
View File
@@ -1,79 +1,66 @@
import logger from '@/utils/logger';
import { config } from '@/config';
import got, { CancelableRequest, Response as GotResponse, OptionsInit, Options, Got } from 'got';
import { destr } from 'destr';
import ofetch from '@/utils/ofetch';
type Response<T> = GotResponse<string> & {
data: T;
status: number;
};
type GotRequestFunction = {
(url: string | URL, options?: Options): CancelableRequest<Response<Record<string, any>>>;
<T>(url: string | URL, options?: Options): CancelableRequest<Response<T>>;
(options: Options): CancelableRequest<Response<Record<string, any>>>;
<T>(options: Options): CancelableRequest<Response<T>>;
};
// @ts-expect-error got instance with custom response type
const custom: {
all?: <T>(list: Array<Promise<T>>) => Promise<Array<T>>;
get: GotRequestFunction;
post: GotRequestFunction;
put: GotRequestFunction;
patch: GotRequestFunction;
head: GotRequestFunction;
delete: GotRequestFunction;
} & GotRequestFunction &
Got = got.extend({
retry: {
limit: config.requestRetry,
statusCodes: [400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 421, 422, 423, 424, 426, 428, 429, 431, 451, 500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 521, 522, 524],
},
hooks: {
beforeRetry: [
(err, count) => {
logger.error(`Request ${err.options.url} fail, retry attempt #${count}: ${err}`);
},
],
beforeRedirect: [
(options, response) => {
logger.http(`Redirecting to ${options.url} for ${response.requestUrl}`);
},
],
afterResponse: [
// @ts-expect-error custom response type
(response: Response<Record<string, any>>) => {
try {
response.data = typeof response.body === 'string' ? JSON.parse(response.body) : response.body;
} catch {
// @ts-expect-error for compatibility
response.data = response.body;
}
response.status = response.statusCode;
return response;
},
],
init: [
(
options: OptionsInit & {
data?: string;
}
) => {
// compatible with axios api
if (options && options.data) {
options.body = options.body || options.data;
}
},
],
},
headers: {
'user-agent': config.ua,
},
timeout: {
request: config.requestTimeout,
},
const gotofetch = ofetch.create({
parseResponse: (responseText) => ({
data: destr(responseText),
body: responseText,
}),
});
custom.all = (list) => Promise.all(list);
export default custom;
export type { Response, Options } from 'got';
const getFakeGot = (defaultOptions?: any) => {
const fakeGot = (request, options?: any) => {
if (!(typeof request === 'string' || request instanceof Request) && request.url) {
options = {
...request,
...options,
};
request = request.url;
}
if (options?.hooks?.beforeRequest) {
for (const hook of options.hooks.beforeRequest) {
hook(options);
}
delete options.hooks;
}
options = {
...defaultOptions,
...options,
};
if (options?.json && !options.body) {
options.body = options.json;
delete options.json;
}
if (options?.form && !options.body) {
const body = new FormData();
for (const key in options.form) {
body.append(key, options.form[key]);
}
options.body = body;
if (!options.headers) {
options.headers = {};
}
delete options.form;
}
if (options?.searchParams) {
request += '?' + new URLSearchParams(options.searchParams).toString();
delete options.searchParams;
}
return gotofetch(request, options);
};
fakeGot.get = (request, options) => fakeGot(request, { ...options, method: 'GET' });
fakeGot.post = (request, options) => fakeGot(request, { ...options, method: 'POST' });
fakeGot.put = (request, options) => fakeGot(request, { ...options, method: 'PUT' });
fakeGot.patch = (request, options) => fakeGot(request, { ...options, method: 'PATCH' });
fakeGot.head = (request, options) => fakeGot(request, { ...options, method: 'HEAD' });
fakeGot.delete = (request, options) => fakeGot(request, { ...options, method: 'DELETE' });
fakeGot.extend = (options) => getFakeGot(options);
return fakeGot;
};
export default getFakeGot();
lib/utils/ofetch.ts
+17
View File
@@ -0,0 +1,17 @@
import { createFetch } from 'ofetch';
import { config } from '@/config';
import logger from '@/utils/logger';
const rofetch = createFetch().create({
retry: config.requestRetry,
retryDelay: 1000,
timeout: config.requestTimeout,
onRequestError({ request, error }) {
logger.error(`Request ${request} fail: ${error}`);
},
headers: {
'user-agent': config.ua,
},
});
export default rofetch;
lib/utils/proxy/index.ts
+15 -1
View File
@@ -2,6 +2,7 @@ import { config } from '@/config';
import { PacProxyAgent } from 'pac-proxy-agent';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { ProxyAgent } from 'undici';
const proxyIsPAC = config.pacUri || config.pacScript;
@@ -24,11 +25,23 @@ if (proxyIsPAC) {
}
let agent: PacProxyAgent<string> | HttpsProxyAgent<string> | SocksProxyAgent | null = null;
let dispatcher: ProxyAgent | null = null;
if (proxyIsPAC) {
agent = new PacProxyAgent(`pac+${proxyUri}`);
} else if (proxyUri) {
if (proxyUri.startsWith('http')) {
agent = new HttpsProxyAgent(proxyUri);
agent = new HttpsProxyAgent(proxyUri, {
headers: {
'proxy-authorization': config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined,
},
});
dispatcher = new ProxyAgent({
uri: proxyUri,
token: config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined,
requestTls: {
rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0',
},
});
} else if (proxyUri.startsWith('socks')) {
agent = new SocksProxyAgent(proxyUri);
}
@@ -36,6 +49,7 @@ if (proxyIsPAC) {
export default {
agent,
dispatcher,
proxyUri,
proxyObj,
proxyUrlHandler,
lib/utils/rand-user-agent.test.ts
+5 -20
View File
@@ -1,7 +1,6 @@
import { describe, expect, it } from 'vitest';
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { config } from '@/config';
import nock from 'nock';
import randUserAgent from '@/utils/rand-user-agent';
const mobileUa = randUserAgent({ browser: 'mobile safari', os: 'ios', device: 'mobile' });
@@ -23,30 +22,16 @@ describe('rand-user-agent', () => {
});
it('should has default random ua', async () => {
nock('https://rsshub.test')
.get('/test')
.reply(function () {
expect(this.req.headers['user-agent']).toBe(config.ua);
expect(this.req.headers['user-agent']).not.toBe(mobileUa);
expect(this.req.headers['user-agent']).not.toBe('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36');
return [200, ''];
});
await got('https://rsshub.test/test');
const response = await ofetch('http://rsshub.test/headers');
expect(response['user-agent']).toBe(config.ua);
});
it('should match ua configurated', async () => {
nock('https://rsshub.test')
.get('/test')
.reply(function () {
return [200, { ua: this.req.headers['user-agent'] }];
});
const resonse = await got('https://rsshub.test/test', {
const response = await ofetch('http://rsshub.test/headers', {
headers: {
'user-agent': mobileUa,
},
});
// @ts-expect-error custom field
expect(resonse.data.ua).toBe(mobileUa);
expect(response['user-agent']).toBe(mobileUa);
});
});
lib/utils/request-rewriter.test.ts
+137
View File
@@ -0,0 +1,137 @@
import { describe, expect, it, vi } from 'vitest';
import undici from 'undici';
import got from 'got';
import http from 'node:http';
process.env.PROXY_URI = 'http://rsshub.proxy:2333/';
process.env.PROXY_AUTH = 'rsshubtest';
process.env.PROXY_URL_REGEX = 'headers';
await import('@/utils/request-rewriter');
const { config } = await import('@/config');
const { default: ofetch } = await import('@/utils/ofetch');
describe('request-rewriter', () => {
it('fetch', async () => {
const fetchSpy = vi.spyOn(undici, 'fetch');
try {
await (await fetch('http://rsshub.test/headers')).json();
} catch {
// ignore
}
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('accept')).toBe('*/*');
expect(headers.get('referer')).toBe('http://rsshub.test');
// proxy
const options = fetchSpy.mock.lastCall?.[1];
const agentKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy agent options');
const agentUri = agentKey ? options?.dispatcher?.[agentKey].uri : null;
expect(agentUri).toBe(process.env.PROXY_URI);
// proxy auth
const headersKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy headers');
const agentHeaders = headersKey ? options?.dispatcher?.[headersKey] : null;
expect(agentHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
// url regex not match
{
try {
await (await fetch('http://rsshub.test/rss')).json();
} catch {
// ignore
}
const options = fetchSpy.mock.lastCall?.[1];
expect(options?.dispatcher).toBeUndefined();
}
});
it('ofetch', async () => {
const fetchSpy = vi.spyOn(undici, 'fetch');
try {
await ofetch('http://rsshub.test/headers', {
retry: 0,
});
} catch {
// ignore
}
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('accept')).toBe('*/*');
expect(headers.get('referer')).toBe('http://rsshub.test');
// proxy
const options = fetchSpy.mock.lastCall?.[1];
const agentKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy agent options');
const agentUri = agentKey ? options?.dispatcher?.[agentKey].uri : null;
expect(agentUri).toBe(process.env.PROXY_URI);
// proxy auth
const headersKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy headers');
const agentHeaders = headersKey ? options?.dispatcher?.[headersKey] : null;
expect(agentHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
// url regex not match
{
try {
await ofetch('http://rsshub.test/rss', {
retry: 0,
});
} catch {
// ignore
}
const options = fetchSpy.mock.lastCall?.[1];
expect(options?.dispatcher).toBeUndefined();
}
});
it('http', async () => {
const httpSpy = vi.spyOn(http, 'request');
try {
await got.get('http://rsshub.test/headers', {
headers: {
'user-agent': undefined,
accept: undefined,
},
});
} catch {
// ignore
}
// headers
const options = httpSpy.mock.lastCall?.[1];
const headers = options?.headers;
expect(headers?.['user-agent']).toBe(config.ua);
expect(headers?.accept).toBe('*/*');
expect(headers?.referer).toBe('http://rsshub.test');
// proxy
const agentUri = options?.agent?.proxy?.href;
expect(agentUri).toBe(process.env.PROXY_URI);
expect(options?.agent?.proxyHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
// url regex not match
{
try {
await got.get('http://rsshub.test/rss', {
headers: {
'user-agent': undefined,
accept: undefined,
},
});
} catch {
// ignore
}
const options = httpSpy.mock.lastCall?.[1];
expect(options?.agent).toBeUndefined();
}
});
});
lib/utils/request-rewriter/fetch.ts
+50
View File
@@ -0,0 +1,50 @@
import logger from '@/utils/logger';
import { config } from '@/config';
import undici, { Request, RequestInfo, RequestInit } from 'undici';
import proxy from '@/utils/proxy';
const wrappedFetch: typeof undici.fetch = (input: RequestInfo, init?: RequestInit) => {
const request = new Request(input, init);
const options: RequestInit = {};
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
// ua
if (!request.headers.get('user-agent')) {
request.headers.set('user-agent', config.ua);
}
// accept
if (!request.headers.get('accept')) {
request.headers.set('accept', '*/*');
}
// referer
if (!request.headers.get('referer')) {
try {
const urlHandler = new URL(request.url);
request.headers.set('referer', urlHandler.origin);
} catch {
// ignore
}
}
// proxy
if (!options.dispatcher && proxy.dispatcher) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
let urlHandler;
try {
urlHandler = new URL(request.url);
} catch {
// ignore
}
if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
options.dispatcher = proxy.dispatcher;
}
}
return undici.fetch(request, options);
};
export default wrappedFetch;
lib/utils/request-rewriter/get.ts
+70
View File
@@ -0,0 +1,70 @@
import http from 'node:http';
import https from 'node:https';
import logger from '@/utils/logger';
import { config } from '@/config';
import proxy from '@/utils/proxy';
type Get = typeof http.get | typeof https.get | typeof http.request | typeof https.request;
const getWrappedGet: <T extends Get>(origin: T) => T = (origin) =>
function (this: any, ...args: Parameters<typeof origin>) {
let url: URL | null;
let options: http.RequestOptions = {};
let callback: ((res: http.IncomingMessage) => void) | undefined;
if (typeof args[0] === 'string' || args[0] instanceof URL) {
url = new URL(args[0]);
if (typeof args[1] === 'object') {
options = args[1];
callback = args[2];
} else if (typeof args[1] === 'function') {
options = {};
callback = args[1];
}
} else {
options = args[0];
try {
url = new URL(options.href || `${options.protocol || 'http:'}//${options.hostname || options.host}${options.path}${options.search || (options.query ? `?${options.query}` : '')}`);
} catch {
url = null;
}
if (typeof args[1] === 'function') {
callback = args[1];
}
}
if (!url) {
return Reflect.apply(origin, this, args) as ReturnType<typeof origin>;
}
logger.debug(`Outgoing request: ${options.method || 'GET'} ${url}`);
options.headers = options.headers || {};
const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase()));
// ua
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
// Accept
if (!headersLowerCaseKeys.has('accept')) {
options.headers.accept = '*/*';
}
// referer
if (!headersLowerCaseKeys.has('referer')) {
options.headers.referer = url.origin;
}
// proxy
if (!options.agent && proxy.agent) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
if (proxyRegex.test(url.toString()) && url.protocol.startsWith('http') && url.host !== proxy.proxyUrlHandler?.host) {
options.agent = proxy.agent;
}
}
return Reflect.apply(origin, this, [url, options, callback]) as ReturnType<typeof origin>;
};
export default getWrappedGet;
lib/utils/request-rewriter/index.ts
+19
View File
@@ -0,0 +1,19 @@
import { Headers, FormData, Request, Response } from 'undici';
import http from 'node:http';
import https from 'node:https';
import fetch from '@/utils/request-rewriter/fetch';
import getWrappedGet from '@/utils/request-rewriter/get';
Object.defineProperties(globalThis, {
fetch: { value: fetch },
Headers: { value: Headers },
FormData: { value: FormData },
Request: { value: Request },
Response: { value: Response },
});
http.get = getWrappedGet(http.get);
http.request = getWrappedGet(http.request);
https.get = getWrappedGet(https.get);
https.request = getWrappedGet(https.request);
lib/utils/request-wrapper.test.ts
-365
View File
@@ -1,365 +0,0 @@
import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest';
import got from '@/utils/got';
import parser from '@/utils/rss-parser';
import nock from 'nock';
import http from 'node:http';
let check: (
request: Request & {
agent: any;
path: string;
}
) => void = () => {};
const simpleResponse = '<rss version="2.0"><channel><item></item></channel></rss>';
beforeEach(() => {
delete process.env.PAC_URI;
delete process.env.PAC_SCRIPT;
});
afterEach(() => {
delete process.env.PROXY_URI;
delete process.env.PROXY_PROTOCOL;
delete process.env.PROXY_HOST;
delete process.env.PROXY_PORT;
delete process.env.PROXY_AUTH;
delete process.env.PROXY_URL_REGEX;
nock.restore();
nock.activate();
check = () => {};
const httpWrap = (func) => {
const origin = func;
return function (url, request) {
if (typeof url === 'object') {
if (url instanceof URL) {
check(request);
} else {
check(url);
}
} else {
check(request);
}
// @ts-expect-error any
// eslint-disable-next-line prefer-rest-params
return Reflect.apply(origin, this, arguments);
};
};
http.get = httpWrap(http.get);
http.request = httpWrap(http.request);
vi.resetModules();
});
describe('got', () => {
it('headers', async () => {
await import('@/utils/request-wrapper');
nock(/rsshub\.test/)
.get(/.*/)
.times(3)
.reply(function () {
expect(this.req.headers.referer).toBe('http://api.rsshub.test');
expect(this.req.headers.host).toBe('api.rsshub.test');
return [200, simpleResponse];
});
await got.get('http://api.rsshub.test/test');
await got.get('http://api.rsshub.test');
await parser.parseURL('http://api.rsshub.test/test');
});
it('proxy-uri socks', async () => {
process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('SocksProxyAgent');
expect(request.agent.proxy.host).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe(2333);
expect(request.agent.proxy.type).toBe(5);
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy-uri http', async () => {
process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('HttpsProxyAgent');
expect(request.agent.proxy.protocol).toBe('http:');
expect(request.agent.proxy.username).toBe('user');
expect(request.agent.proxy.password).toBe('pass');
expect(request.agent.proxy.host).toBe('rsshub.proxy:2333');
expect(request.agent.proxy.hostname).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy-uri https', async () => {
process.env.PROXY_URI = 'https://rsshub.proxy:2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('HttpsProxyAgent');
expect(request.agent.proxy.protocol).toBe('https:');
expect(request.agent.proxy.host).toBe('rsshub.proxy:2333');
expect(request.agent.proxy.hostname).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy socks', async () => {
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('SocksProxyAgent');
expect(request.agent.proxy.host).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe(2333);
expect(request.agent.proxy.type).toBe(5);
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy http', async () => {
process.env.PROXY_PROTOCOL = 'http';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('HttpsProxyAgent');
expect(request.agent.proxy.protocol).toBe('http:');
expect(request.agent.proxy.host).toBe('rsshub.proxy:2333');
expect(request.agent.proxy.hostname).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy https', async () => {
process.env.PROXY_PROTOCOL = 'https';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('HttpsProxyAgent');
expect(request.agent.proxy.protocol).toBe('https:');
expect(request.agent.proxy.host).toBe('rsshub.proxy:2333');
expect(request.agent.proxy.hostname).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('pac-uri http', async () => {
process.env.PAC_URI = 'http://rsshub.proxy:2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('PacProxyAgent');
expect(request.agent.uri.protocol).toBe('http:');
expect(request.agent.uri.host).toBe('rsshub.proxy:2333');
expect(request.agent.uri.hostname).toBe('rsshub.proxy');
expect(request.agent.uri.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('pac-uri https', async () => {
process.env.PAC_URI = 'https://rsshub.proxy:2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('PacProxyAgent');
expect(request.agent.uri.protocol).toBe('https:');
expect(request.agent.uri.host).toBe('rsshub.proxy:2333');
expect(request.agent.uri.hostname).toBe('rsshub.proxy');
expect(request.agent.uri.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('pac-uri ftp', async () => {
process.env.PAC_URI = 'ftp://rsshub.proxy:2333';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('PacProxyAgent');
expect(request.agent.uri.protocol).toBe('ftp:');
expect(request.agent.uri.host).toBe('rsshub.proxy:2333');
expect(request.agent.uri.hostname).toBe('rsshub.proxy');
expect(request.agent.uri.port).toBe('2333');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('pac-uri file', async () => {
process.env.PAC_URI = 'file:///D:/rsshub/proxy';
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('PacProxyAgent');
expect(request.agent.uri.protocol).toBe('file:');
expect(request.agent.uri.pathname).toBe('/D:/rsshub/proxy');
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('pac-script data', async () => {
process.env.PAC_SCRIPT = "function FindProxyForURL(url,host){return 'DIRECT';}";
await import('@/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('PacProxyAgent');
expect(request.agent.uri.protocol).toBe('data:');
expect(request.agent.uri.pathname).toBe("text/javascript;charset=utf-8,function%20FindProxyForURL(url%2Chost)%7Breturn%20'DIRECT'%3B%7D");
};
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('auth', async () => {
process.env.PROXY_AUTH = 'testtest';
process.env.PROXY_PROTOCOL = 'http'; // only http(s) proxies extract auth from Headers
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
await import('@/utils/request-wrapper');
nock(/rsshub\.test/)
.get('/auth')
.times(2)
.reply(function () {
expect(this.req.headers['proxy-authorization']).toBe('Basic testtest');
return [200, simpleResponse];
});
await got.get('http://rsshub.test/auth');
await parser.parseURL('http://rsshub.test/auth');
});
it('url_regex', async () => {
process.env.PROXY_URL_REGEX = 'url_regex';
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
await import('@/utils/request-wrapper');
check = (request) => {
if (request.path === '/url_regex') {
expect(request.agent.constructor.name).toBe('SocksProxyAgent');
expect(request.agent.proxy.host).toBe('rsshub.proxy');
expect(request.agent.proxy.port).toBe(2333);
} else if (request.path === '/proxy') {
expect(request.agent).toBe(undefined);
}
};
nock(/rsshub\.test/)
.get('/url_regex')
.times(2)
.reply(() => [200, simpleResponse]);
nock(/rsshub\.test/)
.get('/proxy')
.times(2)
.reply(() => [200, simpleResponse]);
await got.get('http://rsshub.test/url_regex');
await parser.parseURL('http://rsshub.test/url_regex');
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
});
lib/utils/request-wrapper.ts
-105
View File
@@ -1,105 +0,0 @@
import { config } from '@/config';
import logger from '@/utils/logger';
import http, { type RequestOptions } from 'node:http';
import https from 'node:https';
import proxy from '@/utils/proxy';
let proxyWrapper: (
url: string,
options: RequestOptions & {
headers: Record<string, string>;
}
) => boolean = () => false;
if (proxy.agent) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
const protocolMatch = (protocolLike?: string | null) => protocolLike?.toLowerCase().startsWith('http');
proxyWrapper = (url, options) => {
let urlHandler;
try {
urlHandler = new URL(url);
} catch {
// ignore
}
if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxy.proxyUrlHandler?.host)) {
options.agent = proxy.agent || false;
if (proxy.proxyObj.auth) {
options.headers['Proxy-Authorization'] = `Basic ${proxy.proxyObj.auth}`;
}
return true;
}
return false;
};
}
const requestWrapper = (url: string, options: http.RequestOptions = {}) => {
options.headers = options.headers || {};
const optionsWithHeaders = options as http.RequestOptions & {
headers: Record<string, string>;
};
const headersLowerCaseKeys = new Set(Object.keys(optionsWithHeaders.headers).map((key) => key.toLowerCase()));
let prxied = false;
if (config.proxyStrategy === 'all') {
prxied = proxyWrapper(url, optionsWithHeaders);
} else if (config.proxyStrategy === 'on_retry' && (optionsWithHeaders as any).retryCount) {
// TODO
prxied = proxyWrapper(url, optionsWithHeaders);
}
if (prxied) {
logger.debug(`Proxy for ${url}`);
} else {
logger.debug(`Requesting ${url}`);
}
// ua
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
// Accept
if (!headersLowerCaseKeys.has('accept')) {
options.headers.Accept = '*/*';
}
let urlHandler;
try {
urlHandler = new URL(url);
} catch {
// ignore
}
if (
urlHandler && // referer
!headersLowerCaseKeys.has('referer')
) {
options.headers.referer = urlHandler.origin;
}
};
const httpWrap = (func: typeof http.request) => {
const origin = func;
const warpped: typeof http.request = function (...args) {
let url: string;
let options: http.RequestOptions;
if (args[0] instanceof URL || typeof args[0] === 'string') {
url = args[0].toString();
options = args[1] as http.RequestOptions;
} else {
options = args[0] as http.RequestOptions;
url = `${options.protocol}//${options.hostname || options.host}${options.path}`;
}
requestWrapper(url, options);
// @ts-expect-error apply
return origin.apply(this, args);
};
return warpped;
};
http.get = httpWrap(http.get);
https.get = httpWrap(https.get);
http.request = httpWrap(http.request);
https.request = httpWrap(https.request);
lib/utils/rss-parser.test.ts
+4 -12
View File
@@ -1,17 +1,9 @@
import { describe, expect, it } from 'vitest';
import parser from '@/utils/rss-parser';
import { config } from '@/config';
import nock from 'nock';
describe('got', () => {
it('headers', async () => {
nock('http://rsshub.test')
.get('/test')
.reply(function () {
expect(this.req.headers['user-agent']).toBe(config.ua);
return [200, '<rss version="2.0"><channel><item></item></channel></rss>'];
});
await parser.parseURL('http://rsshub.test/test');
describe('rss-parser', () => {
it('rss', async () => {
const result = await parser.parseURL('http://rsshub.test/rss');
expect(result).toBeTruthy();
});
});
lib/utils/wechat-mp.test.ts
-17
View File
@@ -1,6 +1,5 @@
import { describe, expect, it } from 'vitest';
import { load } from 'cheerio';
import nock from 'nock';
import { fixArticleContent, fetchArticle, finishArticleItem, normalizeUrl } from '@/utils/wechat-mp';
// date from the cache will be an ISO8601 string, so we need to use this function
@@ -91,22 +90,6 @@ describe('wechat-mp', () => {
it('fetchArticle_&_finishArticleItem', async () => {
const ct = 1_636_626_300;
const exampleMpArticlePage =
'\n' +
'<meta name="description" content="summary" />\n' +
'<meta name="author" content="author" />\n' +
'<meta property="og:title" content="title" />\n' +
'<meta property="twitter:card" content="summary" />\n' +
'<div class="rich_media_content" id="js_content" style="visibility: hidden;">description</div>\n' +
'<div class="profile_inner"><strong class="profile_nickname">mpName</strong></div>\n' +
'<script type="text/javascript" nonce="000000000">\n' +
'var appmsg_type = "9";\n' +
`var ct = "${ct}";\n` +
'</script>';
nock('https://mp.weixin.qq.com')
.get('/rsshub_test/wechatMp_fetchArticle')
.reply(() => [200, exampleMpArticlePage]);
const httpsUrl = 'https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle';
const httpUrl = httpsUrl.replace(/^https:\/\//, 'http://');
lib/utils/wechat-mp.ts
+6 -8
View File
@@ -25,7 +25,7 @@
* For more details of these functions, please refer to the jsDoc in the source code.
*/
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { load, type Cheerio, type Element } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import cache from '@/utils/cache';
@@ -196,9 +196,8 @@ const normalizeUrl = (url, bypassHostCheck = false) => {
const fetchArticle = (url, bypassHostCheck = false) => {
url = normalizeUrl(url, bypassHostCheck);
return cache.tryGet(url, async () => {
const response = await got(url);
// @ts-expect-error custom field
const $ = load(response.data);
const data = await ofetch(url);
const $ = load(data);
const title = ($('meta[property="og:title"]').attr('content') || '').replaceAll('\\r', '').replaceAll('\\n', ' ');
const author = $('meta[name=author]').attr('content');
@@ -209,9 +208,8 @@ const fetchArticle = (url, bypassHostCheck = false) => {
const originalUrl = detectOriginalArticleUrl($);
if (originalUrl) {
// try to fetch the description from the original article
const originalResponse = await got(normalizeUrl(originalUrl, bypassHostCheck));
// @ts-expect-error custom field
const original$ = load(originalResponse.data);
const data = await ofetch(normalizeUrl(originalUrl, bypassHostCheck));
const original$ = load(data);
description += fixArticleContent(original$('#js_content'));
}
@@ -230,7 +228,7 @@ const fetchArticle = (url, bypassHostCheck = false) => {
let mpName = $('.profile_nickname').first().text();
mpName = mpName && mpName.trim();
return { title, author, description, summary, pubDate, mpName, link: response.url };
return { title, author, description, summary, pubDate, mpName, link: url };
}) as Promise<{
title: string;
author: string;
package.json
+5 -2
View File
@@ -67,6 +67,7 @@
"crypto-js": "4.2.0",
"currency-symbol-map": "5.1.0",
"dayjs": "1.11.8",
"destr": "2.0.3",
"directory-import": "3.3.1",
"dotenv": "16.4.5",
"entities": "4.5.0",
@@ -75,7 +76,6 @@
"form-data": "4.0.0",
"git-rev-sync": "3.0.2",
"googleapis": "134.0.0",
"got": "14.2.1",
"hono": "4.1.5",
"html-to-text": "9.0.5",
"https-proxy-agent": "7.0.4",
@@ -94,6 +94,7 @@
"module-alias": "2.2.3",
"notion-to-md": "3.1.1",
"oauth-1.0a": "2.2.6",
"ofetch": "1.3.4",
"otplib": "12.0.1",
"pac-proxy-agent": "7.0.1",
"proxy-chain": "2.4.0",
@@ -117,6 +118,7 @@
"tough-cookie": "4.1.3",
"tsx": "4.7.1",
"twitter-api-v2": "1.16.1",
"undici": "6.10.2",
"uuid": "9.0.1",
"winston": "3.13.0",
"xxhash-wasm": "1.0.2",
@@ -160,11 +162,12 @@
"eslint-plugin-unicorn": "51.0.1",
"eslint-plugin-yml": "1.13.2",
"fs-extra": "11.2.0",
"got": "14.2.1",
"husky": "9.0.11",
"js-beautify": "1.15.1",
"lint-staged": "15.2.2",
"mockdate": "3.0.5",
"nock": "13.5.4",
"msw": "2.2.13",
"prettier": "3.2.5",
"remark-parse": "11.0.0",
"supertest": "6.3.4",
pnpm-lock.yaml
Generated
+715 -337
View File
File diff suppressed because it is too large Load Diff
tsconfig.json
+1 -1
View File
@@ -2,7 +2,7 @@
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "node",
"moduleResolution": "bundler",
"strict": true,
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx",
vitest.config.ts
+3 -1
View File
@@ -1,4 +1,4 @@
import { defineConfig } from 'vitest/config';
import { defineConfig, configDefaults } from 'vitest/config';
import tsconfigPaths from 'vite-tsconfig-paths';
export default defineConfig({
@@ -10,5 +10,7 @@ export default defineConfig({
exclude: ['lib/routes/**', 'lib/routes-deprecated/**'],
},
testTimeout: 10000,
setupFiles: ['./lib/setup.test.ts'],
exclude: [...configDefaults.exclude, './lib/setup.test.ts'],
},
});