opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-08-31 01:01:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

test: puppeteer

Browse Source
This commit is contained in:
DIYgod
2024-01-31 02:14:48 +08:00
parent 174752fef5
commit 2e7bd66940
8 changed files with 97 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/utils/proxy/index.ts
+42
View File
@@ -0,0 +1,42 @@
import { config } from '@/config';
const proxyIsPAC = config.pacUri || config.pacScript;
import pacProxy from './pac-proxy';
import unifyProxy from './unify-proxy';
let proxyUri: string | undefined;
let proxyObj: Record<string, any> | undefined;
let proxyUrlHandler: URL | null = null;
if (proxyIsPAC) {
const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy);
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
} else {
const proxy = unifyProxy(config.proxyUri, config.proxy);
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
}
let agent = null;
if (proxyIsPAC) {
const { PacProxyAgent } = require('pac-proxy-agent');
agent = new PacProxyAgent(`pac+${proxyUri}`);
} else if (proxyUri) {
if (proxyUri.startsWith('http')) {
const { HttpsProxyAgent } = require('https-proxy-agent');
agent = new HttpsProxyAgent(proxyUri);
} else if (proxyUri.startsWith('socks')) {
const { SocksProxyAgent } = require('socks-proxy-agent');
agent = new SocksProxyAgent(proxyUri);
}
}
export default {
agent,
proxyUri,
proxyObj,
proxyUrlHandler,
};
lib/utils/pac-proxy.ts → lib/utils/proxy/pac-proxy.ts
View File
lib/utils/unify-proxy.ts → lib/utils/proxy/unify-proxy.ts
View File
lib/utils/puppeteer.js → lib/utils/puppeteer.ts
+26 -17
View File
@@ -1,12 +1,15 @@
const config = require('@/config').value;
const { proxyUri, proxyUrlHandler } = require('./unify-proxy');
let puppeteer = require('puppeteer');
const proxyChain = require('proxy-chain');
const logger = require('./logger');
import { config } from '@/config';
import puppeteer from 'puppeteer';
import proxyChain from 'proxy-chain';
import logger from './logger';
import proxy from './proxy';
import { type PuppeteerExtra, addExtra } from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
const options = {
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
headless: 'new',
headless: 'new' as const,
ignoreHTTPSErrors: true,
};
@@ -15,10 +18,14 @@ const options = {
* @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
* @returns Puppeteer browser
*/
module.exports = async (extraOptions = {}) => {
const outPuppeteer = async (
extraOptions: {
stealth?: boolean;
} = {}
) => {
let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
if (extraOptions.stealth) {
const { addExtra } = require('puppeteer-extra');
puppeteer = addExtra(puppeteer);
insidePuppeteer = addExtra(puppeteer);
// workaround for vercel/nft #54, #283, #304
require('puppeteer-extra-plugin-stealth/evasions/chrome.app');
@@ -41,27 +48,27 @@ module.exports = async (extraOptions = {}) => {
require('puppeteer-extra-plugin-user-preferences');
require('puppeteer-extra-plugin-user-data-dir');
puppeteer.use(require('puppeteer-extra-plugin-stealth')());
insidePuppeteer.use(StealthPlugin());
}
if (proxyUri) {
if (proxyUrlHandler.username || proxyUrlHandler.password) {
if (proxy.proxyUri) {
if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
// only proxies with authentication need to be anonymized
if (proxyUrlHandler.protocol === 'http:') {
options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`);
if (proxy.proxyUrlHandler.protocol === 'http:') {
options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxy.proxyUri)}`);
} else {
logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
}
} else {
// Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
options.args.push(`--proxy-server=${proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
}
}
const browser = await (config.puppeteerWSEndpoint
? puppeteer.connect({
? insidePuppeteer.connect({
browserWSEndpoint: config.puppeteerWSEndpoint,
})
: puppeteer.launch(
: insidePuppeteer.launch(
config.chromiumExecutablePath
? {
executablePath: config.chromiumExecutablePath,
@@ -75,3 +82,5 @@ module.exports = async (extraOptions = {}) => {
return browser;
};
export default outPuppeteer;
lib/utils/request-wrapper.ts
+7 -40
View File
@@ -2,40 +2,7 @@ import { config } from '@/config';
import logger from '@/utils/logger';
import http, { type RequestOptions } from 'node:http';
import https from 'node:https';
const proxyIsPAC = config.pacUri || config.pacScript;
import pacProxy from './pac-proxy';
import unifyProxy from './unify-proxy';
let proxyUri: string | undefined;
let proxyObj: Record<string, any> | undefined;
let proxyUrlHandler: URL | null = null;
if (proxyIsPAC) {
const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy);
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
} else {
const proxy = unifyProxy(config.proxyUri, config.proxy);
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
}
let agent = null;
if (proxyIsPAC) {
const { PacProxyAgent } = require('pac-proxy-agent');
agent = new PacProxyAgent(`pac+${proxyUri}`);
} else if (proxyUri) {
if (proxyUri.startsWith('http')) {
const { HttpsProxyAgent } = require('https-proxy-agent');
agent = new HttpsProxyAgent(proxyUri);
} else if (proxyUri.startsWith('socks')) {
const { SocksProxyAgent } = require('socks-proxy-agent');
agent = new SocksProxyAgent(proxyUri);
}
}
import proxy from '@/utils/proxy';
let proxyWrapper: (
url: string,
@@ -44,8 +11,8 @@ let proxyWrapper: (
}
) => boolean = () => false;
if (agent) {
const proxyRegex = new RegExp(proxyObj.url_regex);
if (proxy.agent) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
const protocolMatch = (protocolLike?: string | null) => protocolLike?.toLowerCase().startsWith('http');
proxyWrapper = (url, options) => {
@@ -55,10 +22,10 @@ if (agent) {
} catch {
// ignore
}
if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler?.host)) {
options.agent = agent;
if (proxyObj.auth) {
options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxy.proxyUrlHandler?.host)) {
options.agent = proxy.agent || false;
if (proxy.proxyObj.auth) {
options.headers['Proxy-Authorization'] = `Basic ${proxy.proxyObj.auth}`;
}
return true;
}
lib/utils/wait.js → lib/utils/wait.ts
+4 -2
View File
@@ -1,5 +1,7 @@
module.exports = function wait(ms) {
function wait(ms: number) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
};
}
export default wait;
test/utils/puppeteer.js → test/utils/puppeteer.ts
+16 -24
View File
@@ -1,7 +1,9 @@
let puppeteer;
const wait = require('../../lib/utils/wait');
import { describe, expect, it, jest, afterEach } from '@jest/globals';
import wait from '../../lib/utils/wait';
import puppeteer from '../../lib/utils/puppeteer';
import { type Browser } from 'puppeteer';
let browser = null;
let browser: Browser | null = null;
afterEach(() => {
if (browser) {
@@ -22,7 +24,6 @@ afterEach(() => {
describe('puppeteer', () => {
it('puppeteer run', async () => {
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
const startTime = Date.now();
const page = await browser.newPage();
@@ -33,23 +34,22 @@ describe('puppeteer', () => {
const html = await page.evaluate(() => document.body.innerHTML);
expect(html.length).toBeGreaterThan(0);
expect(browser.process().exitCode).toBe(null); // browser is still running
expect(browser.process()?.exitCode).toBe(null); // browser is still running
const sleepTime = 31 * 1000 - (Date.now() - startTime); // prevent long loading time from failing the test
if (sleepTime > 0) {
await wait(sleepTime);
}
expect(browser.process().exitCode).toBe(0); // browser is closed
expect(browser.process()?.exitCode).toBe(0); // browser is closed
browser = null;
}, 45000);
if (!process.env.GITHUB_ACTIONS) {
it('puppeteer without stealth', async () => {
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer({ stealth: false });
const page = await browser.newPage();
await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' });
// page rendering is not instant, wait for expected elements to appear
const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.failed`).then((hd) => hd.evaluate((e) => e.textContent))));
const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.failed`).then((hd) => hd?.evaluate((e) => e.textContent))));
// the website return empty string from time to time for no reason
// since we don't really care whether puppeteer without stealth passes the bot test, just let it go
expect(['present (failed)', '']).toContain(webDriverTest);
@@ -57,12 +57,11 @@ describe('puppeteer', () => {
}, 15000);
it('puppeteer with stealth', async () => {
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer({ stealth: true });
const page = await browser.newPage();
await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' });
// page rendering is not instant, wait for expected elements to appear
const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.passed`).then((hd) => hd.evaluate((e) => e.textContent))));
const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.passed`).then((hd) => hd?.evaluate((e) => e.textContent))));
// these are something we really care about
expect(webDriverTest).toBe('missing (passed)');
expect(chromeTest).toBe('present (passed)');
@@ -72,29 +71,26 @@ describe('puppeteer', () => {
it('puppeteer accept http proxy uri w/ auth', async () => {
process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
// trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it
expect(browser.process().spawnargs.some((arg) => /^--proxy-server=http:\/\/.*[^/]$/.test(arg))).toBe(true);
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=http:\/\/.*[^/]$/.test(arg))).toBe(true);
});
it('puppeteer reject https proxy uri w/ auth', async () => {
process.env.PROXY_URI = 'https://user:pass@rsshub.proxy:2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
expect(browser.process().spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
});
it('puppeteer reject socks proxy uri w/ auth', async () => {
process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
expect(browser.process().spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
});
it('puppeteer accept http proxy', async () => {
@@ -102,10 +98,9 @@ describe('puppeteer', () => {
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
expect(browser.process().spawnargs.some((arg) => /^--proxy-server=http:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=http:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
}, 10000);
it('puppeteer accept https proxy', async () => {
@@ -113,10 +108,9 @@ describe('puppeteer', () => {
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
expect(browser.process().spawnargs.some((arg) => /^--proxy-server=https:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=https:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
}, 10000);
it('puppeteer accept socks4a proxy', async () => {
@@ -124,10 +118,9 @@ describe('puppeteer', () => {
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
expect(browser.process().spawnargs.some((arg) => /^--proxy-server=socks4:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks4:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
}, 10000);
it('puppeteer accept socks5h proxy', async () => {
@@ -135,9 +128,8 @@ describe('puppeteer', () => {
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
puppeteer = require('../../lib/utils/puppeteer');
browser = await puppeteer();
expect(browser.process().spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
}, 10000);
});
tsconfig.json
+2 -1
View File
@@ -7,6 +7,7 @@
"paths": {
"@/*": ["./lib/*"]
},
"esModuleInterop": true
"esModuleInterop": true,
"noImplicitAny": false,
}
}