test: puppeteer

lib/utils/proxy/index.ts

@@ -0,0 +1,42 @@
+import { config } from '@/config';
+
+const proxyIsPAC = config.pacUri || config.pacScript;
+
+import pacProxy from './pac-proxy';
+import unifyProxy from './unify-proxy';
+
+let proxyUri: string | undefined;
+let proxyObj: Record<string, any> | undefined;
+let proxyUrlHandler: URL | null = null;
+if (proxyIsPAC) {
+    const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy);
+    proxyUri = proxy.proxyUri;
+    proxyObj = proxy.proxyObj;
+    proxyUrlHandler = proxy.proxyUrlHandler;
+} else {
+    const proxy = unifyProxy(config.proxyUri, config.proxy);
+    proxyUri = proxy.proxyUri;
+    proxyObj = proxy.proxyObj;
+    proxyUrlHandler = proxy.proxyUrlHandler;
+}
+
+let agent = null;
+if (proxyIsPAC) {
+    const { PacProxyAgent } = require('pac-proxy-agent');
+    agent = new PacProxyAgent(`pac+${proxyUri}`);
+} else if (proxyUri) {
+    if (proxyUri.startsWith('http')) {
+        const { HttpsProxyAgent } = require('https-proxy-agent');
+        agent = new HttpsProxyAgent(proxyUri);
+    } else if (proxyUri.startsWith('socks')) {
+        const { SocksProxyAgent } = require('socks-proxy-agent');
+        agent = new SocksProxyAgent(proxyUri);
+    }
+}
+
+export default {
+    agent,
+    proxyUri,
+    proxyObj,
+    proxyUrlHandler,
+};

lib/utils/puppeteer.ts

@@ -1,12 +1,15 @@
-const config = require('@/config').value;
-const { proxyUri, proxyUrlHandler } = require('./unify-proxy');
-let puppeteer = require('puppeteer');
-const proxyChain = require('proxy-chain');
-const logger = require('./logger');
+import { config } from '@/config';
+import puppeteer from 'puppeteer';
+import proxyChain from 'proxy-chain';
+import logger from './logger';
+import proxy from './proxy';
+
+import { type PuppeteerExtra, addExtra } from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 
 const options = {
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
-    headless: 'new',
+    headless: 'new' as const,
     ignoreHTTPSErrors: true,
 };
 
@@ -15,10 +18,14 @@ const options = {
  * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
  * @returns Puppeteer browser
  */
-module.exports = async (extraOptions = {}) => {
+const outPuppeteer = async (
+    extraOptions: {
+        stealth?: boolean;
+    } = {}
+) => {
+    let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
     if (extraOptions.stealth) {
-        const { addExtra } = require('puppeteer-extra');
-        puppeteer = addExtra(puppeteer);
+        insidePuppeteer = addExtra(puppeteer);
 
         // workaround for vercel/nft #54, #283, #304
         require('puppeteer-extra-plugin-stealth/evasions/chrome.app');
@@ -41,27 +48,27 @@ module.exports = async (extraOptions = {}) => {
         require('puppeteer-extra-plugin-user-preferences');
         require('puppeteer-extra-plugin-user-data-dir');
 
-        puppeteer.use(require('puppeteer-extra-plugin-stealth')());
+        insidePuppeteer.use(StealthPlugin());
     }
 
-    if (proxyUri) {
-        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+    if (proxy.proxyUri) {
+        if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
             // only proxies with authentication need to be anonymized
-            if (proxyUrlHandler.protocol === 'http:') {
-                options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`);
+            if (proxy.proxyUrlHandler.protocol === 'http:') {
+                options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxy.proxyUri)}`);
             } else {
                 logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
             }
         } else {
             // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
-            options.args.push(`--proxy-server=${proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
+            options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
         }
     }
     const browser = await (config.puppeteerWSEndpoint
-        ? puppeteer.connect({
+        ? insidePuppeteer.connect({
               browserWSEndpoint: config.puppeteerWSEndpoint,
           })
-        : puppeteer.launch(
+        : insidePuppeteer.launch(
               config.chromiumExecutablePath
                   ? {
                         executablePath: config.chromiumExecutablePath,
@@ -75,3 +82,5 @@ module.exports = async (extraOptions = {}) => {
 
     return browser;
 };
+
+export default outPuppeteer;

lib/utils/request-wrapper.ts

@@ -2,40 +2,7 @@ import { config } from '@/config';
 import logger from '@/utils/logger';
 import http, { type RequestOptions } from 'node:http';
 import https from 'node:https';
-
-const proxyIsPAC = config.pacUri || config.pacScript;
-
-import pacProxy from './pac-proxy';
-import unifyProxy from './unify-proxy';
-
-let proxyUri: string | undefined;
-let proxyObj: Record<string, any> | undefined;
-let proxyUrlHandler: URL | null = null;
-if (proxyIsPAC) {
-    const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy);
-    proxyUri = proxy.proxyUri;
-    proxyObj = proxy.proxyObj;
-    proxyUrlHandler = proxy.proxyUrlHandler;
-} else {
-    const proxy = unifyProxy(config.proxyUri, config.proxy);
-    proxyUri = proxy.proxyUri;
-    proxyObj = proxy.proxyObj;
-    proxyUrlHandler = proxy.proxyUrlHandler;
-}
-
-let agent = null;
-if (proxyIsPAC) {
-    const { PacProxyAgent } = require('pac-proxy-agent');
-    agent = new PacProxyAgent(`pac+${proxyUri}`);
-} else if (proxyUri) {
-    if (proxyUri.startsWith('http')) {
-        const { HttpsProxyAgent } = require('https-proxy-agent');
-        agent = new HttpsProxyAgent(proxyUri);
-    } else if (proxyUri.startsWith('socks')) {
-        const { SocksProxyAgent } = require('socks-proxy-agent');
-        agent = new SocksProxyAgent(proxyUri);
-    }
-}
+import proxy from '@/utils/proxy';
 
 let proxyWrapper: (
     url: string,
@@ -44,8 +11,8 @@ let proxyWrapper: (
     }
 ) => boolean = () => false;
 
-if (agent) {
-    const proxyRegex = new RegExp(proxyObj.url_regex);
+if (proxy.agent) {
+    const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
     const protocolMatch = (protocolLike?: string | null) => protocolLike?.toLowerCase().startsWith('http');
 
     proxyWrapper = (url, options) => {
@@ -55,10 +22,10 @@ if (agent) {
         } catch {
             // ignore
         }
-        if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler?.host)) {
-            options.agent = agent;
-            if (proxyObj.auth) {
-                options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
+        if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxy.proxyUrlHandler?.host)) {
+            options.agent = proxy.agent || false;
+            if (proxy.proxyObj.auth) {
+                options.headers['Proxy-Authorization'] = `Basic ${proxy.proxyObj.auth}`;
             }
             return true;
         }

lib/utils/wait.ts

@@ -1,5 +1,7 @@
-module.exports = function wait(ms) {
+function wait(ms: number) {
     return new Promise((resolve) => {
         setTimeout(resolve, ms);
     });
-};
+}
+
+export default wait;