feat(utils/puppeteer): replace puppeteer-extra-plugin-stealth with rebrowser (#19506)

* feat: use rebrowser

* fix: remove rebrowser-puppeteer-core

* fix: chrome not found

* fix: replace remove-infobar with disable-blink-features

remove-infobar was removed since M76 https://github.com/puppeteer/puppeteer/issues/1765

* feat: remove outdated stealth plugin

* fix: remove puppeteer non stealth test

.devcontainer/devcontainer.json

@@ -38,10 +38,10 @@
 
     "onCreateCommand": "sudo apt-get update && export DEBIAN_FRONTEND=noninteractive && sudo apt-get -y install --no-install-recommends ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-utils redis-server default-jre-headless && sudo apt-get autoremove -y && sudo apt-get clean -y && sudo rm -rf /var/lib/apt/lists/*",
 
-    "updateContentCommand": "export JAVA_HOME=/usr/lib/jvm/default-java && pnpm config set store-dir ~/.local/share/pnpm/store && pnpm i && pnpm rb",
+    "updateContentCommand": "export JAVA_HOME=/usr/lib/jvm/default-java && pnpm config set store-dir ~/.local/share/pnpm/store && pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome",
 
     // Use 'postCreateCommand' to run commands after the container is created.
-    "postCreateCommand": "pnpm i && pnpm rb",
+    "postCreateCommand": "pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome",
 
     // Disable auto start dev env since codespaces sometimes fails to attach to the terminal
     // "postAttachCommand": {

.github/workflows/issue-command.yml

@@ -77,7 +77,7 @@ jobs:
           cache: 'pnpm'
 
       - name: Install dependencies (pnpm)
-        run: pnpm i && pnpm rb
+        run: pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome
 
       - name: Fetch affected routes
         id: fetch-route

.github/workflows/test.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install dependencies (pnpm)
         run: pnpm i
       - name: Run postinstall script for dependencies
-        run: pnpm rb
+        run: pnpm rb && pnpx rebrowser-puppeteer browsers install chrome
       - name: Build routes
         run: pnpm build
       - name: Test all and generate coverage
@@ -81,7 +81,7 @@ jobs:
         run: pnpm i
         env: ${{ fromJSON(matrix.chromium.environment) }}
       - name: Run postinstall script for dependencies
-        run: pnpm rb
+        run: pnpm rb && pnpx rebrowser-puppeteer browsers install chrome
         env: ${{ fromJSON(matrix.chromium.environment) }}
       - name: Build routes
         run: pnpm build

.gitpod.yml

@@ -14,7 +14,7 @@ tasks:
       sudo apt update
       sudo apt install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-util
       sudo apt install -y redis-server
-    init: pnpm i && pnpm rb
+    init: pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome
   - name: app
     command: pnpm run dev
     openMode: tab-after

.idx/dev.nix

@@ -51,7 +51,7 @@
       # Runs when a workspace is first created
       onCreate = {
         # Example: install JS dependencies from NPM
-        pnpm-install = "pnpm i && pnpm rb";
+        pnpm-install = "pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome";
       };
       # Runs when the workspace is (re)started
       onStart = {

Dockerfile

@@ -40,7 +40,7 @@ WORKDIR /ver
 COPY ./package.json /app/
 RUN \
     set -ex && \
-    grep -Po '(?<="puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version
+    grep -Po '(?<="rebrowser-puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version
     # grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \
     # grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version
 
@@ -92,7 +92,7 @@ ARG TARGETPLATFORM
 ARG USE_CHINA_NPM_REGISTRY=0
 ARG PUPPETEER_SKIP_DOWNLOAD=1
 # The official recommended way to use Puppeteer on x86(_64) is to use the bundled Chromium from Puppeteer:
-# https://pptr.dev/faq#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy
+# https://pptr.dev/faq#q-why-doesnt-puppeteer-vxxx-workwith-chromium-vyyy
 RUN \
     set -ex ; \
     if [ "$PUPPETEER_SKIP_DOWNLOAD" = 0 ] && [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
@@ -104,8 +104,9 @@ RUN \
         echo 'Downloading Chromium...' && \
         unset PUPPETEER_SKIP_DOWNLOAD && \
         corepack enable pnpm && \
-        pnpm --allow-build=puppeteer add puppeteer@$(cat /app/.puppeteer_version) --save-prod && \
-        pnpm rb ; \
+        pnpm --allow-build=rebrowser-puppeteer add rebrowser-puppeteer@$(cat /app/.puppeteer_version) --save-prod && \
+        pnpm rb && \
+        pnpx rebrowser-puppeteer browsers install chrome ; \
     else \
         mkdir -p /app/node_modules/.cache/puppeteer ; \
     fi;

eslint.config.mjs

@@ -225,12 +225,7 @@ unicorn.configs.recommended,
 
         // https://github.com/eslint-community/eslint-plugin-n
         // node specific rules
-        'n/no-extraneous-require': ['error', {
-            allowModules: [
-                'puppeteer-extra-plugin-user-preferences',
-                'puppeteer-extra-plugin-user-data-dir',
-            ],
-        }],
+        'n/no-extraneous-require': 'error',
 
         'n/no-deprecated-api': 'warn',
         'n/no-missing-import': 'off',

lib/routes/cmde/index.ts

@@ -17,7 +17,7 @@ export const route: Route = {
 async function handler(ctx) {
     const cate = ctx.req.param('cate') ?? 'xwdt/zxyw';
     const url = `${rootURL}/${cate}/`;
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const data = await cache.tryGet(url, async () => {
         const page = await browser.newPage();
         await page.setRequestInterception(true);

lib/routes/gov/customs/list.ts

@@ -59,7 +59,7 @@ async function handler(ctx) {
             break;
     }
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
 
     const list = await cache.tryGet(
         link,

lib/routes/gov/hangzhou/zwfw.ts

@@ -38,7 +38,7 @@ async function handler() {
     const host = 'https://www.hangzhou.gov.cn/col/col1256349/index.html';
     const response = await ofetch(host);
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const link = host;
     const formatted = response
         .replace('<script type="text/xml">', '')

lib/routes/gov/pbc/goutongjiaoliu.ts

@@ -32,7 +32,7 @@ export const route: Route = {
 async function handler() {
     const link = 'http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html';
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/ielts/index.ts

@@ -26,7 +26,7 @@ async function handler() {
     const html = await cache.tryGet(
         targetUrl,
         async () => {
-            const browser = await puppeteer({ stealth: true });
+            const browser = await puppeteer();
             const page = await browser.newPage();
             await page.setRequestInterception(true);
             page.on('request', (request) => {

lib/routes/missav/new.ts

@@ -39,7 +39,7 @@ async function handler() {
     const baseUrl = 'https://missav.ws';
     const url = `${baseUrl}/${urlPath}`;
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/njust/utils.ts

@@ -3,7 +3,7 @@ import puppeteer from '@/utils/puppeteer';
 
 async function getContent(url, pptr = false) {
     if (pptr) {
-        const browser = await puppeteer({ stealth: true });
+        const browser = await puppeteer();
         try {
             const page = await browser.newPage();
             // 更改 window.navigator.webdriver 值以避开反爬

lib/routes/nuaa/utils/pypasswaf.ts

@@ -6,7 +6,7 @@ import puppeteer from '@/utils/puppeteer';
  * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.headers.cookie`即可
  */
 export default async function getCookie(host) {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/twitter/api/web-api/login.ts

@@ -28,9 +28,7 @@ async function login({ username, password, authenticationSecret }) {
         await loginLimiterQueue.removeTokens(1);
 
         const cookieJar = new CookieJar();
-        const browser = await puppeteer({
-            stealth: true,
-        });
+        const browser = await puppeteer();
         const page = await browser.newPage();
         await page.goto('https://x.com/i/flow/login');
         await page.waitForSelector('input[autocomplete="username"]');

lib/routes/uestc/auto.ts

@@ -31,7 +31,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/uestc/cqe.ts

@@ -51,7 +51,7 @@ async function handler(ctx) {
         throw new InvalidParameterError('type not supported');
     }
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/uestc/scse.ts

@@ -45,7 +45,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/uestc/sice.ts

@@ -32,7 +32,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/uestc/sise.ts

@@ -66,7 +66,7 @@ async function handler(ctx) {
         throw new InvalidParameterError('type not supported');
     }
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

lib/routes/xiaohongshu/util.ts

@@ -31,9 +31,7 @@ const getUser = (url, cache) =>
     cache.tryGet(
         url,
         async () => {
-            const browser = await puppeteer({
-                stealth: true,
-            });
+            const browser = await puppeteer();
             try {
                 const page = await browser.newPage();
                 await page.setRequestInterception(true);

lib/routes/xueqiu/cookies.ts

@@ -7,7 +7,7 @@ export const parseToken = (link: string) =>
     cache.tryGet(
         'xueqiu:token',
         async () => {
-            const browser = await puppeteer({ stealth: true });
+            const browser = await puppeteer();
             const page = await browser.newPage();
             await page.setRequestInterception(true);
             page.on('request', (request) => {

lib/routes/xueqiu/user.ts

@@ -49,7 +49,7 @@ async function handler(ctx) {
     const link = `${rootUrl}/u/${id}`;
     const token = await parseToken(link);
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     try {
         const mainPage = await browser.newPage();
 

lib/utils/puppeteer-utils.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it, vi, afterEach } from 'vitest';
 import { parseCookieArray, constructCookieArray, setCookies, getCookies } from '@/utils/puppeteer-utils';
 import puppeteer from '@/utils/puppeteer';
-import type { Browser } from 'puppeteer';
+import type { Browser } from 'rebrowser-puppeteer';
 
 let browser: Browser | null = null;
 

lib/utils/puppeteer.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it, vi, afterEach } from 'vitest';
 import wait from './wait';
-import { type Browser } from 'puppeteer';
+import { type Browser } from 'rebrowser-puppeteer';
 
 let browser: Browser | null = null;
 
@@ -44,32 +44,19 @@ describe('puppeteer', () => {
         browser = null;
     }, 45000);
 
-    if (!process.env.GITHUB_ACTIONS) {
-        it('puppeteer without stealth', async () => {
-            const { default: puppeteer } = await import('./puppeteer');
-            browser = await puppeteer({ stealth: false });
-            const page = await browser.newPage();
-            await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' });
-            // page rendering is not instant, wait for expected elements to appear
-            const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.failed`).then((hd) => hd?.evaluate((e) => e.textContent))));
-            // the website return empty string from time to time for no reason
-            // since we don't really care whether puppeteer without stealth passes the bot test, just let it go
-            expect(['present (failed)', '']).toContain(webDriverTest);
-            expect(['missing (failed)', '']).toContain(chromeTest);
-        }, 20000);
-
-        it('puppeteer with stealth', async () => {
-            const { default: puppeteer } = await import('./puppeteer');
-            browser = await puppeteer({ stealth: true });
-            const page = await browser.newPage();
-            await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' });
-            // page rendering is not instant, wait for expected elements to appear
-            const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.passed`).then((hd) => hd?.evaluate((e) => e.textContent))));
-            // these are something we really care about
-            expect(webDriverTest).toBe('missing (passed)');
-            expect(chromeTest).toBe('present (passed)');
-        }, 20000);
-    }
+    // if (!process.env.GITHUB_ACTIONS) {
+    it('puppeteer stealth test', async () => {
+        const { default: puppeteer } = await import('./puppeteer');
+        browser = await puppeteer();
+        const page = await browser.newPage();
+        await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' });
+        // page rendering is not instant, wait for expected elements to appear
+        const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.passed`).then((hd) => hd?.evaluate((e) => e.textContent))));
+        // these are something we really care about
+        expect(webDriverTest).toBe('missing (passed)');
+        expect(chromeTest).toBe('present (passed)');
+    }, 20000);
+    // }
 
     it('puppeteer accept http proxy uri w/ auth', async () => {
         process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';

lib/utils/puppeteer.ts

@@ -1,34 +1,29 @@
 import { config } from '@/config';
-import puppeteer, { Browser, Page } from 'puppeteer';
+import puppeteer, { Browser, Page } from 'rebrowser-puppeteer';
 import logger from './logger';
 import proxy from './proxy';
 import { anonymizeProxy } from 'proxy-chain';
 
-import { type PuppeteerExtra, addExtra } from 'puppeteer-extra';
-import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-
 /**
  * @deprecated use getPage instead
- * @param {Object} extraOptions
- * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
  * @returns Puppeteer browser
  */
-const outPuppeteer = async (
-    extraOptions: {
-        stealth?: boolean;
-    } = {}
-) => {
+const outPuppeteer = async () => {
     const options = {
-        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
+        args: [
+            '--no-sandbox',
+            '--disable-setuid-sandbox',
+            '--disable-blink-features=AutomationControlled',
+            '--window-position=0,0',
+            '--ignore-certificate-errors',
+            '--ignore-certificate-errors-spki-list',
+            `--user-agent=${config.ua}`,
+        ],
         headless: true,
         ignoreHTTPSErrors: true,
     };
 
-    let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
-    if (extraOptions.stealth) {
-        insidePuppeteer = addExtra(puppeteer);
-        insidePuppeteer.use(StealthPlugin());
-    }
+    const insidePuppeteer: typeof puppeteer = puppeteer;
 
     const currentProxy = proxy.getCurrentProxy();
     if (currentProxy && proxy.proxyObj.url_regex === '.*') {
@@ -79,14 +74,20 @@ export const getPuppeteerPage = async (
     } = {}
 ) => {
     const options = {
-        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
+        args: [
+            '--no-sandbox',
+            '--disable-setuid-sandbox',
+            '--disable-blink-features=AutomationControlled',
+            '--window-position=0,0',
+            '--ignore-certificate-errors',
+            '--ignore-certificate-errors-spki-list',
+            `--user-agent=${config.ua}`,
+        ],
         headless: true,
         ignoreHTTPSErrors: true,
     };
 
-    let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
-    insidePuppeteer = addExtra(puppeteer);
-    insidePuppeteer.use(StealthPlugin());
+    const insidePuppeteer: typeof puppeteer = puppeteer;
 
     let allowProxy = false;
     const proxyRegex = new RegExp(proxy.proxyObj.url_regex);

package.json

@@ -111,14 +111,10 @@
         "p-map": "7.0.3",
         "pac-proxy-agent": "7.2.0",
         "proxy-chain": "2.5.9",
-        "puppeteer": "24.11.1",
-        "puppeteer-extra": "3.3.6",
-        "puppeteer-extra-plugin-stealth": "2.11.2",
-        "puppeteer-extra-plugin-user-data-dir": "2.4.1",
-        "puppeteer-extra-plugin-user-preferences": "2.4.1",
         "query-string": "9.2.1",
         "rate-limiter-flexible": "7.1.1",
         "re2js": "1.1.0",
+        "rebrowser-puppeteer": "24.8.1",
         "rfc4648": "1.5.4",
         "rss-parser": "3.13.0",
         "sanitize-html": "2.17.0",
@@ -211,8 +207,8 @@
             "@bbob/core",
             "@bbob/parser",
             "@bbob/plugin-helper",
-            "@bbob/preset-html5",
             "@bbob/preset",
+            "@bbob/preset-html5",
             "bufferutil",
             "core-js",
             "es5-ext",
@@ -221,6 +217,7 @@
             "msw",
             "protobufjs",
             "puppeteer",
+            "rebrowser-puppeteer",
             "rolldown",
             "utf-8-validate",
             "vue-demi"