From 857d5989c1f7cb66af62cd9b417151719dd346b5 Mon Sep 17 00:00:00 2001 From: Tony Date: Tue, 1 Jul 2025 04:38:05 +0800 Subject: [PATCH] feat(utils/puppeteer): replace `puppeteer-extra-plugin-stealth` with `rebrowser` (#19506) * feat: use rebrowser * fix: remove rebrowser-puppeteer-core * fix: chrome not found * fix: replace remove-infobar with disable-blink-features remove-infobar was removed since M76 https://github.com/puppeteer/puppeteer/issues/1765 * feat: remove outdated stealth plugin * fix: remove puppeteer non stealth test --- .devcontainer/devcontainer.json | 4 +- .github/workflows/issue-command.yml | 2 +- .github/workflows/test.yml | 4 +- .gitpod.yml | 2 +- .idx/dev.nix | 2 +- Dockerfile | 9 +- eslint.config.mjs | 7 +- lib/routes/cmde/index.ts | 2 +- lib/routes/gov/customs/list.ts | 2 +- lib/routes/gov/hangzhou/zwfw.ts | 2 +- lib/routes/gov/pbc/goutongjiaoliu.ts | 2 +- lib/routes/ielts/index.ts | 2 +- lib/routes/missav/new.ts | 2 +- lib/routes/njust/utils.ts | 2 +- lib/routes/nuaa/utils/pypasswaf.ts | 2 +- lib/routes/twitter/api/web-api/login.ts | 4 +- lib/routes/uestc/auto.ts | 2 +- lib/routes/uestc/cqe.ts | 2 +- lib/routes/uestc/scse.ts | 2 +- lib/routes/uestc/sice.ts | 2 +- lib/routes/uestc/sise.ts | 2 +- lib/routes/xiaohongshu/util.ts | 4 +- lib/routes/xueqiu/cookies.ts | 2 +- lib/routes/xueqiu/user.ts | 2 +- lib/utils/puppeteer-utils.test.ts | 2 +- lib/utils/puppeteer.test.ts | 41 +- lib/utils/puppeteer.ts | 43 +- package.json | 9 +- pnpm-lock.yaml | 1748 +++++++++-------------- 29 files changed, 776 insertions(+), 1135 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 048a2c375c..13c943fd55 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -38,10 +38,10 @@ "onCreateCommand": "sudo apt-get update && export DEBIAN_FRONTEND=noninteractive && sudo apt-get -y install --no-install-recommends ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-utils redis-server default-jre-headless && sudo apt-get autoremove -y && sudo apt-get clean -y && sudo rm -rf /var/lib/apt/lists/*", - "updateContentCommand": "export JAVA_HOME=/usr/lib/jvm/default-java && pnpm config set store-dir ~/.local/share/pnpm/store && pnpm i && pnpm rb", + "updateContentCommand": "export JAVA_HOME=/usr/lib/jvm/default-java && pnpm config set store-dir ~/.local/share/pnpm/store && pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome", // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pnpm i && pnpm rb", + "postCreateCommand": "pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome", // Disable auto start dev env since codespaces sometimes fails to attach to the terminal // "postAttachCommand": { diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index f6872b0219..543b744a6c 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -77,7 +77,7 @@ jobs: cache: 'pnpm' - name: Install dependencies (pnpm) - run: pnpm i && pnpm rb + run: pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome - name: Fetch affected routes id: fetch-route diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f208f23b0b..fb34983228 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: - name: Install dependencies (pnpm) run: pnpm i - name: Run postinstall script for dependencies - run: pnpm rb + run: pnpm rb && pnpx rebrowser-puppeteer browsers install chrome - name: Build routes run: pnpm build - name: Test all and generate coverage @@ -81,7 +81,7 @@ jobs: run: pnpm i env: ${{ fromJSON(matrix.chromium.environment) }} - name: Run postinstall script for dependencies - run: pnpm rb + run: pnpm rb && pnpx rebrowser-puppeteer browsers install chrome env: ${{ fromJSON(matrix.chromium.environment) }} - name: Build routes run: pnpm build diff --git a/.gitpod.yml b/.gitpod.yml index e88bef50a3..f4079c4186 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,7 +14,7 @@ tasks: sudo apt update sudo apt install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-util sudo apt install -y redis-server - init: pnpm i && pnpm rb + init: pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome - name: app command: pnpm run dev openMode: tab-after diff --git a/.idx/dev.nix b/.idx/dev.nix index a8e6be8704..c392c73ca4 100644 --- a/.idx/dev.nix +++ b/.idx/dev.nix @@ -51,7 +51,7 @@ # Runs when a workspace is first created onCreate = { # Example: install JS dependencies from NPM - pnpm-install = "pnpm i && pnpm rb"; + pnpm-install = "pnpm i && pnpm rb && pnpx rebrowser-puppeteer browsers install chrome"; }; # Runs when the workspace is (re)started onStart = { diff --git a/Dockerfile b/Dockerfile index b5d1169a86..b5770c9605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ WORKDIR /ver COPY ./package.json /app/ RUN \ set -ex && \ - grep -Po '(?<="puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version + grep -Po '(?<="rebrowser-puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version # grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \ # grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version @@ -92,7 +92,7 @@ ARG TARGETPLATFORM ARG USE_CHINA_NPM_REGISTRY=0 ARG PUPPETEER_SKIP_DOWNLOAD=1 # The official recommended way to use Puppeteer on x86(_64) is to use the bundled Chromium from Puppeteer: -# https://pptr.dev/faq#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy +# https://pptr.dev/faq#q-why-doesnt-puppeteer-vxxx-workwith-chromium-vyyy RUN \ set -ex ; \ if [ "$PUPPETEER_SKIP_DOWNLOAD" = 0 ] && [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ @@ -104,8 +104,9 @@ RUN \ echo 'Downloading Chromium...' && \ unset PUPPETEER_SKIP_DOWNLOAD && \ corepack enable pnpm && \ - pnpm --allow-build=puppeteer add puppeteer@$(cat /app/.puppeteer_version) --save-prod && \ - pnpm rb ; \ + pnpm --allow-build=rebrowser-puppeteer add rebrowser-puppeteer@$(cat /app/.puppeteer_version) --save-prod && \ + pnpm rb && \ + pnpx rebrowser-puppeteer browsers install chrome ; \ else \ mkdir -p /app/node_modules/.cache/puppeteer ; \ fi; diff --git a/eslint.config.mjs b/eslint.config.mjs index b666a661fe..01dbe35974 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -225,12 +225,7 @@ unicorn.configs.recommended, // https://github.com/eslint-community/eslint-plugin-n // node specific rules - 'n/no-extraneous-require': ['error', { - allowModules: [ - 'puppeteer-extra-plugin-user-preferences', - 'puppeteer-extra-plugin-user-data-dir', - ], - }], + 'n/no-extraneous-require': 'error', 'n/no-deprecated-api': 'warn', 'n/no-missing-import': 'off', diff --git a/lib/routes/cmde/index.ts b/lib/routes/cmde/index.ts index c0d375e22f..99e34f4c5c 100644 --- a/lib/routes/cmde/index.ts +++ b/lib/routes/cmde/index.ts @@ -17,7 +17,7 @@ export const route: Route = { async function handler(ctx) { const cate = ctx.req.param('cate') ?? 'xwdt/zxyw'; const url = `${rootURL}/${cate}/`; - const browser = await puppeteer({ stealth: true }); + const browser = await puppeteer(); const data = await cache.tryGet(url, async () => { const page = await browser.newPage(); await page.setRequestInterception(true); diff --git a/lib/routes/gov/customs/list.ts b/lib/routes/gov/customs/list.ts index 389f9d8ff0..27d1b03b12 100644 --- a/lib/routes/gov/customs/list.ts +++ b/lib/routes/gov/customs/list.ts @@ -59,7 +59,7 @@ async function handler(ctx) { break; } - const browser = await puppeteer({ stealth: true }); + const browser = await puppeteer(); const list = await cache.tryGet( link, diff --git a/lib/routes/gov/hangzhou/zwfw.ts b/lib/routes/gov/hangzhou/zwfw.ts index f2adbd7557..6dd7ff1f56 100644 --- a/lib/routes/gov/hangzhou/zwfw.ts +++ b/lib/routes/gov/hangzhou/zwfw.ts @@ -38,7 +38,7 @@ async function handler() { const host = 'https://www.hangzhou.gov.cn/col/col1256349/index.html'; const response = await ofetch(host); - const browser = await puppeteer({ stealth: true }); + const browser = await puppeteer(); const link = host; const formatted = response .replace('