From 478df5e488c0ad3c0b4dc9b5ad52e1137e5b928a Mon Sep 17 00:00:00 2001 From: Arslan Ablikim Date: Sat, 11 Jul 2026 00:44:03 +0800 Subject: [PATCH] fix(route/makerworld): bypass Cloudflare challenge and adapt to new API shape (#22411) * fix(route/makerworld): bypass Cloudflare challenge and adapt to new API shape makerworld.com now sits behind a Cloudflare bot-management challenge that fingerprints the TLS/HTTP client rather than just headers, so plain fetch requests get a 403 even with a browser-like User-Agent. Fall back to a stealth Playwright page load when that happens. The trending page's data shape also changed: designs moved from `pageProps.popularDesignsData` to `pageProps.v2Props.foryouData.hits[].design`, mixed in with non-design promo/community entries, and `tags`/`startTime` were replaced by `createTime` (no tags exposed on this endpoint anymore). * fix(route/makerworld): always fetch via browser instead of trying ofetch first Cloudflare's challenge fingerprints the client itself, so the plain fetch attempt was guaranteed to always fail with a 403 first. Go straight to the Playwright fallback instead of paying for a request known to fail. --- lib/routes/makerworld/contest.ts | 10 ++------ lib/routes/makerworld/trending.ts | 27 +++++++++------------- lib/routes/makerworld/user-upload.ts | 13 ++--------- lib/routes/makerworld/utils.ts | 34 ++++++++++++++++++++++------ 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/lib/routes/makerworld/contest.ts b/lib/routes/makerworld/contest.ts index 5633f9cee8..5a465cfe14 100644 --- a/lib/routes/makerworld/contest.ts +++ b/lib/routes/makerworld/contest.ts @@ -1,9 +1,7 @@ -import { config } from '@/config'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -import { baseUrl, getNextBuildId } from './utils'; +import { baseUrl, fetchJson, getNextBuildId } from './utils'; export const route: Route = { path: '/contests', @@ -21,11 +19,7 @@ export const route: Route = { async function handler() { const nextBuildId = await getNextBuildId(); - const response = await ofetch(`${baseUrl}/_next/data/${nextBuildId}/en/contests.json`, { - headers: { - 'User-Agent': config.trueUA, - }, - }); + const response = await fetchJson(`${baseUrl}/_next/data/${nextBuildId}/en/contests.json`); const { listConst, previewList } = response.pageProps; const items = [ diff --git a/lib/routes/makerworld/trending.ts b/lib/routes/makerworld/trending.ts index 8ce6364c55..6c3c2555a8 100644 --- a/lib/routes/makerworld/trending.ts +++ b/lib/routes/makerworld/trending.ts @@ -1,9 +1,7 @@ -import { config } from '@/config'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -import { baseUrl, getNextBuildId } from './utils'; +import { baseUrl, fetchJson, getNextBuildId } from './utils'; export const route: Route = { path: '/trending', @@ -21,20 +19,17 @@ export const route: Route = { async function handler() { const nextBuildId = await getNextBuildId(); - const response = await ofetch(`${baseUrl}/_next/data/${nextBuildId}/en.json`, { - headers: { - 'User-Agent': config.trueUA, - }, - }); + const response = await fetchJson(`${baseUrl}/_next/data/${nextBuildId}/en.json`); - const items = response.pageProps.popularDesignsData.map((d) => ({ - title: d.title, - link: `${baseUrl}/en/models/${d.id}-${d.slug}`, - author: d.designCreator.name, - category: d.tags, - pubDate: parseDate(d.startTime), - description: d.designExtension.design_pictures.map((i) => `
${d.name}
${i.name}
`).join(''), - })); + const items = response.pageProps.v2Props.foryouData.hits + .filter((hit) => hit.design?.title) + .map(({ design: d }) => ({ + title: d.title, + link: `${baseUrl}/en/models/${d.id}-${d.slug}`, + author: d.designCreator.name, + pubDate: parseDate(d.createTime), + description: d.designExtension.design_pictures.map((i) => `
${d.title}
${i.name}
`).join(''), + })); return { title: 'Trending Models - MakerWorld', diff --git a/lib/routes/makerworld/user-upload.ts b/lib/routes/makerworld/user-upload.ts index 2bb0f09a91..65bab95cb9 100644 --- a/lib/routes/makerworld/user-upload.ts +++ b/lib/routes/makerworld/user-upload.ts @@ -1,9 +1,7 @@ -import { config } from '@/config'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -import { baseUrl, getNextBuildId } from './utils'; +import { baseUrl, fetchJson, getNextBuildId } from './utils'; export const route: Route = { path: '/user/:handle/upload', @@ -26,14 +24,7 @@ async function handler(ctx) { const { handle } = ctx.req.param(); const nextBuildId = await getNextBuildId(); - const response = await ofetch(`${baseUrl}/_next/data/${nextBuildId}/en/${handle}/upload.json`, { - headers: { - 'User-Agent': config.trueUA, - }, - query: { - handle, - }, - }); + const response = await fetchJson(`${baseUrl}/_next/data/${nextBuildId}/en/${handle}/upload.json`, { handle }); const { userInfo, designs } = response.pageProps; const items = designs.map((d) => ({ diff --git a/lib/routes/makerworld/utils.ts b/lib/routes/makerworld/utils.ts index 4930e63812..f6b0afb778 100644 --- a/lib/routes/makerworld/utils.ts +++ b/lib/routes/makerworld/utils.ts @@ -1,18 +1,38 @@ import { load } from 'cheerio'; -import { config } from '@/config'; import cache from '@/utils/cache'; -import ofetch from '@/utils/ofetch'; +import { getPlaywrightPage } from '@/utils/playwright'; export const baseUrl = 'https://makerworld.com'; +// makerworld.com sits behind a Cloudflare bot-management challenge that fingerprints the +// TLS/HTTP client itself rather than just headers, so a plain fetch always gets a 403 no +// matter what headers are sent. A real (stealth-patched) browser is required to pass it. +const fetchViaBrowser = async (url: string, type: 'html' | 'json' = 'html') => { + const { page, destroy } = await getPlaywrightPage(url, { + onBeforeLoad: async (page) => { + const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); + await page.route('**/*', (route) => { + const request = route.request(); + expectResourceTypes.has(request.resourceType()) ? route.continue() : route.abort(); + }); + }, + }); + try { + return (await page.evaluate(type === 'html' ? () => document.documentElement.innerHTML : () => document.documentElement.textContent)) ?? ''; + } finally { + await destroy(); + } +}; + +export const fetchJson = async (url: string, query?: Record) => { + const finalUrl = query ? `${url}?${new URLSearchParams(query).toString()}` : url; + return JSON.parse(await fetchViaBrowser(finalUrl, 'json')); +}; + export const getNextBuildId = () => cache.tryGet('makerworld:nextBuildId', async () => { - const response = await ofetch(`${baseUrl}/en`, { - headers: { - 'User-Agent': config.trueUA, - }, - }); + const response = await fetchViaBrowser(`${baseUrl}/en`); const $ = load(response); const nextData = JSON.parse($('script#__NEXT_DATA__').text()); return nextData.buildId;