From 596d1b93cb7b8fbfdd21b5b556c68cacb221f182 Mon Sep 17 00:00:00 2001 From: Bainily Date: Thu, 2 Apr 2026 09:49:04 +0800 Subject: [PATCH] =?UTF-8?q?feat(server):=20add=20proactive=20channel=20pro?= =?UTF-8?q?bes=20and=20load-aware=20stable=20routing=20/=20=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=AB=AF=E6=96=B0=E5=A2=9E=E4=B8=BB=E5=8A=A8=E6=8E=A2?= =?UTF-8?q?=E6=B4=BB=E4=B8=8E=E8=B4=9F=E8=BD=BD=E6=84=9F=E7=9F=A5=E7=A8=B3?= =?UTF-8?q?=E5=AE=9A=E8=B7=AF=E7=94=B1=20(#330)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip: preserve local changes before upstream merge * Improve proxy routing stability and add stable-first strategy * Sync explicit-group routing strategy to eligible source routes * Fix proxy retry and runtime health review issues * fix: make proxy channel retry attempts configurable * fix: harden proxy retry recovery and bookkeeping * fix: address pr 257 review comments * feat(proxy): add sticky session channel coordination * feat(server): add proactive channel probes and load-aware stable routing * test(server): stabilize session load routing selection assertion * fix(proxy): harden codex forwarding stability * fix(router): rotate stable-first channels by healthy order * fix(server): address probe review feedback * fix(server): address probe review follow-ups * fix(ci): address CodeQL and core test failures * feat(server): refine stable-first routing recovery and observation * Finish PR 330 settings and review fixes * Fix chat site endpoint CI mock * Address latest PR 330 review comments --------- Co-authored-by: Cita --- .env.example | 4 + docs/configuration.md | 19 + src/server/config.ts | 4 + src/server/contracts/settingsRoutePayloads.ts | 4 + src/server/index.ts | 17 + .../proxy-core/orchestration/endpointFlow.ts | 233 ++++ .../orchestration/upstreamRequest.ts | 138 +++ .../runtime/codexSessionResponseStore.test.ts | 48 + .../runtime/codexSessionResponseStore.ts | 55 + .../runtime/codexWebsocketRuntime.test.ts | 2 + .../runtime/codexWebsocketRuntime.ts | 82 +- src/server/proxy-core/surfaces/chatSurface.ts | 17 +- .../proxy-core/surfaces/geminiSurface.ts | 11 +- .../surfaces/openAiResponsesSurface.ts | 101 +- .../proxy-core/surfaces/sharedSurface.test.ts | 1 + .../proxy-core/surfaces/sharedSurface.ts | 6 +- .../settings.model-availability-probe.test.ts | 106 ++ src/server/routes/api/settings.ts | 39 + .../routes/api/stats.model-probe.test.ts | 157 +++ src/server/routes/api/stats.ts | 102 +- .../proxy/architecture-boundaries.test.ts | 7 + .../routes/proxy/chat.codex-oauth.test.ts | 1 + .../routes/proxy/chat.count-tokens.test.ts | 1 + .../routes/proxy/chat.siteApiEndpoint.test.ts | 1 + src/server/routes/proxy/chat.stream.test.ts | 117 ++ .../downstreamClientContext.routes.test.ts | 1 + src/server/routes/proxy/endpointFlow.test.ts | 31 + src/server/routes/proxy/endpointFlow.ts | 231 +--- src/server/routes/proxy/images.edits.test.ts | 1 + .../proxy/responses.codex-oauth.test.ts | 135 +++ .../proxy/responses.compact-upstream.test.ts | 1 + .../routes/proxy/responses.websocket.test.ts | 188 ++- src/server/routes/proxy/responsesWebsocket.ts | 22 +- src/server/routes/proxy/search.test.ts | 1 + .../routes/proxy/upstreamEndpoint.test.ts | 72 ++ src/server/routes/proxy/upstreamEndpoint.ts | 28 +- src/server/routes/proxy/upstreamError.ts | 82 +- src/server/routes/proxy/upstreamUrl.ts | 59 +- src/server/routes/proxy/videos.test.ts | 1 + src/server/services/backgroundTaskService.ts | 15 + .../channelRecoveryProbeService.test.ts | 348 ++++++ .../services/channelRecoveryProbeService.ts | 299 +++++ .../modelAvailabilityProbeService.test.ts | 507 ++++++++ .../services/modelAvailabilityProbeService.ts | 447 +++++++ .../services/proxyChannelCoordinator.test.ts | 67 ++ .../services/proxyChannelCoordinator.ts | 75 ++ src/server/services/proxyDebugTraceRuntime.ts | 2 +- src/server/services/proxyRetryPolicy.test.ts | 14 +- src/server/services/proxyRetryPolicy.ts | 25 + src/server/services/runtimeDispatch.ts | 1 + src/server/services/runtimeModelProbe.test.ts | 116 ++ src/server/services/runtimeModelProbe.ts | 250 ++++ src/server/services/tokenRouter.cache.test.ts | 86 +- .../services/tokenRouter.selection.test.ts | 545 ++++++++- src/server/services/tokenRouter.test.ts | 4 +- src/server/services/tokenRouter.ts | 1025 +++++++++++++++-- .../services/upstreamEndpointRuntime.ts | 6 + .../services/upstreamEndpointRuntimeMemory.ts | 7 + .../anthropic/messages/compatibility.ts | 52 + .../anthropic/messages/conversion.test.ts | 24 + .../anthropic/messages/conversion.ts | 49 +- .../anthropic/messages/inbound.test.ts | 60 + .../openai/responses/continuation.test.ts | 125 ++ .../openai/responses/continuation.ts | 143 +++ .../transformers/shared/chatFormatsCore.ts | 6 + src/web/api.ts | 1 + src/web/i18n.supplement.ts | 8 + src/web/i18n.tsx | 8 + src/web/pages/Settings.tsx | 98 ++ src/web/pages/TokenRoutes.tsx | 10 +- ...settings.model-availability-probe.test.tsx | 157 +++ .../ModelAvailabilityProbeConfirmModal.tsx | 105 ++ src/web/pages/token-routes/RouteCard.tsx | 24 +- src/web/pages/token-routes/routingStrategy.ts | 22 + .../tokenRoutes.routing-strategy.test.tsx | 15 +- 75 files changed, 6365 insertions(+), 507 deletions(-) create mode 100644 src/server/proxy-core/orchestration/endpointFlow.ts create mode 100644 src/server/proxy-core/orchestration/upstreamRequest.ts create mode 100644 src/server/proxy-core/runtime/codexSessionResponseStore.test.ts create mode 100644 src/server/proxy-core/runtime/codexSessionResponseStore.ts create mode 100644 src/server/routes/api/settings.model-availability-probe.test.ts create mode 100644 src/server/routes/api/stats.model-probe.test.ts create mode 100644 src/server/services/channelRecoveryProbeService.test.ts create mode 100644 src/server/services/channelRecoveryProbeService.ts create mode 100644 src/server/services/modelAvailabilityProbeService.test.ts create mode 100644 src/server/services/modelAvailabilityProbeService.ts create mode 100644 src/server/services/runtimeDispatch.ts create mode 100644 src/server/services/runtimeModelProbe.test.ts create mode 100644 src/server/services/runtimeModelProbe.ts create mode 100644 src/server/services/upstreamEndpointRuntime.ts create mode 100644 src/server/transformers/openai/responses/continuation.test.ts create mode 100644 src/server/transformers/openai/responses/continuation.ts create mode 100644 src/web/pages/settings.model-availability-probe.test.tsx create mode 100644 src/web/pages/settings/ModelAvailabilityProbeConfirmModal.tsx diff --git a/.env.example b/.env.example index 029ad7c..a6484ce 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,10 @@ SERVERCHAN_KEY= TELEGRAM_ENABLED=false TELEGRAM_BOT_TOKEN= TELEGRAM_CHAT_ID= +MODEL_AVAILABILITY_PROBE_CONCURRENCY=1 +MODEL_AVAILABILITY_PROBE_ENABLED=false +MODEL_AVAILABILITY_PROBE_INTERVAL_MS=1800000 +MODEL_AVAILABILITY_PROBE_TIMEOUT_MS=15000 PORT=4000 DATA_DIR=./data NOTIFY_COOLDOWN_SEC=300 diff --git a/docs/configuration.md b/docs/configuration.md index 28a0776..12bb653 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -128,6 +128,25 @@ Metapi 的路由引擎按多因子加权选择最优通道。 |--------|------|--------|------| | `TOKEN_ROUTER_CACHE_TTL_MS` | Token 路由缓存 TTL（毫秒） | `1500` | >= 100 | +### 模型弹活探测 + +用于异步校正“上游 `/models` 能看到，但实际调用不可用”的假阳性模型。 + +| 变量名 | 说明 | 默认值 | 范围 | +|--------|------|--------|------| +| `MODEL_AVAILABILITY_PROBE_ENABLED` | 启用后台模型可用性探测 | `false` | `true` / `false` | +| `MODEL_AVAILABILITY_PROBE_INTERVAL_MS` | 后台定时探测间隔（毫秒） | `1800000` | >= 60000 | +| `MODEL_AVAILABILITY_PROBE_TIMEOUT_MS` | 单次探测超时（毫秒） | `15000` | >= 3000 | +| `MODEL_AVAILABILITY_PROBE_CONCURRENCY` | 单账号模型探测并发数，默认串行避免对上游形成批量测活 | `1` | 1 ~ 16 | + +说明： + +- 探测任务在后台异步执行，不阻塞前台请求。 +- 只对看起来像对话/推理模型的条目发最小化探测请求，明显的 embedding、rerank、moderation、tts、image/video 等模型会先跳过。 +- 只有“明确模型不存在 / 不支持 / 无权限”的错误才会写回不可用；超时、5xx、网络抖动会归为“不确定”，不会误杀现有模型。 +- 为了避免对上游形成意外的批量测活，默认关闭；只有显式开启后才会按间隔执行，且不会在服务启动后立刻扫全量模型。 +- 同一批探测任务带去重，上一轮未结束时不会并发堆积；默认并发为 `1`，建议只在确认上游允许主动探测时启用。 + ### 路由预设建议 | 场景 | COST_WEIGHT | BALANCE_WEIGHT | USAGE_WEIGHT | 说明 | diff --git a/src/server/config.ts b/src/server/config.ts index 36ff57f..18fcbbc 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -140,6 +140,10 @@ export function buildConfig(env: NodeJS.ProcessEnv) { proxyDebugTargetModel: (env.PROXY_DEBUG_TARGET_MODEL || '').trim(), proxyDebugRetentionHours: Math.max(1, Math.trunc(parseNumber(env.PROXY_DEBUG_RETENTION_HOURS, 24))), proxyDebugMaxBodyBytes: Math.max(1024, Math.trunc(parseNumber(env.PROXY_DEBUG_MAX_BODY_BYTES, 262_144))), + modelAvailabilityProbeEnabled: parseBoolean(env.MODEL_AVAILABILITY_PROBE_ENABLED, false), + modelAvailabilityProbeIntervalMs: Math.max(60_000, Math.trunc(parseNumber(env.MODEL_AVAILABILITY_PROBE_INTERVAL_MS, 30 * 60 * 1000))), + modelAvailabilityProbeTimeoutMs: Math.max(3_000, Math.trunc(parseNumber(env.MODEL_AVAILABILITY_PROBE_TIMEOUT_MS, 15_000))), + modelAvailabilityProbeConcurrency: Math.max(1, Math.min(16, Math.trunc(parseNumber(env.MODEL_AVAILABILITY_PROBE_CONCURRENCY, 1)))), proxyLogRetentionDays: Math.max(0, Math.trunc(parseNumber(env.PROXY_LOG_RETENTION_DAYS, 30))), proxyLogRetentionPruneIntervalMinutes: Math.max(1, Math.trunc(parseNumber(env.PROXY_LOG_RETENTION_PRUNE_INTERVAL_MINUTES, 30))), proxyFileRetentionDays: Math.max(0, Math.trunc(parseNumber(env.PROXY_FILE_RETENTION_DAYS, 30))), diff --git a/src/server/contracts/settingsRoutePayloads.ts b/src/server/contracts/settingsRoutePayloads.ts index df6ff08..bfcb8c5 100644 --- a/src/server/contracts/settingsRoutePayloads.ts +++ b/src/server/contracts/settingsRoutePayloads.ts @@ -4,6 +4,7 @@ const backupExportTypeSchema = z.enum(['all', 'accounts', 'preferences']); const migrationDialectSchema = z.enum(['sqlite', 'mysql', 'postgres']); const runtimeSettingsPayloadSchema = z.object({ + modelAvailabilityProbeEnabled: z.boolean().optional(), webhookEnabled: z.boolean().optional(), barkEnabled: z.boolean().optional(), serverChanEnabled: z.boolean().optional(), @@ -86,6 +87,9 @@ function formatSettingsPayloadError(error: z.ZodError): string { if (firstPath === 'webhookEnabled') { return 'Webhook 开关格式无效：需要 boolean'; } + if (firstPath === 'modelAvailabilityProbeEnabled') { + return '批量测活开关格式无效：需要 boolean'; + } if (firstPath === 'barkEnabled') { return 'Bark 开关格式无效：需要 boolean'; } diff --git a/src/server/index.ts b/src/server/index.ts index 46dbaae..1c084b7 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -38,6 +38,14 @@ import { ensureOauthIdentityBackfill } from './services/oauth/oauthIdentityBackf import { ensureOauthProviderSitesExist } from './services/oauth/oauthSiteRegistry.js'; import { startOAuthLoopbackCallbackServers, stopOAuthLoopbackCallbackServers } from './services/oauth/localCallbackServer.js'; import { startSiteAnnouncementPolling, stopSiteAnnouncementPolling } from './services/siteAnnouncementPollingService.js'; +import { + startModelAvailabilityProbeScheduler, + stopModelAvailabilityProbeScheduler, +} from './services/modelAvailabilityProbeService.js'; +import { + startChannelRecoveryProbeScheduler, + stopChannelRecoveryProbeScheduler, +} from './services/channelRecoveryProbeService.js'; import { startUpdateCenterPolling, stopUpdateCenterPolling } from './services/updateCenterPollingService.js'; import { reloadBackupWebdavScheduler } from './services/backupService.js'; import { ensureRuntimeDatabaseReady } from './runtimeDatabaseBootstrap.js'; @@ -144,6 +152,11 @@ function applyRuntimeSettings(settingsMap: Map) { const systemProxyUrl = parseSettingFromMap(settingsMap, 'system_proxy_url'); if (typeof systemProxyUrl === 'string') config.systemProxyUrl = systemProxyUrl; + const modelAvailabilityProbeEnabled = parseSettingFromMap(settingsMap, 'model_availability_probe_enabled'); + if (typeof modelAvailabilityProbeEnabled === 'boolean') { + config.modelAvailabilityProbeEnabled = modelAvailabilityProbeEnabled; + } + const codexUpstreamWebsocketEnabled = parseSettingFromMap(settingsMap, 'codex_upstream_websocket_enabled'); if (typeof codexUpstreamWebsocketEnabled === 'boolean') { config.codexUpstreamWebsocketEnabled = codexUpstreamWebsocketEnabled; @@ -494,6 +507,8 @@ if (existsSync(webDir)) { await startScheduler(); await reloadBackupWebdavScheduler(); startSiteAnnouncementPolling(); +startModelAvailabilityProbeScheduler(); +startChannelRecoveryProbeScheduler(); startUpdateCenterPolling(); try { await startOAuthLoopbackCallbackServers(); @@ -507,6 +522,8 @@ app.addHook('onClose', async () => { stopUpdateCenterPolling(); stopProxyFileRetentionService(); stopProxyLogRetentionService(); + stopModelAvailabilityProbeScheduler(); + stopChannelRecoveryProbeScheduler(); await stopOAuthLoopbackCallbackServers(); }); diff --git a/src/server/proxy-core/orchestration/endpointFlow.ts b/src/server/proxy-core/orchestration/endpointFlow.ts new file mode 100644 index 0000000..30cad2e --- /dev/null +++ b/src/server/proxy-core/orchestration/endpointFlow.ts @@ -0,0 +1,233 @@ +import { fetch } from 'undici'; +import { readRuntimeResponseText } from '../executors/types.js'; +import { withSiteProxyRequestInit } from '../../services/siteProxy.js'; +import { + buildUpstreamUrl, + summarizeUpstreamError, + type UpstreamEndpoint, +} from './upstreamRequest.js'; + +export type BuiltEndpointRequest = { + endpoint: UpstreamEndpoint; + path: string; + headers: Record; + body: Record; + runtime?: { + executor: 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude'; + modelName?: string; + stream?: boolean; + oauthProjectId?: string | null; + action?: 'generateContent' | 'streamGenerateContent' | 'countTokens'; + }; +}; + +export type EndpointAttemptContext = { + endpointIndex: number; + endpointCount: number; + request: BuiltEndpointRequest; + targetUrl: string; + response: Awaited>; + rawErrText: string; + recoverApplied?: boolean; +}; + +export type EndpointAttemptSuccessContext = { + endpointIndex: number; + endpointCount: number; + request: BuiltEndpointRequest; + targetUrl: string; + response: Awaited>; + recoverApplied?: boolean; +}; + +export type EndpointRecoverResult = { + upstream: Awaited>; + upstreamPath: string; + request?: BuiltEndpointRequest; + targetUrl?: string; +} | null; + +export type EndpointFlowResult = + | { + ok: true; + upstream: Awaited>; + upstreamPath: string; + } + | { + ok: false; + status: number; + errText: string; + rawErrText?: string; + }; + +export type ExecuteEndpointFlowInput = { + siteUrl: string; + proxyUrl?: string | null; + disableCrossProtocolFallback?: boolean; + endpointCandidates: UpstreamEndpoint[]; + buildRequest: (endpoint: UpstreamEndpoint, endpointIndex: number) => BuiltEndpointRequest; + dispatchRequest?: ( + request: BuiltEndpointRequest, + targetUrl: string, + ) => Promise>>; + tryRecover?: (ctx: EndpointAttemptContext) => Promise; + shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean; + shouldAbortRemainingEndpoints?: (ctx: EndpointAttemptContext & { errText: string }) => boolean; + onDowngrade?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise; + onAttemptFailure?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise; + onAttemptSuccess?: (ctx: EndpointAttemptSuccessContext) => void | Promise; +}; + +export function withUpstreamPath(path: string, message: string): string { + return `[upstream:${path}] ${message}`; +} + +async function runEndpointFlowHook( + hook: ((ctx: T) => void | Promise) | undefined, + ctx: T, + hookName: string, +): Promise { + if (!hook) return; + try { + await hook(ctx); + } catch (error) { + console.error(`endpointFlow ${hookName} hook failed`, error); + } +} + +export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Promise { + const endpointCount = input.endpointCandidates.length; + if (endpointCount <= 0) { + return { + ok: false, + status: 502, + errText: 'Upstream request failed', + }; + } + + let finalStatus = 0; + let finalErrText = 'unknown error'; + let finalRawErrText: string | undefined; + + for (let endpointIndex = 0; endpointIndex < endpointCount; endpointIndex += 1) { + const endpoint = input.endpointCandidates[endpointIndex] as UpstreamEndpoint; + const request = input.buildRequest(endpoint, endpointIndex); + const defaultTarget = buildUpstreamUrl(input.siteUrl, request.path); + const targetUrl = input.proxyUrl + ? buildUpstreamUrl(input.proxyUrl, request.path) + : defaultTarget; + + let response = input.dispatchRequest + ? await input.dispatchRequest(request, targetUrl) + : await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, { + method: 'POST', + headers: request.headers, + body: JSON.stringify(request.body), + })); + + if (response.ok) { + await runEndpointFlowHook(input.onAttemptSuccess, { + endpointIndex, + endpointCount, + request, + targetUrl, + response, + recoverApplied: false, + }, 'onAttemptSuccess'); + return { + ok: true, + upstream: response, + upstreamPath: request.path, + }; + } + + let rawErrText = await readRuntimeResponseText(response).catch(() => 'unknown error'); + const baseContext: EndpointAttemptContext = { + endpointIndex, + endpointCount, + request, + targetUrl, + response, + rawErrText, + recoverApplied: false, + }; + + if (input.tryRecover) { + const recovered = await input.tryRecover(baseContext); + baseContext.recoverApplied = recovered !== null + || baseContext.request !== request + || baseContext.response !== response + || baseContext.rawErrText !== rawErrText; + if (recovered?.upstream?.ok) { + const recoveredRequest = recovered.request ?? baseContext.request; + const recoveredTargetUrl = recovered.targetUrl ?? ( + input.proxyUrl + ? buildUpstreamUrl(input.proxyUrl, recovered.upstreamPath) + : buildUpstreamUrl(input.siteUrl, recovered.upstreamPath) + ); + await runEndpointFlowHook(input.onAttemptSuccess, { + endpointIndex, + endpointCount, + request: recoveredRequest, + targetUrl: recoveredTargetUrl, + response: recovered.upstream, + recoverApplied: true, + }, 'onAttemptSuccess'); + return { + ok: true, + upstream: recovered.upstream, + upstreamPath: recovered.upstreamPath, + }; + } + } + + rawErrText = baseContext.rawErrText; + response = baseContext.response; + const errText = withUpstreamPath( + baseContext.request.path, + summarizeUpstreamError(response.status, rawErrText), + ); + await runEndpointFlowHook(input.onAttemptFailure, { + ...baseContext, + errText, + }, 'onAttemptFailure'); + + const isLastEndpoint = endpointIndex >= endpointCount - 1; + if (input.disableCrossProtocolFallback && !isLastEndpoint) { + finalStatus = response.status; + finalErrText = errText; + finalRawErrText = rawErrText; + break; + } + const shouldAbortRemainingEndpoints = !isLastEndpoint && !!input.shouldAbortRemainingEndpoints?.({ + ...baseContext, + errText, + }); + if (shouldAbortRemainingEndpoints) { + finalStatus = response.status; + finalErrText = errText; + finalRawErrText = rawErrText; + break; + } + const shouldDowngrade = !isLastEndpoint && !!input.shouldDowngrade?.(baseContext); + if (shouldDowngrade) { + await runEndpointFlowHook(input.onDowngrade, { + ...baseContext, + errText, + }, 'onDowngrade'); + continue; + } + + finalStatus = response.status; + finalErrText = errText; + finalRawErrText = rawErrText; + break; + } + + return { + ok: false, + status: finalStatus || 502, + errText: finalErrText || 'unknown error', + rawErrText: finalRawErrText, + }; +} diff --git a/src/server/proxy-core/orchestration/upstreamRequest.ts b/src/server/proxy-core/orchestration/upstreamRequest.ts new file mode 100644 index 0000000..50ca29d --- /dev/null +++ b/src/server/proxy-core/orchestration/upstreamRequest.ts @@ -0,0 +1,138 @@ +export type UpstreamEndpoint = 'chat' | 'messages' | 'responses'; + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object'; +} + +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} + +function extractJsonErrorMessage(rawText: string): string { + try { + const parsed = JSON.parse(rawText) as unknown; + const root = isRecord(parsed) ? parsed : null; + const error = (root && isRecord(root.error)) ? root.error : root; + if (!error) return ''; + + const message = typeof error.message === 'string' ? collapseWhitespace(error.message) : ''; + if (message) return message; + + const code = typeof error.code === 'string' ? collapseWhitespace(error.code) : ''; + const type = typeof error.type === 'string' ? collapseWhitespace(error.type) : ''; + return [type, code].filter((part) => part.length > 0).join('/'); + } catch { + return ''; + } +} + +function extractHtmlTitle(rawText: string): string { + const match = rawText.match(/]*>([^<>]*)<\/title>/i); + if (!match?.[1]) return ''; + return collapseWhitespace(match[1]); +} + +function extractCloudflareHtmlSummary(rawText: string, status: number): string { + if (!/cloudflare/i.test(rawText)) return ''; + const title = extractHtmlTitle(rawText); + const codeMatch = ( + title.match(/\b(\d{3,4})\s*:\s*([^\|<]+)/i) + || rawText.match(/Error code\s*(\d{3,4})/i) + ); + const code = codeMatch?.[1] || (status > 0 ? String(status) : ''); + const reason = collapseWhitespace( + (typeof codeMatch?.[2] === 'string' ? codeMatch[2] : '') + || (status >= 500 ? 'origin host error' : 'request blocked') + ); + if (code) return `Cloudflare ${code}: ${reason}`; + return `Cloudflare: ${reason}`; +} + +function extractHtmlSummary(rawText: string, status: number): string { + if (!/(]*>([^<>]*)<\/h1>/i)?.[1] || ''; + return collapseWhitespace(heading); +} + +function formatUrlOrigin(url: URL): string { + const username = url.username ? encodeURIComponent(url.username) : ''; + const password = url.password ? encodeURIComponent(url.password) : ''; + const auth = username + ? `${username}${password ? `:${password}` : ''}@` + : ''; + + return `${url.protocol}//${auth}${url.host}`; +} + +function joinPath(basePath: string, requestPath: string): string { + const base = basePath.replace(/\/+$/, ''); + const path = requestPath.startsWith('/') ? requestPath : `/${requestPath}`; + + if (!base || base === '/') return path || '/'; + if (!path || path === '/') return base; + return `${base}${path}`; +} + +export function summarizeUpstreamError(status: number, rawErrorText: string): string { + const statusPrefix = status > 0 + ? `Upstream returned HTTP ${status}` + : 'Upstream request failed'; + + const raw = typeof rawErrorText === 'string' ? rawErrorText.trim() : ''; + if (!raw) return statusPrefix; + + const jsonMessage = extractJsonErrorMessage(raw); + if (jsonMessage) return `${statusPrefix}: ${jsonMessage}`; + + const htmlMessage = extractHtmlSummary(raw, status); + if (htmlMessage) return `${statusPrefix}: ${htmlMessage}`; + + const compact = collapseWhitespace(raw); + if (!compact) return statusPrefix; + if (compact.length <= 400) return `${statusPrefix}: ${compact}`; + return `${statusPrefix}: ${compact.slice(0, 400)}...(truncated)`; +} + +export function buildUpstreamUrl(siteUrl: string, requestPath: string): string { + const baseRaw = typeof siteUrl === 'string' ? siteUrl.trim() : ''; + const pathRaw = typeof requestPath === 'string' ? requestPath.trim() : ''; + const fallbackBase = baseRaw.replace(/\/+$/, ''); + let path = pathRaw.startsWith('/') ? pathRaw : `/${pathRaw}`; + + if (!fallbackBase) return path || '/'; + if (!path || path === '/') return fallbackBase; + + try { + const parsed = new URL(baseRaw); + const basePath = parsed.pathname.replace(/\/+$/, ''); + const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(basePath); + if (baseHasVersionSuffix) { + if (path === '/v1') { + path = '/'; + } else if (path.startsWith('/v1/')) { + path = path.slice('/v1'.length) || '/'; + } + } + + const joinedPath = joinPath(basePath, path); + return `${formatUrlOrigin(parsed)}${joinedPath}${parsed.search}${parsed.hash}`; + } catch { + const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(fallbackBase); + if (baseHasVersionSuffix) { + if (path === '/v1') { + path = '/'; + } else if (path.startsWith('/v1/')) { + path = path.slice('/v1'.length) || '/'; + } + } + + return `${fallbackBase}${path}`; + } +} diff --git a/src/server/proxy-core/runtime/codexSessionResponseStore.test.ts b/src/server/proxy-core/runtime/codexSessionResponseStore.test.ts new file mode 100644 index 0000000..5b1ce24 --- /dev/null +++ b/src/server/proxy-core/runtime/codexSessionResponseStore.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { + buildCodexSessionResponseStoreKey, + getCodexSessionResponseId, + resetCodexSessionResponseStore, + setCodexSessionResponseId, +} from './codexSessionResponseStore.js'; + +describe('codexSessionResponseStore', () => { + it('evicts the oldest session id when the store exceeds the cap', () => { + resetCodexSessionResponseStore(); + + for (let index = 0; index <= 10_000; index += 1) { + setCodexSessionResponseId(`session-${index}`, `resp-${index}`); + } + + expect(getCodexSessionResponseId('session-0')).toBeNull(); + expect(getCodexSessionResponseId('session-1')).toBe('resp-1'); + expect(getCodexSessionResponseId('session-10000')).toBe('resp-10000'); + + resetCodexSessionResponseStore(); + }); + + it('namespaces identical downstream session ids by channel scope', () => { + resetCodexSessionResponseStore(); + + const keyA = buildCodexSessionResponseStoreKey({ + sessionId: 'session-1', + siteId: 10, + accountId: 20, + channelId: 30, + }); + const keyB = buildCodexSessionResponseStoreKey({ + sessionId: 'session-1', + siteId: 10, + accountId: 21, + channelId: 31, + }); + + setCodexSessionResponseId(keyA, 'resp-a'); + setCodexSessionResponseId(keyB, 'resp-b'); + + expect(getCodexSessionResponseId(keyA)).toBe('resp-a'); + expect(getCodexSessionResponseId(keyB)).toBe('resp-b'); + + resetCodexSessionResponseStore(); + }); +}); diff --git a/src/server/proxy-core/runtime/codexSessionResponseStore.ts b/src/server/proxy-core/runtime/codexSessionResponseStore.ts new file mode 100644 index 0000000..9863b54 --- /dev/null +++ b/src/server/proxy-core/runtime/codexSessionResponseStore.ts @@ -0,0 +1,55 @@ +const MAX_CODEX_SESSION_RESPONSE_IDS = 10_000; + +const codexSessionResponseIds = new Map(); + +function normalizeSessionId(sessionId: string): string { + return sessionId.trim(); +} + +export function buildCodexSessionResponseStoreKey(input: { + sessionId: string; + siteId?: number | null; + accountId?: number | null; + channelId?: number | null; +}): string { + const normalizedSessionId = normalizeSessionId(input.sessionId); + if (!normalizedSessionId) return ''; + const parts = [ + Number.isFinite(input.siteId as number) && Number(input.siteId) > 0 ? `site:${Math.trunc(Number(input.siteId))}` : '', + Number.isFinite(input.accountId as number) && Number(input.accountId) > 0 ? `account:${Math.trunc(Number(input.accountId))}` : '', + Number.isFinite(input.channelId as number) && Number(input.channelId) > 0 ? `channel:${Math.trunc(Number(input.channelId))}` : '', + `session:${normalizedSessionId}`, + ].filter(Boolean); + return parts.join('|'); +} + +export function getCodexSessionResponseId(sessionId: string): string | null { + const normalized = normalizeSessionId(sessionId); + if (!normalized) return null; + return codexSessionResponseIds.get(normalized) ?? null; +} + +export function setCodexSessionResponseId(sessionId: string, responseId: string): void { + const normalizedSessionId = normalizeSessionId(sessionId); + const normalizedResponseId = responseId.trim(); + if (!normalizedSessionId || !normalizedResponseId) return; + if (codexSessionResponseIds.has(normalizedSessionId)) { + codexSessionResponseIds.delete(normalizedSessionId); + } + codexSessionResponseIds.set(normalizedSessionId, normalizedResponseId); + while (codexSessionResponseIds.size > MAX_CODEX_SESSION_RESPONSE_IDS) { + const oldestKey = codexSessionResponseIds.keys().next().value; + if (!oldestKey) break; + codexSessionResponseIds.delete(oldestKey); + } +} + +export function clearCodexSessionResponseId(sessionId: string): void { + const normalized = normalizeSessionId(sessionId); + if (!normalized) return; + codexSessionResponseIds.delete(normalized); +} + +export function resetCodexSessionResponseStore(): void { + codexSessionResponseIds.clear(); +} diff --git a/src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts b/src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts index 5630e97..26a9569 100644 --- a/src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts +++ b/src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts @@ -1,6 +1,7 @@ import { AddressInfo } from 'node:net'; import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { WebSocketServer } from 'ws'; +import { resetCodexSessionResponseStore } from './codexSessionResponseStore.js'; describe('codexWebsocketRuntime', () => { let upstreamServer: WebSocketServer; @@ -25,6 +26,7 @@ describe('codexWebsocketRuntime', () => { }); beforeEach(() => { + resetCodexSessionResponseStore(); upstreamConnectionCount = 0; upstreamRequests = []; upstreamMessageHandler = (socket, parsed, requestIndex) => { diff --git a/src/server/proxy-core/runtime/codexWebsocketRuntime.ts b/src/server/proxy-core/runtime/codexWebsocketRuntime.ts index 4e1eba0..b861b4c 100644 --- a/src/server/proxy-core/runtime/codexWebsocketRuntime.ts +++ b/src/server/proxy-core/runtime/codexWebsocketRuntime.ts @@ -1,10 +1,22 @@ import type { IncomingMessage } from 'node:http'; import WebSocket from 'ws'; +import { + extractResponsesTerminalResponseId, + isResponsesPreviousResponseNotFoundError, + shouldInferResponsesPreviousResponseId, + stripResponsesPreviousResponseId, + withResponsesPreviousResponseId, +} from '../../transformers/openai/responses/continuation.js'; import { buildCodexWebsocketHandshakeHeaders, buildCodexWebsocketRequestBody, toCodexWebsocketUrl, } from './codexWebsocketHeaders.js'; +import { + clearCodexSessionResponseId, + getCodexSessionResponseId, + setCodexSessionResponseId, +} from './codexSessionResponseStore.js'; import { createCodexWebsocketSessionStore } from './codexWebsocketSessionStore.js'; import type { CodexWebsocketRuntimeResult, @@ -190,6 +202,23 @@ function clearSessionSocket(session: CodexWebsocketSession, socket: WebSocket): session.socketUrl = null; } +function buildContinuationAwareRuntimeBody( + sessionId: string, + body: Record, +): Record { + const rememberedResponseId = getCodexSessionResponseId(sessionId); + if (!shouldInferResponsesPreviousResponseId(body, rememberedResponseId)) { + return body; + } + return withResponsesPreviousResponseId(body, rememberedResponseId); +} + +function rememberSessionResponseId(sessionId: string, payload: unknown): void { + const responseId = extractResponsesTerminalResponseId(payload); + if (!responseId) return; + setCodexSessionResponseId(sessionId, responseId); +} + async function ensureSessionSocket( session: CodexWebsocketSession, input: CodexWebsocketRuntimeSendInput, @@ -232,9 +261,11 @@ async function ensureSessionSocket( }; } -async function sendSessionRequest( +async function sendSessionRequestAttempt( session: CodexWebsocketSession, - input: CodexWebsocketRuntimeSendInput, + input: CodexWebsocketRuntimeSendInput & { + body: Record; + }, ): Promise { const { socket, reusedSession } = await ensureSessionSocket(session, input); const events: Array> = []; @@ -262,7 +293,13 @@ async function sendSessionRequest( events.push(parsed); if (!isTerminalEvent(parsed)) return; if (settled) return; - if (isRuntimeErrorEvent(parsed)) { + if ( + isRuntimeErrorEvent(parsed) + || isResponsesPreviousResponseNotFoundError({ + payload: parsed, + rawErrText: extractTerminalErrorMessage(parsed), + }) + ) { settled = true; cleanup(); clearSessionSocket(session, socket); @@ -274,6 +311,7 @@ async function sendSessionRequest( })); return; } + rememberSessionResponseId(session.sessionId, parsed); settled = true; cleanup(); resolve({ @@ -307,6 +345,43 @@ async function sendSessionRequest( }); } +async function sendSessionRequest( + session: CodexWebsocketSession, + input: CodexWebsocketRuntimeSendInput, +): Promise { + let currentBody = buildContinuationAwareRuntimeBody(session.sessionId, input.body); + let previousResponseRecoveryTried = false; + + for (;;) { + try { + return await sendSessionRequestAttempt(session, { + ...input, + body: currentBody, + }); + } catch (error) { + if ( + previousResponseRecoveryTried + || !(error instanceof CodexWebsocketRuntimeError) + || !isResponsesPreviousResponseNotFoundError({ + payload: error.payload ?? error.events[error.events.length - 1], + rawErrText: error.message, + }) + ) { + throw error; + } + + const previousResponseRecovery = stripResponsesPreviousResponseId(currentBody); + if (!previousResponseRecovery.removed) { + throw error; + } + + previousResponseRecoveryTried = true; + clearCodexSessionResponseId(session.sessionId); + currentBody = previousResponseRecovery.body; + } + } +} + export function createCodexWebsocketRuntime(input?: { sessionStore?: CodexWebsocketSessionStore; }) { @@ -334,6 +409,7 @@ export function createCodexWebsocketRuntime(input?: { await closeSocket(session.socket); session.socket = null; session.socketUrl = null; + clearCodexSessionResponseId(sessionId); }, async closeAllSessions(): Promise { diff --git a/src/server/proxy-core/surfaces/chatSurface.ts b/src/server/proxy-core/surfaces/chatSurface.ts index 5b289f3..6d198b0 100644 --- a/src/server/proxy-core/surfaces/chatSurface.ts +++ b/src/server/proxy-core/surfaces/chatSurface.ts @@ -9,7 +9,7 @@ import { buildClaudeCountTokensUpstreamRequest, buildUpstreamEndpointRequest, resolveUpstreamEndpointCandidates, -} from '../../routes/proxy/upstreamEndpoint.js'; +} from '../../services/upstreamEndpointRuntime.js'; import { getUpstreamEndpointRuntimeStateSnapshot, recordUpstreamEndpointFailure, @@ -20,10 +20,11 @@ import { getDownstreamRoutingPolicy, recordDownstreamCostUsage, } from '../../routes/proxy/downstreamPolicy.js'; -import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js'; +import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js'; import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js'; import { openAiChatTransformer } from '../../transformers/openai/chat/index.js'; import { anthropicMessagesTransformer } from '../../transformers/anthropic/messages/index.js'; +import { shouldPreferResponsesForAnthropicContinuation } from '../../transformers/anthropic/messages/compatibility.js'; import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js'; import { ProxyInputFileResolutionError, @@ -47,6 +48,7 @@ import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/con import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js'; import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js'; import { acquireSurfaceChannelLease, bindSurfaceStickyChannel, @@ -170,6 +172,10 @@ export async function handleChatSurfaceRequest( } const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(resolvedOpenAiBody); const hasNonImageFileInput = conversationFileSummary.hasDocument; + const wantsContinuationAwareResponses = ( + downstreamFormat === 'claude' + && shouldPreferResponsesForAnthropicContinuation(claudeOriginalBody) + ); const codexSessionCacheKey = deriveCodexSessionCacheKey({ downstreamFormat, body: downstreamFormat === 'claude' ? claudeOriginalBody : request.body, @@ -283,6 +289,7 @@ export async function handleChatSurfaceRequest( { hasNonImageFileInput, conversationFileSummary, + wantsContinuationAwareResponses, }, ), ]; @@ -297,6 +304,7 @@ export async function handleChatSurfaceRequest( requestCapabilities: { hasNonImageFileInput, conversationFileSummary, + wantsContinuationAwareResponses, }, }; await safeUpdateSurfaceProxyDebugCandidates(debugTrace, { @@ -309,6 +317,7 @@ export async function handleChatSurfaceRequest( stickyPreferredChannelId, oauthProvider: oauth?.provider || null, isCodexSite, + wantsContinuationAwareResponses, }, }); const buildProviderHeaders = () => ( @@ -389,6 +398,10 @@ export async function handleChatSurfaceRequest( buildRequest: (endpoint) => buildEndpointRequest(endpoint), dispatchRequest, tryRecover, + shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback( + ctx.response.status, + ctx.rawErrText || ctx.errText, + ), onAttemptFailure: async (ctx) => { const memoryWrite = recordUpstreamEndpointFailure({ ...endpointRuntimeContext, diff --git a/src/server/proxy-core/surfaces/geminiSurface.ts b/src/server/proxy-core/surfaces/geminiSurface.ts index 7dc1a54..4addfdf 100644 --- a/src/server/proxy-core/surfaces/geminiSurface.ts +++ b/src/server/proxy-core/surfaces/geminiSurface.ts @@ -14,12 +14,12 @@ import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refres import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js'; import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js'; import { getDownstreamRoutingPolicy } from '../../routes/proxy/downstreamPolicy.js'; -import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js'; +import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js'; import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js'; import { buildUpstreamEndpointRequest, resolveUpstreamEndpointCandidates, -} from '../../routes/proxy/upstreamEndpoint.js'; +} from '../../services/upstreamEndpointRuntime.js'; import { getUpstreamEndpointRuntimeStateSnapshot, recordUpstreamEndpointFailure, @@ -35,12 +35,13 @@ import { unwrapGeminiCliPayload, wrapGeminiCliRequest, } from '../../routes/proxy/geminiCliCompat.js'; -import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js'; +import { dispatchRuntimeRequest } from '../../services/runtimeDispatch.js'; import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js'; import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js'; import { buildSurfaceProxyDebugResponseHeaders, captureSurfaceProxyDebugSuccessResponseBody, @@ -1224,6 +1225,10 @@ export async function geminiProxyRoute(app: FastifyInstance) { buildRequest: (endpoint) => buildEndpointRequest(endpoint), dispatchRequest, tryRecover: endpointStrategy.tryRecover, + shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback( + ctx.response.status, + ctx.rawErrText || ctx.errText, + ), onAttemptFailure: async (ctx) => { const memoryWrite = recordUpstreamEndpointFailure({ ...endpointRuntimeContext, diff --git a/src/server/proxy-core/surfaces/openAiResponsesSurface.ts b/src/server/proxy-core/surfaces/openAiResponsesSurface.ts index 31e88fe..72df039 100644 --- a/src/server/proxy-core/surfaces/openAiResponsesSurface.ts +++ b/src/server/proxy-core/surfaces/openAiResponsesSurface.ts @@ -4,17 +4,24 @@ import { config } from '../../config.js'; import { reportProxyAllFailed } from '../../services/alertService.js'; import { hasProxyUsagePayload, mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js'; import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js'; +import { + extractResponsesTerminalResponseId, + isResponsesPreviousResponseNotFoundError, + shouldInferResponsesPreviousResponseId, + stripResponsesPreviousResponseId, + withResponsesPreviousResponseId, +} from '../../transformers/openai/responses/continuation.js'; import { buildUpstreamEndpointRequest, resolveUpstreamEndpointCandidates, -} from '../../routes/proxy/upstreamEndpoint.js'; +} from '../../services/upstreamEndpointRuntime.js'; import { getUpstreamEndpointRuntimeStateSnapshot, recordUpstreamEndpointFailure, recordUpstreamEndpointSuccess, } from '../../services/upstreamEndpointRuntimeMemory.js'; import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js'; -import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js'; +import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js'; import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js'; import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js'; import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js'; @@ -39,12 +46,19 @@ import { import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js'; import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js'; import { runCodexHttpSessionTask } from '../runtime/codexHttpSessionQueue.js'; +import { + buildCodexSessionResponseStoreKey, + clearCodexSessionResponseId, + getCodexSessionResponseId, + setCodexSessionResponseId, +} from '../runtime/codexSessionResponseStore.js'; import { summarizeConversationFileInputsInOpenAiBody, summarizeConversationFileInputsInResponsesBody, } from '../capabilities/conversationFileCapabilities.js'; import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js'; import { acquireSurfaceChannelLease, bindSurfaceStickyChannel, @@ -96,6 +110,12 @@ function isResponsesWebsocketTransportRequest(headers: Record): && String(rawValue).trim() === '1'); } +function rememberCodexSessionResponseId(sessionId: string, payload: unknown): void { + const responseId = extractResponsesTerminalResponseId(payload); + if (!responseId) return; + setCodexSessionResponseId(sessionId, responseId); +} + function normalizeIncludeList(value: unknown): string[] { if (typeof value === 'string') { const trimmed = value.trim(); @@ -345,6 +365,20 @@ export async function handleOpenAiResponsesSurfaceRequest( const modelName = selected.actualModel || requestedModel; const oauth = getOauthInfoFromAccount(selected.account); const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex'; + const codexSessionId = isCodexSite + ? getCodexSessionHeaderValue(request.headers as Record) + : ''; + const codexSessionStoreKey = ( + isCodexSite + && codexSessionId + ) + ? buildCodexSessionResponseStoreKey({ + sessionId: codexSessionId, + siteId: selected.site.id, + accountId: selected.account.id, + channelId: selected.channel.id, + }) + : ''; const owner = getProxyResourceOwner(request); let normalizedResponsesBody: Record = { ...requestEnvelope.parsed.normalizedBody, @@ -427,6 +461,20 @@ export async function handleOpenAiResponsesSurfaceRequest( const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => { const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => { const upstreamStream = isStream || (isCodexSite && endpoint === 'responses'); + const responsesOriginalBody = ( + endpoint === 'responses' + && isCodexSite + && codexSessionStoreKey + && shouldInferResponsesPreviousResponseId( + normalizedResponsesBody, + getCodexSessionResponseId(codexSessionStoreKey), + ) + ) + ? withResponsesPreviousResponseId( + normalizedResponsesBody, + getCodexSessionResponseId(codexSessionStoreKey)!, + ) + : normalizedResponsesBody; const endpointRequest = buildUpstreamEndpointRequest({ endpoint, modelName, @@ -438,7 +486,7 @@ export async function handleOpenAiResponsesSurfaceRequest( siteUrl: siteApiBaseUrl, openaiBody: openAiBody, downstreamFormat: 'responses', - responsesOriginalBody: normalizedResponsesBody, + responsesOriginalBody, downstreamHeaders: request.headers as Record, providerHeaders: buildProviderHeaders(), }); @@ -469,7 +517,7 @@ export async function handleOpenAiResponsesSurfaceRequest( } const sessionId = getCodexSessionHeaderValue(endpointRequest.headers); return runCodexHttpSessionTask( - sessionId, + codexSessionStoreKey || sessionId, () => baseDispatchRequest(endpointRequest, targetUrl), ); }; @@ -496,6 +544,35 @@ export async function handleOpenAiResponsesSurfaceRequest( return recovered; } } + if ( + ctx.request.endpoint === 'responses' + && isResponsesPreviousResponseNotFoundError({ + rawErrText: ctx.rawErrText, + }) + ) { + if (codexSessionStoreKey) { + clearCodexSessionResponseId(codexSessionStoreKey); + } + const previousResponseRecovery = stripResponsesPreviousResponseId(ctx.request.body); + if (previousResponseRecovery.removed) { + const recoveredRequest = { + ...ctx.request, + body: previousResponseRecovery.body, + }; + const recoveredResponse = await dispatchRequest(recoveredRequest, ctx.targetUrl); + if (recoveredResponse.ok) { + return { + upstream: recoveredResponse, + upstreamPath: recoveredRequest.path, + request: recoveredRequest, + targetUrl: ctx.targetUrl, + }; + } + ctx.request = recoveredRequest; + ctx.response = recoveredResponse; + ctx.rawErrText = await readRuntimeResponseText(recoveredResponse).catch(() => 'unknown error'); + } + } return endpointStrategy.tryRecover(ctx); }; @@ -507,6 +584,10 @@ export async function handleOpenAiResponsesSurfaceRequest( buildRequest: (endpoint) => buildEndpointRequest(endpoint), dispatchRequest, tryRecover, + shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback( + ctx.response.status, + ctx.rawErrText || ctx.errText, + ), onAttemptFailure: async (ctx) => { const memoryWrite = recordUpstreamEndpointFailure({ ...endpointRuntimeContext, @@ -699,6 +780,9 @@ export async function handleOpenAiResponsesSurfaceRequest( if (payload && typeof payload === 'object') { upstreamUsagePresent = upstreamUsagePresent || hasProxyUsagePayload(payload); parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(payload)); + if (codexSessionStoreKey) { + rememberCodexSessionResponseId(codexSessionStoreKey, payload); + } } }, writeLines, @@ -762,6 +846,9 @@ export async function handleOpenAiResponsesSurfaceRequest( if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') { upstreamData = unwrapGeminiCliPayload(upstreamData); } + if (codexSessionStoreKey) { + rememberCodexSessionResponseId(codexSessionStoreKey, upstreamData); + } parsedUsage = parseProxyUsage(upstreamData); upstreamUsagePresent = upstreamUsagePresent || hasProxyUsagePayload(upstreamData); @@ -867,6 +954,9 @@ export async function handleOpenAiResponsesSurfaceRequest( `event: ${terminalEventType}\ndata: ${JSON.stringify({ type: terminalEventType, response: collectedPayload })}\n\n`, 'data: [DONE]\n\n', ]); + if (codexSessionStoreKey) { + rememberCodexSessionResponseId(codexSessionStoreKey, collectedPayload); + } reply.raw.end(); const latency = Date.now() - startTime; await finalizeStreamSuccess( @@ -1025,6 +1115,9 @@ export async function handleOpenAiResponsesSurfaceRequest( if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') { upstreamData = unwrapGeminiCliPayload(upstreamData); } + if (codexSessionStoreKey) { + rememberCodexSessionResponseId(codexSessionStoreKey, upstreamData); + } const latency = Date.now() - startTime; const parsedUsage = parseProxyUsage(upstreamData); const upstreamUsagePresent = hasProxyUsagePayload(upstreamData); diff --git a/src/server/proxy-core/surfaces/sharedSurface.test.ts b/src/server/proxy-core/surfaces/sharedSurface.test.ts index d30fa64..018ceb7 100644 --- a/src/server/proxy-core/surfaces/sharedSurface.test.ts +++ b/src/server/proxy-core/surfaces/sharedSurface.test.ts @@ -85,6 +85,7 @@ vi.mock('../../services/alertRules.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args), + shouldAbortSameSiteEndpointFallback: () => false, })); vi.mock('../../services/oauth/quota.js', () => ({ diff --git a/src/server/proxy-core/surfaces/sharedSurface.ts b/src/server/proxy-core/surfaces/sharedSurface.ts index fb49cbc..ae6323d 100644 --- a/src/server/proxy-core/surfaces/sharedSurface.ts +++ b/src/server/proxy-core/surfaces/sharedSurface.ts @@ -11,9 +11,9 @@ import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js'; import { resolveProxyLogBilling } from '../../routes/proxy/proxyBilling.js'; import type { DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; -import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js'; -import type { BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js'; -import { buildUpstreamUrl } from '../../routes/proxy/upstreamUrl.js'; +import { dispatchRuntimeRequest } from '../../services/runtimeDispatch.js'; +import type { BuiltEndpointRequest } from '../orchestration/endpointFlow.js'; +import { buildUpstreamUrl } from '../orchestration/upstreamRequest.js'; import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js'; import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js'; import { proxyChannelCoordinator } from '../../services/proxyChannelCoordinator.js'; diff --git a/src/server/routes/api/settings.model-availability-probe.test.ts b/src/server/routes/api/settings.model-availability-probe.test.ts new file mode 100644 index 0000000..e908591 --- /dev/null +++ b/src/server/routes/api/settings.model-availability-probe.test.ts @@ -0,0 +1,106 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { eq } from 'drizzle-orm'; + +const { startModelAvailabilityProbeSchedulerMock, stopModelAvailabilityProbeSchedulerMock } = vi.hoisted(() => ({ + startModelAvailabilityProbeSchedulerMock: vi.fn(), + stopModelAvailabilityProbeSchedulerMock: vi.fn(), +})); + +vi.mock('../../services/modelAvailabilityProbeService.js', () => ({ + startModelAvailabilityProbeScheduler: startModelAvailabilityProbeSchedulerMock, + stopModelAvailabilityProbeScheduler: stopModelAvailabilityProbeSchedulerMock, +})); + +type ConfigModule = typeof import('../../config.js'); +type DbModule = typeof import('../../db/index.js'); + +describe('settings model availability probe runtime setting', () => { + let app: FastifyInstance; + let config: ConfigModule['config']; + let db: DbModule['db']; + let schema: DbModule['schema']; + let dataDir = ''; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-settings-model-probe-')); + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const dbModule = await import('../../db/index.js'); + const configModule = await import('../../config.js'); + const settingsRoutesModule = await import('./settings.js'); + + db = dbModule.db; + schema = dbModule.schema; + config = configModule.config; + + app = Fastify(); + await app.register(settingsRoutesModule.settingsRoutes); + }); + + beforeEach(async () => { + await db.delete(schema.settings).run(); + config.modelAvailabilityProbeEnabled = false; + startModelAvailabilityProbeSchedulerMock.mockReset(); + stopModelAvailabilityProbeSchedulerMock.mockReset(); + }); + + afterAll(async () => { + await app.close(); + rmSync(dataDir, { recursive: true, force: true }); + delete process.env.DATA_DIR; + }); + + it('persists enabling the model availability probe and starts the scheduler', async () => { + const updateResponse = await app.inject({ + method: 'PUT', + url: '/api/settings/runtime', + payload: { + modelAvailabilityProbeEnabled: true, + }, + }); + + expect(updateResponse.statusCode).toBe(200); + const updated = updateResponse.json() as { modelAvailabilityProbeEnabled?: boolean }; + expect(updated.modelAvailabilityProbeEnabled).toBe(true); + expect(config.modelAvailabilityProbeEnabled).toBe(true); + expect(startModelAvailabilityProbeSchedulerMock).toHaveBeenCalledTimes(1); + expect(stopModelAvailabilityProbeSchedulerMock).not.toHaveBeenCalled(); + + const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'model_availability_probe_enabled')).get(); + expect(saved?.value).toBe(JSON.stringify(true)); + + const getResponse = await app.inject({ + method: 'GET', + url: '/api/settings/runtime', + }); + expect(getResponse.statusCode).toBe(200); + expect((getResponse.json() as { modelAvailabilityProbeEnabled?: boolean }).modelAvailabilityProbeEnabled).toBe(true); + }); + + it('persists disabling the model availability probe and stops the scheduler', async () => { + config.modelAvailabilityProbeEnabled = true; + + const updateResponse = await app.inject({ + method: 'PUT', + url: '/api/settings/runtime', + payload: { + modelAvailabilityProbeEnabled: false, + }, + }); + + expect(updateResponse.statusCode).toBe(200); + const updated = updateResponse.json() as { modelAvailabilityProbeEnabled?: boolean }; + expect(updated.modelAvailabilityProbeEnabled).toBe(false); + expect(config.modelAvailabilityProbeEnabled).toBe(false); + expect(stopModelAvailabilityProbeSchedulerMock).toHaveBeenCalledTimes(1); + expect(startModelAvailabilityProbeSchedulerMock).not.toHaveBeenCalled(); + + const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'model_availability_probe_enabled')).get(); + expect(saved?.value).toBe(JSON.stringify(false)); + }); +}); diff --git a/src/server/routes/api/settings.ts b/src/server/routes/api/settings.ts index 11f5357..6ae95da 100644 --- a/src/server/routes/api/settings.ts +++ b/src/server/routes/api/settings.ts @@ -40,12 +40,17 @@ import { invalidateSiteProxyCache, normalizeSiteProxyUrl, withExplicitProxyReque import { performFactoryReset } from '../../services/factoryResetService.js'; import { normalizeLogCleanupRetentionDays } from '../../services/logCleanupService.js'; import { stopProxyLogRetentionService } from '../../services/proxyLogRetentionService.js'; +import { + startModelAvailabilityProbeScheduler, + stopModelAvailabilityProbeScheduler, +} from '../../services/modelAvailabilityProbeService.js'; type RoutingWeights = typeof config.routingWeights; interface RuntimeSettingsBody { proxyToken?: string; systemProxyUrl?: string; + modelAvailabilityProbeEnabled?: boolean; codexUpstreamWebsocketEnabled?: boolean; disableCrossProtocolFallback?: boolean; proxySessionChannelConcurrencyLimit?: number; @@ -404,6 +409,16 @@ function applyImportedSettingToRuntime(key: string, value: unknown) { config.systemProxyUrl = normalizeSiteProxyUrl(value) || ''; return; } + case 'model_availability_probe_enabled': { + if (typeof value !== 'boolean') return; + config.modelAvailabilityProbeEnabled = value; + if (value) { + startModelAvailabilityProbeScheduler(); + } else { + stopModelAvailabilityProbeScheduler(); + } + return; + } case 'codex_upstream_websocket_enabled': { if (typeof value !== 'boolean') return; config.codexUpstreamWebsocketEnabled = value; @@ -687,6 +702,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') { logCleanupUsageLogsEnabled: config.logCleanupUsageLogsEnabled, logCleanupProgramLogsEnabled: config.logCleanupProgramLogsEnabled, logCleanupRetentionDays: config.logCleanupRetentionDays, + modelAvailabilityProbeEnabled: config.modelAvailabilityProbeEnabled, codexUpstreamWebsocketEnabled: config.codexUpstreamWebsocketEnabled, disableCrossProtocolFallback: config.disableCrossProtocolFallback, proxySessionChannelConcurrencyLimit: config.proxySessionChannelConcurrencyLimit, @@ -1101,6 +1117,29 @@ export async function settingsRoutes(app: FastifyInstance) { invalidateSiteProxyCache(); } + if (body.modelAvailabilityProbeEnabled !== undefined) { + let nextValue = false; + try { + nextValue = parseBooleanFlag(body.modelAvailabilityProbeEnabled, '批量测活开关'); + } catch (err: any) { + return reply.code(400).send({ + success: false, + message: err?.message || '批量测活开关格式无效', + }); + } + + if (nextValue !== config.modelAvailabilityProbeEnabled) { + changedLabels.push(nextValue ? '开启批量测活' : '关闭批量测活'); + } + await upsertSetting('model_availability_probe_enabled', nextValue); + config.modelAvailabilityProbeEnabled = nextValue; + if (nextValue) { + startModelAvailabilityProbeScheduler(); + } else { + stopModelAvailabilityProbeScheduler(); + } + } + if (body.codexUpstreamWebsocketEnabled !== undefined) { let nextValue = false; try { diff --git a/src/server/routes/api/stats.model-probe.test.ts b/src/server/routes/api/stats.model-probe.test.ts new file mode 100644 index 0000000..4cc9242 --- /dev/null +++ b/src/server/routes/api/stats.model-probe.test.ts @@ -0,0 +1,157 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const buildModelAvailabilityProbeTaskDedupeKeyMock = vi.fn(); +const queueModelAvailabilityProbeTaskMock = vi.fn(); +const getBackgroundTaskMock = vi.fn(); +const getRunningTaskByDedupeKeyMock = vi.fn(); +const waitForBackgroundTaskCompletionMock = vi.fn(); + +vi.mock('../../services/modelAvailabilityProbeService.js', async () => { + const actual = await vi.importActual('../../services/modelAvailabilityProbeService.js'); + return { + ...actual, + buildModelAvailabilityProbeTaskDedupeKey: (...args: unknown[]) => buildModelAvailabilityProbeTaskDedupeKeyMock(...args), + queueModelAvailabilityProbeTask: (...args: unknown[]) => queueModelAvailabilityProbeTaskMock(...args), + }; +}); + +vi.mock('../../services/backgroundTaskService.js', async () => { + const actual = await vi.importActual('../../services/backgroundTaskService.js'); + return { + ...actual, + getBackgroundTask: (...args: unknown[]) => getBackgroundTaskMock(...args), + getRunningTaskByDedupeKey: (...args: unknown[]) => getRunningTaskByDedupeKeyMock(...args), + waitForBackgroundTaskCompletion: (...args: unknown[]) => waitForBackgroundTaskCompletionMock(...args), + }; +}); + +describe('/api/models/probe', () => { + let app: FastifyInstance; + let dataDir = ''; + let originalDataDir: string | undefined; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-stats-model-probe-')); + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const routesModule = await import('./stats.js'); + app = Fastify(); + await app.register(routesModule.statsRoutes); + }); + + beforeEach(() => { + buildModelAvailabilityProbeTaskDedupeKeyMock.mockReset(); + queueModelAvailabilityProbeTaskMock.mockReset(); + getBackgroundTaskMock.mockReset(); + getRunningTaskByDedupeKeyMock.mockReset(); + waitForBackgroundTaskCompletionMock.mockReset(); + }); + + afterAll(async () => { + await app.close(); + if (originalDataDir === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = originalDataDir; + } + }); + + it('rejects non-object request bodies instead of defaulting to a full probe', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/models/probe', + headers: { + 'content-type': 'application/json', + }, + payload: '"oops"', + }); + + expect(response.statusCode).toBe(400); + expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled(); + }); + + it('rejects loosely formatted account ids instead of truncating them', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/models/probe', + payload: { + accountId: '1e2', + }, + }); + + expect(response.statusCode).toBe(400); + expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled(); + }); + + it('rejects account ids that exceed the safe integer range', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/models/probe', + payload: { + accountId: '9007199254740993', + }, + }); + + expect(response.statusCode).toBe(400); + expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled(); + }); + + it('reuses the running deduped task for wait=true requests', async () => { + buildModelAvailabilityProbeTaskDedupeKeyMock.mockReturnValue('model-availability-probe-7'); + getRunningTaskByDedupeKeyMock.mockReturnValue({ + id: 'task-7', + status: 'running', + }); + const completedTask = { + id: 'task-7', + status: 'succeeded', + result: { + results: [], + summary: { + totalAccounts: 1, + success: 1, + failed: 0, + skipped: 0, + scanned: 3, + supported: 2, + unsupported: 1, + inconclusive: 0, + skippedModels: 0, + updatedRows: 1, + rebuiltRoutes: true, + }, + }, + }; + waitForBackgroundTaskCompletionMock.mockResolvedValue(completedTask); + + const response = await app.inject({ + method: 'POST', + url: '/api/models/probe', + payload: { + accountId: 7, + wait: true, + }, + }); + + expect(response.statusCode).toBe(200); + expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled(); + expect(waitForBackgroundTaskCompletionMock).toHaveBeenCalledWith('task-7'); + expect(response.json()).toMatchObject({ + success: true, + reused: true, + jobId: 'task-7', + summary: { + totalAccounts: 1, + scanned: 3, + supported: 2, + unsupported: 1, + }, + }); + }); +}); diff --git a/src/server/routes/api/stats.ts b/src/server/routes/api/stats.ts index 8ccaee6..b153d2a 100644 --- a/src/server/routes/api/stats.ts +++ b/src/server/routes/api/stats.ts @@ -6,8 +6,18 @@ import { refreshModelsForAccount } from '../../services/modelService.js'; import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js'; import { buildModelAnalysis } from '../../services/modelAnalysisService.js'; import { fallbackTokenCost, fetchModelPricingCatalog } from '../../services/modelPricingService.js'; +import { + buildModelAvailabilityProbeTaskDedupeKey, + queueModelAvailabilityProbeTask, + type ModelAvailabilityProbeExecutionResult, +} from '../../services/modelAvailabilityProbeService.js'; import { getUpstreamModelDescriptionsCached } from '../../services/upstreamModelDescriptionService.js'; -import { getRunningTaskByDedupeKey, startBackgroundTask } from '../../services/backgroundTaskService.js'; +import { + getBackgroundTask, + getRunningTaskByDedupeKey, + startBackgroundTask, + waitForBackgroundTaskCompletion, +} from '../../services/backgroundTaskService.js'; import { parseCheckinRewardAmount } from '../../services/checkinRewardParser.js'; import { estimateRewardWithTodayIncomeFallback } from '../../services/todayIncomeRewardService.js'; import { @@ -39,6 +49,10 @@ function parseBooleanFlag(raw?: string): boolean { return normalized === '1' || normalized === 'true' || normalized === 'yes'; } +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + const MODELS_MARKETPLACE_BASE_TTL_MS = 15_000; const MODELS_MARKETPLACE_PRICING_TTL_MS = 90_000; const limitModelTokenCandidatesRead = createRateLimitGuard({ @@ -1462,6 +1476,92 @@ export async function statsRoutes(app: FastifyInstance) { return { success: true, refresh, rebuild }; }); + app.post<{ Body?: { accountId?: number; wait?: boolean } }>('/api/models/probe', async (request, reply) => { + const requestBody = request.body; + if (requestBody !== undefined && !isRecord(requestBody)) { + return reply.code(400).send({ success: false, message: '请求体必须是对象' }); + } + + const rawAccountId = requestBody?.accountId as unknown; + const normalizedAccountId = rawAccountId === undefined || rawAccountId === null + ? '' + : String(rawAccountId).trim(); + const hasAccountId = normalizedAccountId !== ''; + const parsedAccountId = hasAccountId && /^[1-9]\d*$/.test(normalizedAccountId) + ? Number(normalizedAccountId) + : undefined; + const accountId = parsedAccountId !== undefined && Number.isSafeInteger(parsedAccountId) + ? parsedAccountId + : undefined; + const wait = requestBody?.wait === true; + + if (hasAccountId && accountId === undefined) { + return reply.code(400).send({ success: false, message: '账号 ID 无效' }); + } + + if (wait) { + const taskTitle = accountId ? `探测模型可用性 #${accountId}` : '探测全部模型可用性'; + const dedupeKey = buildModelAvailabilityProbeTaskDedupeKey(accountId); + const runningTask = getRunningTaskByDedupeKey(dedupeKey); + const { task, reused } = runningTask + ? { task: runningTask, reused: true } + : queueModelAvailabilityProbeTask({ + accountId, + title: taskTitle, + }); + const completedTask = await waitForBackgroundTaskCompletion(task.id); + if (!completedTask) { + return reply.code(500).send({ success: false, message: '模型可用性探测任务不存在或已过期' }); + } + if (completedTask.status === 'failed') { + return reply.code(500).send({ + success: false, + reused, + jobId: completedTask.id, + status: completedTask.status, + message: completedTask.error || '模型可用性探测失败', + }); + } + const result = completedTask.result as ModelAvailabilityProbeExecutionResult | null; + if (!result) { + return reply.code(500).send({ + success: false, + reused, + jobId: completedTask.id, + status: completedTask.status, + message: '模型可用性探测结果为空', + }); + } + if (accountId && result.summary.totalAccounts === 0) { + return reply.code(404).send({ success: false, message: '账号不存在' }); + } + return { + success: true, + reused, + jobId: completedTask.id, + status: completedTask.status, + ...result, + }; + } + + const taskTitle = accountId ? `探测模型可用性 #${accountId}` : '探测全部模型可用性'; + const { task, reused } = queueModelAvailabilityProbeTask({ + accountId, + title: taskTitle, + }); + + return reply.code(202).send({ + success: true, + queued: true, + reused, + jobId: task.id, + status: task.status, + message: reused + ? '模型可用性探测任务进行中，请稍后查看任务列表' + : '已开始模型可用性探测，请稍后查看任务列表', + }); + }); + // Site distribution – per-site aggregate data app.get('/api/stats/site-distribution', async () => { const accountRows = await db.select({ diff --git a/src/server/routes/proxy/architecture-boundaries.test.ts b/src/server/routes/proxy/architecture-boundaries.test.ts index 094c5f4..77e5229 100644 --- a/src/server/routes/proxy/architecture-boundaries.test.ts +++ b/src/server/routes/proxy/architecture-boundaries.test.ts @@ -104,6 +104,13 @@ describe('proxy route architecture boundaries', () => { expect(source).not.toContain('export function getUpstreamEndpointRuntimeStateSnapshot('); }); + it('keeps endpoint flow orchestration owned by proxy-core instead of the route layer', () => { + const source = readSource('./endpointFlow.ts'); + expect(source).toContain("from '../../proxy-core/orchestration/endpointFlow.js'"); + expect(source).not.toContain('async function runEndpointFlowHook<'); + expect(source).not.toContain('export async function executeEndpointFlow('); + }); + it('keeps gemini runtime closure in transformer-owned helpers', () => { const source = readSource('./gemini.ts'); const surfaceSource = readSource('../../proxy-core/surfaces/geminiSurface.ts'); diff --git a/src/server/routes/proxy/chat.codex-oauth.test.ts b/src/server/routes/proxy/chat.codex-oauth.test.ts index 2d6b106..cd47ddb 100644 --- a/src/server/routes/proxy/chat.codex-oauth.test.ts +++ b/src/server/routes/proxy/chat.codex-oauth.test.ts @@ -55,6 +55,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/chat.count-tokens.test.ts b/src/server/routes/proxy/chat.count-tokens.test.ts index c2d2c05..86d9403 100644 --- a/src/server/routes/proxy/chat.count-tokens.test.ts +++ b/src/server/routes/proxy/chat.count-tokens.test.ts @@ -59,6 +59,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/chat.siteApiEndpoint.test.ts b/src/server/routes/proxy/chat.siteApiEndpoint.test.ts index 7434780..420882f 100644 --- a/src/server/routes/proxy/chat.siteApiEndpoint.test.ts +++ b/src/server/routes/proxy/chat.siteApiEndpoint.test.ts @@ -59,6 +59,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/chat.stream.test.ts b/src/server/routes/proxy/chat.stream.test.ts index 9b2497e..9be8ba5 100644 --- a/src/server/routes/proxy/chat.stream.test.ts +++ b/src/server/routes/proxy/chat.stream.test.ts @@ -60,6 +60,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); @@ -817,6 +818,122 @@ describe('chat proxy stream behavior', () => { expect(response.body).toContain('event: message_stop'); }); + it('normalizes null Claude message content before proxying on /v1/messages', async () => { + selectChannelMock.mockReturnValue({ + channel: { id: 11, routeId: 22 }, + site: { + name: 'claude-site', + url: 'https://upstream.example.com', + platform: 'claude', + }, + account: { id: 33, username: 'demo-user' }, + tokenName: 'default', + tokenValue: 'sk-demo', + actualModel: 'claude-opus-4-6', + }); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ + id: 'msg_1', + type: 'message', + role: 'assistant', + model: 'claude-opus-4-6', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + usage: { input_tokens: 1, output_tokens: 1 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'POST', + url: '/v1/messages', + payload: { + model: 'claude-opus-4-6', + max_tokens: 256, + messages: [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: null }, + ], + }, + }); + + expect(response.statusCode).toBe(200); + const forwardedBody = JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string); + expect(forwardedBody.messages).toEqual([ + { role: 'user', content: 'hello' }, + ]); + }); + + it('prefers responses for Claude tool_result follow-ups that include continuation hints', async () => { + selectChannelMock.mockReturnValue({ + channel: { id: 11, routeId: 22 }, + site: { + name: 'openai-site', + url: 'https://upstream.example.com', + platform: 'openai', + }, + account: { id: 33, username: 'demo-user' }, + tokenName: 'default', + tokenValue: 'sk-demo', + actualModel: 'gpt-5.4', + }); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ + id: 'resp_1', + object: 'response', + model: 'gpt-5.4', + created_at: 1_706_000_000, + status: 'completed', + output: [{ + id: 'msg_1', + type: 'message', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'done' }], + }], + usage: { + input_tokens: 5, + output_tokens: 1, + total_tokens: 6, + }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'POST', + url: '/v1/messages', + payload: { + model: 'claude-opus-4-6', + max_tokens: 256, + previous_response_id: 'resp_prev_1', + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_missing', + content: [{ type: 'text', text: '{"matches":1}' }], + }, + { type: 'text', text: 'continue' }, + ], + }, + ], + }, + }); + + expect(response.statusCode).toBe(200); + expect(String(fetchMock.mock.calls[0]?.[0])).toContain('/v1/responses'); + const forwardedBody = JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string); + expect(forwardedBody.previous_response_id).toBe('resp_prev_1'); + expect(forwardedBody.input[0]).toEqual({ + type: 'function_call_output', + call_id: 'toolu_missing', + output: '{"matches":1}', + }); + }); + it('converts OpenAI tool_calls SSE into Claude tool_use stream events on /v1/messages', async () => { const encoder = new TextEncoder(); const upstreamBody = new ReadableStream({ diff --git a/src/server/routes/proxy/downstreamClientContext.routes.test.ts b/src/server/routes/proxy/downstreamClientContext.routes.test.ts index d484541..fb44848 100644 --- a/src/server/routes/proxy/downstreamClientContext.routes.test.ts +++ b/src/server/routes/proxy/downstreamClientContext.routes.test.ts @@ -58,6 +58,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/endpointFlow.test.ts b/src/server/routes/proxy/endpointFlow.test.ts index 9a0a4b6..0267ae7 100644 --- a/src/server/routes/proxy/endpointFlow.test.ts +++ b/src/server/routes/proxy/endpointFlow.test.ts @@ -224,6 +224,37 @@ describe('executeEndpointFlow', () => { expect(onAttemptSuccess.mock.calls[0]?.[0]?.request?.path).toBe('/v1/chat/completions'); }); + it('stops same-site endpoint fallback when the failure is classified as a site outage', async () => { + fetchMock + .mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({ + error: { message: 'Service temporarily unavailable', type: 'upstream_error' }, + }), { + status: 503, + headers: { 'content-type': 'application/json' }, + }))) + .mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }))); + + const result = await executeEndpointFlow({ + siteUrl: 'https://example.com', + endpointCandidates: ['responses', 'chat'], + buildRequest: (endpoint) => endpoint === 'responses' + ? requestFor('/v1/responses') + : { ...requestFor('/v1/chat/completions'), endpoint }, + shouldAbortRemainingEndpoints: () => true, + shouldDowngrade: () => true, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(503); + expect(result.errText).toContain('Service temporarily unavailable'); + } + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it('accepts recovered response from tryRecover hook', async () => { fetchMock.mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({ error: { message: 'upstream_error', type: 'upstream_error' }, diff --git a/src/server/routes/proxy/endpointFlow.ts b/src/server/routes/proxy/endpointFlow.ts index 464ce80..5e27be6 100644 --- a/src/server/routes/proxy/endpointFlow.ts +++ b/src/server/routes/proxy/endpointFlow.ts @@ -1,220 +1,13 @@ -import { fetch } from 'undici'; -import { readRuntimeResponseText } from '../../proxy-core/executors/types.js'; -import { withSiteProxyRequestInit } from '../../services/siteProxy.js'; -import { summarizeUpstreamError } from './upstreamError.js'; -import type { UpstreamEndpoint } from './upstreamEndpoint.js'; -import { buildUpstreamUrl } from './upstreamUrl.js'; +export { + executeEndpointFlow, + withUpstreamPath, +} from '../../proxy-core/orchestration/endpointFlow.js'; -export type BuiltEndpointRequest = { - endpoint: UpstreamEndpoint; - path: string; - headers: Record; - body: Record; - runtime?: { - executor: 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude'; - modelName?: string; - stream?: boolean; - oauthProjectId?: string | null; - action?: 'generateContent' | 'streamGenerateContent' | 'countTokens'; - }; -}; - -export type EndpointAttemptContext = { - endpointIndex: number; - endpointCount: number; - request: BuiltEndpointRequest; - targetUrl: string; - response: Awaited>; - rawErrText: string; - recoverApplied?: boolean; -}; - -export type EndpointAttemptSuccessContext = { - endpointIndex: number; - endpointCount: number; - request: BuiltEndpointRequest; - targetUrl: string; - response: Awaited>; - recoverApplied?: boolean; -}; - -export type EndpointRecoverResult = { - upstream: Awaited>; - upstreamPath: string; - request?: BuiltEndpointRequest; - targetUrl?: string; -} | null; - -export type EndpointFlowResult = - | { - ok: true; - upstream: Awaited>; - upstreamPath: string; - } - | { - ok: false; - status: number; - errText: string; - rawErrText?: string; - }; - -export function withUpstreamPath(path: string, message: string): string { - return `[upstream:${path}] ${message}`; -} - -type ExecuteEndpointFlowInput = { - siteUrl: string; - proxyUrl?: string | null; - disableCrossProtocolFallback?: boolean; - endpointCandidates: UpstreamEndpoint[]; - buildRequest: (endpoint: UpstreamEndpoint, endpointIndex: number) => BuiltEndpointRequest; - dispatchRequest?: ( - request: BuiltEndpointRequest, - targetUrl: string, - ) => Promise>>; - tryRecover?: (ctx: EndpointAttemptContext) => Promise; - shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean; - onDowngrade?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise; - onAttemptFailure?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise; - onAttemptSuccess?: (ctx: EndpointAttemptSuccessContext) => void | Promise; -}; - -async function runEndpointFlowHook( - hook: ((ctx: T) => void | Promise) | undefined, - ctx: T, - hookName: string, -): Promise { - if (!hook) return; - try { - await hook(ctx); - } catch (error) { - console.error(`endpointFlow ${hookName} hook failed`, error); - } -} -export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Promise { - const endpointCount = input.endpointCandidates.length; - if (endpointCount <= 0) { - return { - ok: false, - status: 502, - errText: 'Upstream request failed', - }; - } - - let finalStatus = 0; - let finalErrText = 'unknown error'; - let finalRawErrText: string | undefined; - - for (let endpointIndex = 0; endpointIndex < endpointCount; endpointIndex += 1) { - const endpoint = input.endpointCandidates[endpointIndex] as UpstreamEndpoint; - const request = input.buildRequest(endpoint, endpointIndex); - const defaultTarget = buildUpstreamUrl(input.siteUrl, request.path); - const targetUrl = input.proxyUrl - ? buildUpstreamUrl(input.proxyUrl, request.path) - : defaultTarget; - - let response = input.dispatchRequest - ? await input.dispatchRequest(request, targetUrl) - : await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, { - method: 'POST', - headers: request.headers, - body: JSON.stringify(request.body), - })); - - if (response.ok) { - await runEndpointFlowHook(input.onAttemptSuccess, { - endpointIndex, - endpointCount, - request, - targetUrl, - response, - recoverApplied: false, - }, 'onAttemptSuccess'); - return { - ok: true, - upstream: response, - upstreamPath: request.path, - }; - } - - let rawErrText = await readRuntimeResponseText(response).catch(() => 'unknown error'); - const baseContext: EndpointAttemptContext = { - endpointIndex, - endpointCount, - request, - targetUrl, - response, - rawErrText, - recoverApplied: false, - }; - - if (input.tryRecover) { - const recovered = await input.tryRecover(baseContext); - baseContext.recoverApplied = recovered !== null - || baseContext.request !== request - || baseContext.response !== response - || baseContext.rawErrText !== rawErrText; - if (recovered?.upstream?.ok) { - const recoveredRequest = recovered.request ?? baseContext.request; - const recoveredTargetUrl = recovered.targetUrl ?? ( - input.proxyUrl - ? buildUpstreamUrl(input.proxyUrl, recovered.upstreamPath) - : buildUpstreamUrl(input.siteUrl, recovered.upstreamPath) - ); - await runEndpointFlowHook(input.onAttemptSuccess, { - endpointIndex, - endpointCount, - request: recoveredRequest, - targetUrl: recoveredTargetUrl, - response: recovered.upstream, - recoverApplied: true, - }, 'onAttemptSuccess'); - return { - ok: true, - upstream: recovered.upstream, - upstreamPath: recovered.upstreamPath, - }; - } - } - - // Normalize again in case recoverer performed additional probes and updated the response text. - rawErrText = baseContext.rawErrText; - response = baseContext.response; - const errText = withUpstreamPath( - baseContext.request.path, - summarizeUpstreamError(response.status, rawErrText), - ); - await runEndpointFlowHook(input.onAttemptFailure, { - ...baseContext, - errText, - }, 'onAttemptFailure'); - - const isLastEndpoint = endpointIndex >= endpointCount - 1; - if (input.disableCrossProtocolFallback && !isLastEndpoint) { - finalStatus = response.status; - finalErrText = errText; - finalRawErrText = rawErrText; - break; - } - const shouldDowngrade = !isLastEndpoint && !!input.shouldDowngrade?.(baseContext); - if (shouldDowngrade) { - await runEndpointFlowHook(input.onDowngrade, { - ...baseContext, - errText, - }, 'onDowngrade'); - continue; - } - - finalStatus = response.status; - finalErrText = errText; - finalRawErrText = rawErrText; - break; - } - - return { - ok: false, - status: finalStatus || 502, - errText: finalErrText || 'unknown error', - rawErrText: finalRawErrText, - }; -} +export type { + BuiltEndpointRequest, + EndpointAttemptContext, + EndpointAttemptSuccessContext, + EndpointRecoverResult, + EndpointFlowResult, + ExecuteEndpointFlowInput, +} from '../../proxy-core/orchestration/endpointFlow.js'; diff --git a/src/server/routes/proxy/images.edits.test.ts b/src/server/routes/proxy/images.edits.test.ts index c130a46..061b5e3 100644 --- a/src/server/routes/proxy/images.edits.test.ts +++ b/src/server/routes/proxy/images.edits.test.ts @@ -52,6 +52,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/responses.codex-oauth.test.ts b/src/server/routes/proxy/responses.codex-oauth.test.ts index 445e1ef..f7c0fd2 100644 --- a/src/server/routes/proxy/responses.codex-oauth.test.ts +++ b/src/server/routes/proxy/responses.codex-oauth.test.ts @@ -3,6 +3,7 @@ import Fastify, { type FastifyInstance } from 'fastify'; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { config } from '../../config.js'; import { resetCodexHttpSessionQueue } from '../../proxy-core/runtime/codexHttpSessionQueue.js'; +import { resetCodexSessionResponseStore } from '../../proxy-core/runtime/codexSessionResponseStore.js'; const fetchMock = vi.fn(); const selectChannelMock = vi.fn(); @@ -69,6 +70,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); @@ -174,6 +176,7 @@ describe('responses proxy codex oauth refresh', () => { beforeEach(() => { resetCodexHttpSessionQueue(); + resetCodexSessionResponseStore(); config.proxyEmptyContentFailEnabled = false; config.proxyStickySessionEnabled = originalProxyStickySessionEnabled; config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit; @@ -215,6 +218,7 @@ describe('responses proxy codex oauth refresh', () => { }); selectPreferredChannelMock.mockReturnValue(null); selectNextChannelMock.mockReturnValue(null); + selectPreferredChannelMock.mockReturnValue(null); refreshOauthAccessTokenSingleflightMock.mockResolvedValue({ accessToken: 'fresh-access-token', accountId: 33, @@ -488,6 +492,137 @@ describe('responses proxy codex oauth refresh', () => { expect(forwardedBody.prompt_cache_key).toBe('codex-cache-123'); }); + it('infers previous_response_id for codex tool-output follow-up turns on the same downstream session', async () => { + fetchMock + .mockResolvedValueOnce(new Response(JSON.stringify({ + id: 'resp_codex_prev_1', + object: 'response', + model: 'gpt-5.2-codex', + status: 'completed', + output_text: 'tool call issued', + usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })) + .mockResolvedValueOnce(new Response(JSON.stringify({ + id: 'resp_codex_prev_2', + object: 'response', + model: 'gpt-5.2-codex', + status: 'completed', + output_text: 'tool result accepted', + usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const headers = { + session_id: 'session-http-prev-1', + 'user-agent': 'CodexClient/1.0', + }; + + const firstResponse = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers, + payload: { + model: 'gpt-5.2-codex', + input: 'start codex tool flow', + }, + }); + expect(firstResponse.statusCode).toBe(200); + + const secondResponse = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers, + payload: { + model: 'gpt-5.2-codex', + input: [ + { + id: 'tool_out_1', + type: 'function_call_output', + call_id: 'call_1', + output: '{"ok":true}', + }, + ], + }, + }); + + expect(secondResponse.statusCode).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(2); + + const [, secondOptions] = fetchMock.mock.calls[1] as [string, any]; + const secondBody = JSON.parse(secondOptions.body); + expect(secondBody.previous_response_id).toBe('resp_codex_prev_1'); + expect(secondBody.input).toEqual([ + { + id: 'tool_out_1', + type: 'function_call_output', + call_id: 'call_1', + output: '{"ok":true}', + }, + ]); + }); + + it('drops stale previous_response_id and retries codex responses requests once', async () => { + fetchMock + .mockResolvedValueOnce(new Response(JSON.stringify({ + error: { + message: 'previous_response_not_found', + code: 'previous_response_not_found', + type: 'invalid_request_error', + }, + }), { + status: 400, + headers: { 'content-type': 'application/json' }, + })) + .mockResolvedValueOnce(new Response(JSON.stringify({ + id: 'resp_codex_prev_recovered', + object: 'response', + model: 'gpt-5.2-codex', + status: 'completed', + output_text: 'recovered after dropping stale previous_response_id', + usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers: { + session_id: 'session-http-prev-recovery', + }, + payload: { + model: 'gpt-5.2-codex', + previous_response_id: 'resp_stale', + input: [ + { + id: 'tool_out_retry_1', + type: 'function_call_output', + call_id: 'call_retry_1', + output: '{"retry":true}', + }, + ], + }, + }); + + expect(response.statusCode).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(2); + + const [, firstOptions] = fetchMock.mock.calls[0] as [string, any]; + const [, secondOptions] = fetchMock.mock.calls[1] as [string, any]; + const firstBody = JSON.parse(firstOptions.body); + const secondBody = JSON.parse(secondOptions.body); + + expect(firstBody.previous_response_id).toBe('resp_stale'); + expect(secondBody.previous_response_id).toBeUndefined(); + expect(secondBody.input).toEqual(firstBody.input); + }); + it('strips generic downstream headers before forwarding codex responses upstream', async () => { fetchMock.mockResolvedValue(new Response(JSON.stringify({ id: 'resp_codex_header_filter', diff --git a/src/server/routes/proxy/responses.compact-upstream.test.ts b/src/server/routes/proxy/responses.compact-upstream.test.ts index 7d6166f..a34c6b8 100644 --- a/src/server/routes/proxy/responses.compact-upstream.test.ts +++ b/src/server/routes/proxy/responses.compact-upstream.test.ts @@ -57,6 +57,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/responses.websocket.test.ts b/src/server/routes/proxy/responses.websocket.test.ts index 5418494..7bc12da 100644 --- a/src/server/routes/proxy/responses.websocket.test.ts +++ b/src/server/routes/proxy/responses.websocket.test.ts @@ -2,12 +2,14 @@ import Fastify, { type FastifyInstance } from 'fastify'; import { createServer, type Server } from 'node:http'; import { AddressInfo } from 'node:net'; import WebSocket, { WebSocketServer } from 'ws'; -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { config } from '../../config.js'; +import { resetCodexSessionResponseStore } from '../../proxy-core/runtime/codexSessionResponseStore.js'; const fetchMock = vi.fn(); const selectChannelMock = vi.fn(); const selectNextChannelMock = vi.fn(); +const selectPreferredChannelMock = vi.fn(); const previewSelectedChannelMock = vi.fn(); const recordSuccessMock = vi.fn(); const recordFailureMock = vi.fn(); @@ -21,6 +23,7 @@ const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => estimatedCostFromQuota: 0, recoveredFromSelfLog: false, })); +const trackedClientSockets = new Set(); const dbInsertMock = vi.fn((_arg?: any) => ({ values: () => ({ run: () => undefined, @@ -35,6 +38,7 @@ vi.mock('../../services/tokenRouter.js', () => ({ tokenRouter: { selectChannel: (...args: unknown[]) => selectChannelMock(...args), selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args), + selectPreferredChannel: (...args: unknown[]) => selectPreferredChannelMock(...args), previewSelectedChannel: (...args: unknown[]) => previewSelectedChannelMock(...args), recordSuccess: (...args: unknown[]) => recordSuccessMock(...args), recordFailure: (...args: unknown[]) => recordFailureMock(...args), @@ -78,6 +82,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); @@ -256,16 +261,26 @@ function createDeferred() { } function createClientSocket(baseUrl: string, headers: Record = {}) { - return new WebSocket(`${baseUrl}/v1/responses`, { + const socket = new WebSocket(`${baseUrl}/v1/responses`, { headers: { Authorization: 'Bearer sk-global-proxy-token', ...headers, }, }); + trackedClientSockets.add(socket); + socket.once('close', () => { + trackedClientSockets.delete(socket); + }); + return socket; } function createClientSocketForPath(path: string, headers: Record = {}) { - return new WebSocket(path, { headers }); + const socket = new WebSocket(path, { headers }); + trackedClientSockets.add(socket); + socket.once('close', () => { + trackedClientSockets.delete(socket); + }); + return socket; } describe('responses websocket transport', () => { @@ -274,6 +289,7 @@ describe('responses websocket transport', () => { let app: FastifyInstance; let baseUrl: string; let upstreamServer: WebSocketServer; + let upstreamSockets: Set; let upstreamSiteUrl: string; let upstreamConnectionCount: number; let upstreamUpgradeHeaders: Record; @@ -294,7 +310,12 @@ describe('responses websocket transport', () => { baseUrl = `ws://127.0.0.1:${address.port}`; upstreamServer = new WebSocketServer({ port: 0 }); + upstreamSockets = new Set(); upstreamServer.on('connection', (socket, request) => { + upstreamSockets.add(socket); + socket.once('close', () => { + upstreamSockets.delete(socket); + }); upstreamConnectionCount += 1; upstreamUpgradeHeaders = Object.fromEntries( Object.entries(request.headers) @@ -329,9 +350,11 @@ describe('responses websocket transport', () => { }); beforeEach(() => { + resetCodexSessionResponseStore(); fetchMock.mockReset(); selectChannelMock.mockReset(); selectNextChannelMock.mockReset(); + selectPreferredChannelMock.mockReset(); previewSelectedChannelMock.mockReset(); recordSuccessMock.mockReset(); recordFailureMock.mockReset(); @@ -346,6 +369,7 @@ describe('responses websocket transport', () => { const selectedChannel = createSelectedChannel(); selectChannelMock.mockReturnValue(selectedChannel); selectNextChannelMock.mockReturnValue(null); + selectPreferredChannelMock.mockReturnValue(null); previewSelectedChannelMock.mockResolvedValue(selectedChannel); upstreamConnectionCount = 0; upstreamUpgradeHeaders = {}; @@ -386,8 +410,29 @@ describe('responses websocket transport', () => { }; }); + afterEach(() => { + for (const socket of trackedClientSockets) { + try { + socket.terminate(); + } catch {} + } + trackedClientSockets.clear(); + }); + afterAll(async () => { (config as any).codexUpstreamWebsocketEnabled = originalCodexUpstreamWebsocketEnabled; + for (const socket of trackedClientSockets) { + try { + socket.terminate(); + } catch {} + } + trackedClientSockets.clear(); + for (const socket of upstreamSockets || []) { + try { + socket.terminate(); + } catch {} + } + upstreamSockets?.clear(); if (rejectedUpgradeServer) { await new Promise((resolve) => rejectedUpgradeServer.close(() => resolve())); } @@ -551,6 +596,143 @@ describe('responses websocket transport', () => { expect(upstreamUpgradeHeaders['openai-beta']).toContain('responses_websockets=2099-01-01'); }); + it('infers previous_response_id for websocket tool-output follow-up turns when the client omits it', async () => { + const selectedChannel = createSelectedChannel({ + siteUrl: upstreamSiteUrl, + }); + selectChannelMock.mockReturnValue(selectedChannel); + previewSelectedChannelMock.mockResolvedValue(selectedChannel); + + const socket = createClientSocket(baseUrl, { + session_id: 'ws-session-prev-infer', + }); + await waitForSocketOpen(socket); + + const firstResponsePromise = waitForSocketMessageMatching( + socket, + (message) => message?.type === 'response.completed', + ); + socket.send(JSON.stringify({ + type: 'response.create', + model: 'gpt-5.4', + input: [], + })); + const firstResponse = await firstResponsePromise; + + const secondResponsePromise = waitForSocketMessageMatching( + socket, + (message) => message?.type === 'response.completed' && message?.response?.id === 'resp_upstream_2', + ); + socket.send(JSON.stringify({ + type: 'response.create', + model: 'gpt-5.4', + input: [ + { + id: 'tool_out_ws_1', + type: 'function_call_output', + call_id: 'call_ws_1', + output: '{"ok":true}', + }, + ], + })); + await secondResponsePromise; + socket.close(); + + expect(firstResponse?.response?.id).toBe('resp_upstream_1'); + expect(upstreamRequests).toHaveLength(2); + expect(upstreamRequests[1]).toMatchObject({ + type: 'response.create', + previous_response_id: 'resp_upstream_1', + input: [ + { + id: 'tool_out_ws_1', + type: 'function_call_output', + call_id: 'call_ws_1', + output: '{"ok":true}', + }, + ], + }); + }); + + it('retries websocket turns once without previous_response_id when the upstream reports previous_response_not_found', async () => { + const selectedChannel = createSelectedChannel({ + siteUrl: upstreamSiteUrl, + }); + selectChannelMock.mockReturnValue(selectedChannel); + previewSelectedChannelMock.mockResolvedValue(selectedChannel); + upstreamMessageHandler = (socket, parsed, requestIndex) => { + if (requestIndex === 1) { + socket.send(JSON.stringify({ + type: 'error', + error: { + message: 'previous_response_not_found', + code: 'previous_response_not_found', + type: 'invalid_request_error', + }, + })); + return; + } + socket.send(JSON.stringify({ + type: 'response.completed', + response: { + id: 'resp_upstream_recovered', + object: 'response', + model: parsed.model || 'gpt-5.4', + status: 'completed', + output: [], + usage: { + input_tokens: 1, + output_tokens: 1, + total_tokens: 2, + }, + }, + })); + }; + + const socket = createClientSocket(baseUrl, { + session_id: 'ws-session-prev-recovery', + }); + await waitForSocketOpen(socket); + + const responsePromise = waitForSocketMessageMatching( + socket, + (message) => message?.type === 'response.completed' && message?.response?.id === 'resp_upstream_recovered', + ); + socket.send(JSON.stringify({ + type: 'response.create', + model: 'gpt-5.4', + previous_response_id: 'resp_stale_ws', + input: [ + { + id: 'tool_out_ws_retry_1', + type: 'function_call_output', + call_id: 'call_ws_retry_1', + output: '{"retry":true}', + }, + ], + })); + await responsePromise; + socket.close(); + + expect(upstreamRequests).toHaveLength(2); + expect(upstreamRequests[0]).toMatchObject({ + type: 'response.create', + previous_response_id: 'resp_stale_ws', + }); + expect(upstreamRequests[1]).toMatchObject({ + type: 'response.create', + input: [ + { + id: 'tool_out_ws_retry_1', + type: 'function_call_output', + call_id: 'call_ws_retry_1', + output: '{"retry":true}', + }, + ], + }); + expect(upstreamRequests[1]?.previous_response_id).toBeUndefined(); + }); + it('falls back to the HTTP responses executor when the upstream codex websocket upgrade returns 426', async () => { const selectedChannel = createSelectedChannel({ siteUrl: rejectedUpgradeSiteUrl, diff --git a/src/server/routes/proxy/responsesWebsocket.ts b/src/server/routes/proxy/responsesWebsocket.ts index 08470bd..e0e6b6b 100644 --- a/src/server/routes/proxy/responsesWebsocket.ts +++ b/src/server/routes/proxy/responsesWebsocket.ts @@ -4,6 +4,7 @@ import type { IncomingMessage } from 'node:http'; import type { Duplex } from 'node:stream'; import { WebSocketServer, type RawData, type WebSocket } from 'ws'; import { createCodexWebsocketRuntime, CodexWebsocketRuntimeError } from '../../proxy-core/runtime/codexWebsocketRuntime.js'; +import { buildCodexSessionResponseStoreKey } from '../../proxy-core/runtime/codexSessionResponseStore.js'; import { authorizeDownstreamToken, consumeManagedKeyRequest, @@ -541,13 +542,23 @@ async function handleResponsesWebsocketConnection( const websocketSessionId = headerValueToTrimmedString(request.headers['session_id']) || headerValueToTrimmedString(request.headers['session-id']) || randomUUID(); + const runtimeSessionKeys = new Set(); let lastRequest: Record | null = null; let lastResponseOutput: unknown[] = []; let selectedChannel: SelectedChannel | null = null; let messageQueue = Promise.resolve(); socket.once('close', () => { - void codexWebsocketRuntime.closeSession(websocketSessionId); + const sessionKeys = runtimeSessionKeys.size > 0 + ? Array.from(runtimeSessionKeys) + : [websocketSessionId]; + void Promise.all(sessionKeys.map(async (sessionKey) => { + try { + await codexWebsocketRuntime.closeSession(sessionKey); + } catch { + // Ignore close-time cleanup failures after downstream disconnects. + } + })); }); socket.on('message', (raw) => { @@ -634,8 +645,15 @@ async function handleResponsesWebsocketConnection( const requestUrl = `${codexWebsocketChannel.site.url.replace(/\/+$/, '')}${prepared.path}`; try { - const runtimeResult = await codexWebsocketRuntime.sendRequest({ + const websocketRuntimeSessionKey = buildCodexSessionResponseStoreKey({ sessionId: websocketSessionId, + siteId: codexWebsocketChannel.site.id, + accountId: codexWebsocketChannel.account.id, + channelId: codexWebsocketChannel.channel.id, + }) || websocketSessionId; + runtimeSessionKeys.add(websocketRuntimeSessionKey); + const runtimeResult = await codexWebsocketRuntime.sendRequest({ + sessionId: websocketRuntimeSessionKey, requestUrl, headers: prepared.headers, body: prepared.body, diff --git a/src/server/routes/proxy/search.test.ts b/src/server/routes/proxy/search.test.ts index 870e1df..eb38164 100644 --- a/src/server/routes/proxy/search.test.ts +++ b/src/server/routes/proxy/search.test.ts @@ -48,6 +48,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + shouldAbortSameSiteEndpointFallback: () => false, RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/routes/proxy/upstreamEndpoint.test.ts b/src/server/routes/proxy/upstreamEndpoint.test.ts index b45e170..c907bdb 100644 --- a/src/server/routes/proxy/upstreamEndpoint.test.ts +++ b/src/server/routes/proxy/upstreamEndpoint.test.ts @@ -8,6 +8,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ })); import { + buildClaudeCountTokensUpstreamRequest, buildMinimalJsonHeadersForCompatibility, buildUpstreamEndpointRequest, isUnsupportedMediaTypeError, @@ -188,6 +189,23 @@ describe('resolveUpstreamEndpointCandidates', () => { expect(codexOrder).toEqual(['responses']); }); + it('prefers responses for claude continuation follow-ups that carry orphan tool results', async () => { + const order = await resolveUpstreamEndpointCandidates( + { + ...baseContext, + site: { ...baseContext.site, platform: 'openai' }, + }, + 'gpt-5.4', + 'claude', + 'claude-opus-4-6', + { + wantsContinuationAwareResponses: true, + }, + ); + + expect(order).toEqual(['responses', 'messages', 'chat']); + }); + it('prefers document-capable endpoints when downstream content contains non-image files', async () => { const order = await resolveUpstreamEndpointCandidates( { @@ -2053,6 +2071,60 @@ describe('buildUpstreamEndpointRequest', () => { }); }); + it('drops responses-style continuation fields before proxying native Claude messages upstream', () => { + const request = buildUpstreamEndpointRequest({ + endpoint: 'messages', + modelName: 'claude-opus-4-6', + stream: false, + tokenValue: 'sk-test', + sitePlatform: 'claude', + siteUrl: 'https://example.com', + downstreamFormat: 'claude', + openaiBody: { + model: 'ignored', + messages: [{ role: 'user', content: 'ignored' }], + }, + claudeOriginalBody: { + model: 'claude-opus-4-6', + max_tokens: 256, + previous_response_id: 'resp_prev_1', + prompt_cache_key: 'cache-key-1', + messages: [{ role: 'user', content: 'hello' }], + }, + }); + + expect(request.body).toEqual({ + model: 'claude-opus-4-6', + max_tokens: 256, + messages: [{ role: 'user', content: 'hello' }], + stream: false, + }); + }); + + it('drops responses-style continuation fields before proxying Claude count_tokens upstream', () => { + const request = buildClaudeCountTokensUpstreamRequest({ + modelName: 'claude-opus-4-6', + tokenValue: 'sk-test', + sitePlatform: 'claude', + claudeBody: { + model: 'claude-opus-4-6', + max_tokens: 256, + previous_response_id: 'resp_prev_1', + prompt_cache_key: 'cache-key-1', + messages: [{ role: 'user', content: 'hello' }], + }, + }); + + expect(request.body).toMatchObject({ + model: 'claude-opus-4-6', + messages: [{ role: 'user' }], + }); + expect(request.body).not.toHaveProperty('previous_response_id'); + expect(request.body).not.toHaveProperty('prompt_cache_key'); + expect(request.body).not.toHaveProperty('max_tokens'); + expect(request.body).not.toHaveProperty('maxTokens'); + }); + it('preserves multimodal OpenAI user content when converting to Responses input', () => { const request = buildUpstreamEndpointRequest({ endpoint: 'responses', diff --git a/src/server/routes/proxy/upstreamEndpoint.ts b/src/server/routes/proxy/upstreamEndpoint.ts index 02d97c2..6ce16ca 100644 --- a/src/server/routes/proxy/upstreamEndpoint.ts +++ b/src/server/routes/proxy/upstreamEndpoint.ts @@ -3,6 +3,7 @@ import { rankConversationFileEndpoints, type ConversationFileInputSummary, } from '../../proxy-core/capabilities/conversationFileCapabilities.js'; +import type { UpstreamEndpoint } from '../../proxy-core/orchestration/upstreamRequest.js'; import { resolveProviderProfile } from '../../proxy-core/providers/registry.js'; import { config } from '../../config.js'; import { fetchModelPricingCatalog } from '../../services/modelPricingService.js'; @@ -46,7 +47,7 @@ export { shouldPreferResponsesAfterLegacyChatError, }; -export type UpstreamEndpoint = 'chat' | 'messages' | 'responses'; +export type { UpstreamEndpoint } from '../../proxy-core/orchestration/upstreamRequest.js'; export type EndpointPreference = DownstreamFormat | 'responses'; type ChannelContext = { @@ -225,6 +226,15 @@ function extractClaudeBetasFromBody(body: Record): { }; } +function stripClaudeMessagesContinuationFields( + body: Record, +): Record { + const next = { ...body }; + delete next.previous_response_id; + delete next.prompt_cache_key; + return next; +} + function buildAntigravityRuntimeHeaders(input: { baseHeaders: Record; stream: boolean; @@ -529,6 +539,7 @@ export async function resolveUpstreamEndpointCandidates( hasNonImageFileInput?: boolean; conversationFileSummary?: ConversationFileInputSummary; wantsNativeResponsesReasoning?: boolean; + wantsContinuationAwareResponses?: boolean; }, ): Promise { const sitePlatform = normalizePlatformName(context.site.platform); @@ -540,6 +551,7 @@ export async function resolveUpstreamEndpointCandidates( const preferMessagesForClaudeModel = capabilityProfile.preferMessagesForClaudeModel; const hasNonImageFileInput = capabilityProfile.hasNonImageFileInput; const wantsNativeResponsesReasoning = capabilityProfile.wantsNativeResponsesReasoning; + const wantsContinuationAwareResponses = capabilityProfile.wantsContinuationAwareResponses; const applyRuntimePreference = (candidates: UpstreamEndpoint[]) => ( applyUpstreamEndpointRuntimePreference(candidates, { siteId: context.site.id, @@ -587,9 +599,11 @@ export async function resolveUpstreamEndpointCandidates( })() : preferred; const prioritizedPreferredEndpoints: UpstreamEndpoint[] = ( - wantsNativeResponsesReasoning - && preferMessagesForClaudeModel - && preferredWithCapabilities.includes('responses') + preferredWithCapabilities.includes('responses') + && ( + wantsContinuationAwareResponses + || (wantsNativeResponsesReasoning && preferMessagesForClaudeModel) + ) ) ? [ 'responses', @@ -865,7 +879,7 @@ export function buildUpstreamEndpointRequest(input: { && input.forceNormalizeClaudeBody !== true ) ? { - ...input.claudeOriginalBody, + ...stripClaudeMessagesContinuationFields(input.claudeOriginalBody), model: input.modelName, stream: input.stream, } @@ -876,7 +890,7 @@ export function buildUpstreamEndpointRequest(input: { && input.forceNormalizeClaudeBody === true ) ? sanitizeAnthropicMessagesBody({ - ...input.claudeOriginalBody, + ...stripClaudeMessagesContinuationFields(input.claudeOriginalBody), model: input.modelName, stream: input.stream, }) @@ -1034,7 +1048,7 @@ export function buildClaudeCountTokensUpstreamRequest(input: { const sitePlatform = normalizePlatformName(input.sitePlatform); const claudeHeaders = extractClaudePassthroughHeaders(input.downstreamHeaders); const { body: bodyWithoutBetas, betas } = extractClaudeBetasFromBody({ - ...input.claudeBody, + ...stripClaudeMessagesContinuationFields(input.claudeBody), model: input.modelName, }); const sanitizedBody = sanitizeAnthropicMessagesBody(bodyWithoutBetas); diff --git a/src/server/routes/proxy/upstreamError.ts b/src/server/routes/proxy/upstreamError.ts index 6799035..1bb20c3 100644 --- a/src/server/routes/proxy/upstreamError.ts +++ b/src/server/routes/proxy/upstreamError.ts @@ -1,81 +1 @@ -function isRecord(value: unknown): value is Record { - return !!value && typeof value === 'object'; -} - -function collapseWhitespace(text: string): string { - return text.replace(/\s+/g, ' ').trim(); -} - -function extractJsonErrorMessage(rawText: string): string { - try { - const parsed = JSON.parse(rawText) as unknown; - const root = isRecord(parsed) ? parsed : null; - const error = (root && isRecord(root.error)) ? root.error : root; - if (!error) return ''; - - const message = typeof error.message === 'string' ? collapseWhitespace(error.message) : ''; - if (message) return message; - - const code = typeof error.code === 'string' ? collapseWhitespace(error.code) : ''; - const type = typeof error.type === 'string' ? collapseWhitespace(error.type) : ''; - return [type, code].filter((part) => part.length > 0).join('/'); - } catch { - return ''; - } -} - -function extractHtmlTitle(rawText: string): string { - const match = rawText.match(/]*>([\s\S]*?)<\/title>/i); - if (!match?.[1]) return ''; - return collapseWhitespace(match[1].replace(/<[^>]+>/g, '')); -} - -function extractCloudflareHtmlSummary(rawText: string, status: number): string { - if (!/cloudflare/i.test(rawText)) return ''; - const title = extractHtmlTitle(rawText); - const codeMatch = ( - title.match(/\b(\d{3,4})\s*:\s*([^\|<]+)/i) - || rawText.match(/Error code\s*(\d{3,4})/i) - ); - const code = codeMatch?.[1] || (status > 0 ? String(status) : ''); - const reason = collapseWhitespace( - (typeof codeMatch?.[2] === 'string' ? codeMatch[2] : '') - || (status >= 500 ? 'origin host error' : 'request blocked') - ); - if (code) return `Cloudflare ${code}: ${reason}`; - return `Cloudflare: ${reason}`; -} - -function extractHtmlSummary(rawText: string, status: number): string { - if (!/(]*>([\s\S]*?)<\/h1>/i)?.[1] || ''; - return collapseWhitespace(heading.replace(/<[^>]+>/g, '')); -} - -export function summarizeUpstreamError(status: number, rawErrorText: string): string { - const statusPrefix = status > 0 - ? `Upstream returned HTTP ${status}` - : 'Upstream request failed'; - - const raw = typeof rawErrorText === 'string' ? rawErrorText.trim() : ''; - if (!raw) return statusPrefix; - - const jsonMessage = extractJsonErrorMessage(raw); - if (jsonMessage) return `${statusPrefix}: ${jsonMessage}`; - - const htmlMessage = extractHtmlSummary(raw, status); - if (htmlMessage) return `${statusPrefix}: ${htmlMessage}`; - - const compact = collapseWhitespace(raw); - if (!compact) return statusPrefix; - if (compact.length <= 400) return `${statusPrefix}: ${compact}`; - return `${statusPrefix}: ${compact.slice(0, 400)}...(truncated)`; -} - +export { summarizeUpstreamError } from '../../proxy-core/orchestration/upstreamRequest.js'; diff --git a/src/server/routes/proxy/upstreamUrl.ts b/src/server/routes/proxy/upstreamUrl.ts index 97ae8c8..5b1ab9e 100644 --- a/src/server/routes/proxy/upstreamUrl.ts +++ b/src/server/routes/proxy/upstreamUrl.ts @@ -1,58 +1 @@ -function formatUrlOrigin(url: URL): string { - // URL.origin drops credentials; preserve them if present. - const username = url.username ? encodeURIComponent(url.username) : ''; - const password = url.password ? encodeURIComponent(url.password) : ''; - const auth = username - ? `${username}${password ? `:${password}` : ''}@` - : ''; - - return `${url.protocol}//${auth}${url.host}`; -} - -function joinPath(basePath: string, requestPath: string): string { - const base = basePath.replace(/\/+$/, ''); - const path = requestPath.startsWith('/') ? requestPath : `/${requestPath}`; - - if (!base || base === '/') return path || '/'; - if (!path || path === '/') return base; - return `${base}${path}`; -} - -export function buildUpstreamUrl(siteUrl: string, requestPath: string): string { - const baseRaw = typeof siteUrl === 'string' ? siteUrl.trim() : ''; - const pathRaw = typeof requestPath === 'string' ? requestPath.trim() : ''; - const fallbackBase = baseRaw.replace(/\/+$/, ''); - let path = pathRaw.startsWith('/') ? pathRaw : `/${pathRaw}`; - - if (!fallbackBase) return path || '/'; - if (!path || path === '/') return fallbackBase; - - try { - const parsed = new URL(baseRaw); - const basePath = parsed.pathname.replace(/\/+$/, ''); - const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(basePath); - if (baseHasVersionSuffix) { - if (path === '/v1') { - path = '/'; - } else if (path.startsWith('/v1/')) { - path = path.slice('/v1'.length) || '/'; - } - } - - const joinedPath = joinPath(basePath, path); - return `${formatUrlOrigin(parsed)}${joinedPath}${parsed.search}${parsed.hash}`; - } catch { - // Ignore URL parsing errors and fall back to naive join. - const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(fallbackBase); - if (baseHasVersionSuffix) { - if (path === '/v1') { - path = '/'; - } else if (path.startsWith('/v1/')) { - path = path.slice('/v1'.length) || '/'; - } - } - - return `${fallbackBase}${path}`; - } -} - +export { buildUpstreamUrl } from '../../proxy-core/orchestration/upstreamRequest.js'; diff --git a/src/server/routes/proxy/videos.test.ts b/src/server/routes/proxy/videos.test.ts index 4722e7e..4a0d0bb 100644 --- a/src/server/routes/proxy/videos.test.ts +++ b/src/server/routes/proxy/videos.test.ts @@ -53,6 +53,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ })); vi.mock('../../services/proxyRetryPolicy.js', () => ({ + shouldAbortSameSiteEndpointFallback: () => false, shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args), RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); diff --git a/src/server/services/backgroundTaskService.ts b/src/server/services/backgroundTaskService.ts index 3684b52..7daeab2 100644 --- a/src/server/services/backgroundTaskService.ts +++ b/src/server/services/backgroundTaskService.ts @@ -301,6 +301,21 @@ export function getRunningTaskByDedupeKey(key: string): BackgroundTask | null { return task; } +export async function waitForBackgroundTaskCompletion(taskId: string, pollIntervalMs = 25): Promise { + const safePollIntervalMs = Math.max(5, Math.trunc(pollIntervalMs || 0)); + while (true) { + const task = getBackgroundTask(taskId); + if (!task) return null; + if (task.status !== 'pending' && task.status !== 'running') { + return task; + } + await new Promise((resolve) => { + const timer = setTimeout(resolve, safePollIntervalMs); + timer.unref?.(); + }); + } +} + export function summarizeCheckinResults(results: Array<{ result?: any }>): { total: number; success: number; skipped: number; failed: number } { const summary = { total: results.length, success: 0, skipped: 0, failed: 0 }; for (const item of results) { diff --git a/src/server/services/channelRecoveryProbeService.test.ts b/src/server/services/channelRecoveryProbeService.test.ts new file mode 100644 index 0000000..d638137 --- /dev/null +++ b/src/server/services/channelRecoveryProbeService.test.ts @@ -0,0 +1,348 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { eq } from 'drizzle-orm'; + +const probeRuntimeModelMock = vi.fn(); + +vi.mock('./runtimeModelProbe.js', () => ({ + probeRuntimeModel: (...args: unknown[]) => probeRuntimeModelMock(...args), +})); + +type DbModule = typeof import('../db/index.js'); +type RecoveryModule = typeof import('./channelRecoveryProbeService.js'); +type CoordinatorModule = typeof import('./proxyChannelCoordinator.js'); +type ConfigModule = typeof import('../config.js'); +type TokenRouterModule = typeof import('./tokenRouter.js'); + +describe('channelRecoveryProbeService', () => { + let db: DbModule['db']; + let schema: DbModule['schema']; + let runChannelRecoveryProbeSweep: RecoveryModule['runChannelRecoveryProbeSweep']; + let resetChannelRecoveryProbeState: RecoveryModule['resetChannelRecoveryProbeState']; + let proxyChannelCoordinator: CoordinatorModule['proxyChannelCoordinator']; + let resetProxyChannelCoordinatorState: CoordinatorModule['resetProxyChannelCoordinatorState']; + let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache']; + let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState']; + let config: ConfigModule['config']; + let dataDir = ''; + let originalDataDir: string | undefined; + let originalConcurrencyLimit = 0; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-channel-recovery-probe-')); + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = dataDir; + + await import('../db/migrate.js'); + const dbModule = await import('../db/index.js'); + const recoveryModule = await import('./channelRecoveryProbeService.js'); + const coordinatorModule = await import('./proxyChannelCoordinator.js'); + const configModule = await import('../config.js'); + const tokenRouterModule = await import('./tokenRouter.js'); + + db = dbModule.db; + schema = dbModule.schema; + runChannelRecoveryProbeSweep = recoveryModule.runChannelRecoveryProbeSweep; + resetChannelRecoveryProbeState = recoveryModule.resetChannelRecoveryProbeState; + proxyChannelCoordinator = coordinatorModule.proxyChannelCoordinator; + resetProxyChannelCoordinatorState = coordinatorModule.resetProxyChannelCoordinatorState; + invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache; + resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState; + config = configModule.config; + originalConcurrencyLimit = config.proxySessionChannelConcurrencyLimit; + }); + + beforeEach(async () => { + probeRuntimeModelMock.mockReset(); + probeRuntimeModelMock.mockResolvedValue({ + status: 'supported', + latencyMs: 320, + reason: 'probe succeeded', + }); + config.proxySessionChannelConcurrencyLimit = 1; + resetChannelRecoveryProbeState(); + resetProxyChannelCoordinatorState(); + invalidateTokenRouterCache(); + resetSiteRuntimeHealthState(); + + await db.delete(schema.routeChannels).run(); + await db.delete(schema.tokenRoutes).run(); + await db.delete(schema.settings).run(); + await db.delete(schema.accountTokens).run(); + await db.delete(schema.accounts).run(); + await db.delete(schema.sites).run(); + }); + + afterAll(() => { + config.proxySessionChannelConcurrencyLimit = originalConcurrencyLimit; + resetChannelRecoveryProbeState(); + resetProxyChannelCoordinatorState(); + invalidateTokenRouterCache(); + resetSiteRuntimeHealthState(); + rmSync(dataDir, { recursive: true, force: true }); + if (originalDataDir === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = originalDataDir; + } + }); + + it('clears cooldown markers when a background probe succeeds', async () => { + const site = await db.insert(schema.sites).values({ + name: 'recovery-site', + url: 'https://recovery-site.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'recovery-user', + accessToken: 'access-recovery', + apiToken: 'sk-recovery', + status: 'active', + }).returning().get(); + + const token = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: 'default', + token: 'sk-recovery-token', + enabled: true, + isDefault: true, + }).returning().get(); + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4', + enabled: true, + }).returning().get(); + + const channel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: account.id, + tokenId: token.id, + enabled: true, + cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(), + lastFailAt: new Date().toISOString(), + consecutiveFailCount: 2, + cooldownLevel: 1, + }).returning().get(); + + await runChannelRecoveryProbeSweep(); + + expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1); + expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({ + modelName: 'gpt-5.4', + tokenValue: 'sk-recovery-token', + }); + + const refreshed = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, channel.id)) + .get(); + expect(refreshed?.cooldownUntil).toBeNull(); + expect(refreshed?.lastFailAt).toBeNull(); + expect(refreshed?.consecutiveFailCount).toBe(0); + expect(refreshed?.cooldownLevel).toBe(0); + }); + + it('also probes active leased channels in the background', async () => { + const site = await db.insert(schema.sites).values({ + name: 'active-site', + url: 'https://active-site.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'active-user', + accessToken: 'access-active', + apiToken: 'sk-active', + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'session', + }), + }).returning().get(); + + const token = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: 'default', + token: 'sk-active-token', + enabled: true, + isDefault: true, + }).returning().get(); + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.2', + enabled: true, + }).returning().get(); + + const channel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: account.id, + tokenId: token.id, + enabled: true, + }).returning().get(); + + const lease = await proxyChannelCoordinator.acquireChannelLease({ + channelId: channel.id, + accountExtraConfig: account.extraConfig, + }); + expect(lease.status).toBe('acquired'); + if (lease.status !== 'acquired') return; + + await runChannelRecoveryProbeSweep(); + + expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1); + expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({ + modelName: 'gpt-5.2', + tokenValue: 'sk-active-token', + }); + + lease.lease.release(); + }); + + it('skips provider-directed quota cooldown channels during recovery sweeps', async () => { + const site = await db.insert(schema.sites).values({ + name: 'quota-site', + url: 'https://quota-site.example.com', + platform: 'codex', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'quota-user', + accessToken: 'access-quota', + apiToken: 'sk-quota', + status: 'active', + }).returning().get(); + + const quotaToken = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: 'quota-token', + token: 'sk-quota-token', + enabled: true, + isDefault: false, + }).returning().get(); + + const retryToken = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: 'retry-token', + token: 'sk-retry-token', + enabled: true, + isDefault: false, + }).returning().get(); + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4', + enabled: true, + }).returning().get(); + + await db.insert(schema.routeChannels).values([ + { + routeId: route.id, + accountId: account.id, + tokenId: quotaToken.id, + enabled: true, + cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(), + lastFailAt: new Date().toISOString(), + failCount: 0, + consecutiveFailCount: 0, + cooldownLevel: 0, + }, + { + routeId: route.id, + accountId: account.id, + tokenId: retryToken.id, + enabled: true, + cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(), + lastFailAt: new Date().toISOString(), + failCount: 2, + consecutiveFailCount: 0, + cooldownLevel: 0, + }, + ]).run(); + + await runChannelRecoveryProbeSweep(); + + expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1); + expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({ + tokenValue: 'sk-retry-token', + modelName: 'gpt-5.4', + }); + }); + + it('prioritizes never-probed active channels before reprobing recently started ones', async () => { + const site = await db.insert(schema.sites).values({ + name: 'priority-site', + url: 'https://priority-site.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const leases: Array<{ release: () => void }> = []; + try { + for (let index = 1; index <= 5; index += 1) { + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: `priority-user-${index}`, + accessToken: `access-priority-${index}`, + apiToken: `sk-priority-${index}`, + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'session', + }), + }).returning().get(); + + const token = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: `token-${index}`, + token: `sk-priority-token-${index}`, + enabled: true, + isDefault: true, + }).returning().get(); + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: `gpt-5.4-${index}`, + enabled: true, + }).returning().get(); + + const channel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: account.id, + tokenId: token.id, + enabled: true, + }).returning().get(); + + const lease = await proxyChannelCoordinator.acquireChannelLease({ + channelId: channel.id, + accountExtraConfig: account.extraConfig, + }); + expect(lease.status).toBe('acquired'); + if (lease.status === 'acquired') { + leases.push(lease.lease); + } + } + + const startedAt = Date.UTC(2026, 3, 1, 0, 0, 0); + await runChannelRecoveryProbeSweep(startedAt); + + expect(probeRuntimeModelMock).toHaveBeenCalledTimes(4); + expect(probeRuntimeModelMock.mock.calls.map((call) => call[0]?.tokenValue)).not.toContain('sk-priority-token-5'); + + probeRuntimeModelMock.mockClear(); + + await runChannelRecoveryProbeSweep(startedAt + 5 * 60 * 1000); + + expect(probeRuntimeModelMock).toHaveBeenCalledTimes(4); + const secondSweepTokens = probeRuntimeModelMock.mock.calls.map((call) => call[0]?.tokenValue); + expect(secondSweepTokens).toContain('sk-priority-token-5'); + } finally { + for (const lease of leases) { + lease.release(); + } + } + }); +}); diff --git a/src/server/services/channelRecoveryProbeService.ts b/src/server/services/channelRecoveryProbeService.ts new file mode 100644 index 0000000..e1750c1 --- /dev/null +++ b/src/server/services/channelRecoveryProbeService.ts @@ -0,0 +1,299 @@ +import { and, eq, gt, inArray, isNotNull } from 'drizzle-orm'; +import { db, schema } from '../db/index.js'; +import { isUsableAccountToken } from './accountTokenService.js'; +import { getOauthInfoFromAccount } from './oauth/oauthAccount.js'; +import { proxyChannelCoordinator } from './proxyChannelCoordinator.js'; +import { probeRuntimeModel } from './runtimeModelProbe.js'; +import { tokenRouter } from './tokenRouter.js'; +import { isExactTokenRouteModelPattern } from '../../shared/tokenRoutePatterns.js'; + +type RecoveryProbeSource = 'cooldown' | 'active'; + +type RecoveryProbeCandidate = { + source: RecoveryProbeSource; + channelId: number; + modelName: string; + tokenValue: string; + account: typeof schema.accounts.$inferSelect; + site: typeof schema.sites.$inferSelect; +}; + +const CHANNEL_RECOVERY_SWEEP_INTERVAL_MS = 30_000; +const CHANNEL_RECOVERY_PROBE_TIMEOUT_MS = 12_000; +// Keep recovery probes conservative so they do not look like bulk health checks to upstream providers. +const CHANNEL_RECOVERY_PROBE_CONCURRENCY = 1; +const CHANNEL_RECOVERY_MAX_BATCH = 4; +const CHANNEL_RECOVERY_COOLDOWN_RECHECK_MS = 30_000; +const CHANNEL_RECOVERY_ACTIVE_RECHECK_MS = 5 * 60_000; + +let recoveryProbeSchedulerTimer: ReturnType | null = null; +let recoveryProbeSweepInFlight: Promise | null = null; +const recoveryProbeInFlightKeys = new Set(); +const recoveryProbeLastStartedAtByKey = new Map(); + +function shouldUnrefTimer(timer: ReturnType | ReturnType) { + if (typeof (timer as { unref?: () => void }).unref === 'function') { + (timer as { unref: () => void }).unref(); + } +} + +function buildRecoveryProbeKey(channelId: number, modelName: string): string { + return `${channelId}:${String(modelName || '').trim().toLowerCase()}`; +} + +function resolveRecoveryProbeWindowMs(source: RecoveryProbeSource): number { + return source === 'cooldown' + ? CHANNEL_RECOVERY_COOLDOWN_RECHECK_MS + : CHANNEL_RECOVERY_ACTIVE_RECHECK_MS; +} + +function resolveProbeModelName(row: { + route_channels: typeof schema.routeChannels.$inferSelect; + token_routes: typeof schema.tokenRoutes.$inferSelect; +}): string { + const sourceModel = String(row.route_channels.sourceModel || '').trim(); + if (sourceModel) return sourceModel; + const routeModelPattern = String(row.token_routes.modelPattern || '').trim(); + return isExactTokenRouteModelPattern(routeModelPattern) ? routeModelPattern : ''; +} + +function resolveProbeTokenValue(row: { + route_channels: typeof schema.routeChannels.$inferSelect; + accounts: typeof schema.accounts.$inferSelect; + account_tokens: typeof schema.accountTokens.$inferSelect | null; +}): string | null { + if (typeof row.route_channels.tokenId === 'number' && row.route_channels.tokenId > 0) { + if (!row.account_tokens || !isUsableAccountToken(row.account_tokens)) return null; + const tokenValue = String(row.account_tokens.token || '').trim(); + return tokenValue || null; + } + + if (getOauthInfoFromAccount(row.accounts)) { + const accessToken = String(row.accounts.accessToken || '').trim(); + return accessToken || null; + } + + const fallbackApiToken = String(row.accounts.apiToken || '').trim(); + return fallbackApiToken || null; +} + +function isProviderDirectedCooldown(row: { + route_channels: typeof schema.routeChannels.$inferSelect; +}): boolean { + return !!row.route_channels.cooldownUntil + && (row.route_channels.failCount ?? 0) <= 0 + && (row.route_channels.consecutiveFailCount ?? 0) <= 0 + && (row.route_channels.cooldownLevel ?? 0) <= 0; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T, index: number) => Promise, +): Promise { + const safeConcurrency = Math.max(1, Math.min(items.length || 1, Math.trunc(concurrency || 1))); + let nextIndex = 0; + const runWorker = async () => { + while (true) { + const currentIndex = nextIndex; + nextIndex += 1; + if (currentIndex >= items.length) return; + await worker(items[currentIndex] as T, currentIndex); + } + }; + + await Promise.all(Array.from({ length: safeConcurrency }, () => runWorker())); +} + +async function loadCoolingProbeCandidates(nowIso: string): Promise { + const rows = await db.select() + .from(schema.routeChannels) + .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id)) + .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id)) + .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id)) + .leftJoin(schema.accountTokens, eq(schema.routeChannels.tokenId, schema.accountTokens.id)) + .where(and( + eq(schema.routeChannels.enabled, true), + eq(schema.accounts.status, 'active'), + eq(schema.sites.status, 'active'), + isNotNull(schema.routeChannels.cooldownUntil), + gt(schema.routeChannels.cooldownUntil, nowIso), + )) + .all(); + + return rows.flatMap((row) => { + if (isProviderDirectedCooldown(row)) return []; + const modelName = resolveProbeModelName(row); + const tokenValue = resolveProbeTokenValue(row); + if (!modelName || !tokenValue) return []; + return [{ + source: 'cooldown' as const, + channelId: row.route_channels.id, + modelName, + tokenValue, + account: row.accounts, + site: row.sites, + }]; + }); +} + +async function loadActiveProbeCandidates(activeChannelIds: number[]): Promise { + if (activeChannelIds.length <= 0) return []; + + const rows = await db.select() + .from(schema.routeChannels) + .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id)) + .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id)) + .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id)) + .leftJoin(schema.accountTokens, eq(schema.routeChannels.tokenId, schema.accountTokens.id)) + .where(and( + eq(schema.routeChannels.enabled, true), + eq(schema.accounts.status, 'active'), + eq(schema.sites.status, 'active'), + inArray(schema.routeChannels.id, activeChannelIds), + )) + .all(); + + return rows.flatMap((row) => { + const modelName = resolveProbeModelName(row); + const tokenValue = resolveProbeTokenValue(row); + if (!modelName || !tokenValue) return []; + return [{ + source: 'active' as const, + channelId: row.route_channels.id, + modelName, + tokenValue, + account: row.accounts, + site: row.sites, + }]; + }); +} + +function mergeRecoveryProbeCandidates(candidates: RecoveryProbeCandidate[]): RecoveryProbeCandidate[] { + const merged = new Map(); + for (const candidate of candidates) { + const existing = merged.get(candidate.channelId); + if (!existing || (existing.source === 'active' && candidate.source === 'cooldown')) { + merged.set(candidate.channelId, candidate); + } + } + return Array.from(merged.values()); +} + +function shouldProbeCandidate(candidate: RecoveryProbeCandidate, nowMs: number): boolean { + const key = buildRecoveryProbeKey(candidate.channelId, candidate.modelName); + if (recoveryProbeInFlightKeys.has(key)) return false; + const lastStartedAt = recoveryProbeLastStartedAtByKey.get(key) ?? 0; + return (nowMs - lastStartedAt) >= resolveRecoveryProbeWindowMs(candidate.source); +} + +function compareRecoveryProbeCandidatePriority(left: RecoveryProbeCandidate, right: RecoveryProbeCandidate): number { + const leftKey = buildRecoveryProbeKey(left.channelId, left.modelName); + const rightKey = buildRecoveryProbeKey(right.channelId, right.modelName); + const leftLastStartedAt = recoveryProbeLastStartedAtByKey.get(leftKey); + const rightLastStartedAt = recoveryProbeLastStartedAtByKey.get(rightKey); + + if (leftLastStartedAt == null && rightLastStartedAt == null) { + return left.channelId - right.channelId; + } + if (leftLastStartedAt == null) return -1; + if (rightLastStartedAt == null) return 1; + if (leftLastStartedAt !== rightLastStartedAt) { + return leftLastStartedAt - rightLastStartedAt; + } + return left.channelId - right.channelId; +} + +async function runRecoveryProbeCandidate(candidate: RecoveryProbeCandidate, nowMs: number): Promise { + const key = buildRecoveryProbeKey(candidate.channelId, candidate.modelName); + recoveryProbeInFlightKeys.add(key); + recoveryProbeLastStartedAtByKey.set(key, nowMs); + try { + const result = await probeRuntimeModel({ + site: candidate.site, + account: candidate.account, + modelName: candidate.modelName, + tokenValue: candidate.tokenValue, + timeoutMs: CHANNEL_RECOVERY_PROBE_TIMEOUT_MS, + }); + if (result.status === 'supported') { + await tokenRouter.recordProbeSuccess( + candidate.channelId, + result.latencyMs ?? 0, + candidate.modelName, + ); + } + } catch (error) { + console.warn(`[channel-recovery-probe] channel ${candidate.channelId} probe failed`, error); + } finally { + recoveryProbeInFlightKeys.delete(key); + } +} + +export async function runChannelRecoveryProbeSweep(nowMs = Date.now()): Promise { + if (recoveryProbeSweepInFlight) { + await recoveryProbeSweepInFlight; + return; + } + + recoveryProbeSweepInFlight = (async () => { + const nowIso = new Date(nowMs).toISOString(); + const activeChannelIds = proxyChannelCoordinator.getActiveChannelIds(); + const [coolingCandidates, activeCandidates] = await Promise.all([ + loadCoolingProbeCandidates(nowIso), + loadActiveProbeCandidates(activeChannelIds), + ]); + + const merged = mergeRecoveryProbeCandidates([ + ...coolingCandidates, + ...activeCandidates, + ]); + const dueCandidates = merged + .filter((candidate) => shouldProbeCandidate(candidate, nowMs)) + .sort(compareRecoveryProbeCandidatePriority) + .slice(0, CHANNEL_RECOVERY_MAX_BATCH); + if (dueCandidates.length <= 0) return; + + await mapWithConcurrency( + dueCandidates, + CHANNEL_RECOVERY_PROBE_CONCURRENCY, + async (candidate) => runRecoveryProbeCandidate(candidate, nowMs), + ); + })().finally(() => { + recoveryProbeSweepInFlight = null; + }); + + await recoveryProbeSweepInFlight; +} + +export function startChannelRecoveryProbeScheduler(intervalMs = CHANNEL_RECOVERY_SWEEP_INTERVAL_MS) { + stopChannelRecoveryProbeScheduler(); + const safeIntervalMs = Math.max(10_000, Math.trunc(intervalMs || 0)); + recoveryProbeSchedulerTimer = setInterval(() => { + void runChannelRecoveryProbeSweep().catch((error) => { + console.warn('[channel-recovery-probe] background sweep failed', error); + }); + }, safeIntervalMs); + shouldUnrefTimer(recoveryProbeSchedulerTimer); + void runChannelRecoveryProbeSweep().catch((error) => { + console.warn('[channel-recovery-probe] initial sweep failed', error); + }); + return { + enabled: true, + intervalMs: safeIntervalMs, + }; +} + +export function stopChannelRecoveryProbeScheduler() { + if (recoveryProbeSchedulerTimer) { + clearInterval(recoveryProbeSchedulerTimer); + recoveryProbeSchedulerTimer = null; + } +} + +export function resetChannelRecoveryProbeState() { + stopChannelRecoveryProbeScheduler(); + recoveryProbeSweepInFlight = null; + recoveryProbeInFlightKeys.clear(); + recoveryProbeLastStartedAtByKey.clear(); +} diff --git a/src/server/services/modelAvailabilityProbeService.test.ts b/src/server/services/modelAvailabilityProbeService.test.ts new file mode 100644 index 0000000..c0bb56b --- /dev/null +++ b/src/server/services/modelAvailabilityProbeService.test.ts @@ -0,0 +1,507 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { eq } from 'drizzle-orm'; + +const resolveUpstreamEndpointCandidatesMock = vi.fn(); +const buildUpstreamEndpointRequestMock = vi.fn(); +const dispatchRuntimeRequestMock = vi.fn(); +const resolveChannelProxyUrlMock = vi.fn(); +const withSiteRecordProxyRequestInitMock = vi.fn(); +const rebuildRoutesOnlyMock = vi.fn(); + +vi.mock('./upstreamEndpointRuntime.js', () => ({ + resolveUpstreamEndpointCandidates: (...args: unknown[]) => resolveUpstreamEndpointCandidatesMock(...args), + buildUpstreamEndpointRequest: (...args: unknown[]) => buildUpstreamEndpointRequestMock(...args), +})); + +vi.mock('./runtimeDispatch.js', () => ({ + dispatchRuntimeRequest: (...args: unknown[]) => dispatchRuntimeRequestMock(...args), +})); + +vi.mock('./siteProxy.js', () => ({ + resolveChannelProxyUrl: (...args: unknown[]) => resolveChannelProxyUrlMock(...args), + withSiteRecordProxyRequestInit: (...args: unknown[]) => withSiteRecordProxyRequestInitMock(...args), +})); + +vi.mock('./routeRefreshWorkflow.js', () => ({ + rebuildRoutesOnly: (...args: unknown[]) => rebuildRoutesOnlyMock(...args), +})); + +type DbModule = typeof import('../db/index.js'); +type ProbeModule = typeof import('./modelAvailabilityProbeService.js'); + +describe('modelAvailabilityProbeService', () => { + let db: DbModule['db']; + let schema: DbModule['schema']; + let executeModelAvailabilityProbe: ProbeModule['executeModelAvailabilityProbe']; + let resetProbeExecutionState: ProbeModule['__resetModelAvailabilityProbeExecutionStateForTests']; + let dataDir = ''; + let originalDataDir: string | undefined; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-model-probe-')); + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = dataDir; + + await import('../db/migrate.js'); + const dbModule = await import('../db/index.js'); + const probeModule = await import('./modelAvailabilityProbeService.js'); + + db = dbModule.db; + schema = dbModule.schema; + executeModelAvailabilityProbe = probeModule.executeModelAvailabilityProbe; + resetProbeExecutionState = probeModule.__resetModelAvailabilityProbeExecutionStateForTests; + }); + + beforeEach(async () => { + resolveUpstreamEndpointCandidatesMock.mockReset(); + buildUpstreamEndpointRequestMock.mockReset(); + dispatchRuntimeRequestMock.mockReset(); + resolveChannelProxyUrlMock.mockReset(); + withSiteRecordProxyRequestInitMock.mockReset(); + rebuildRoutesOnlyMock.mockReset(); + rebuildRoutesOnlyMock.mockResolvedValue(undefined); + resetProbeExecutionState(); + + resolveUpstreamEndpointCandidatesMock.mockResolvedValue(['chat']); + buildUpstreamEndpointRequestMock.mockImplementation((input: { modelName?: string; endpoint?: string }) => ({ + path: '/v1/chat/completions', + headers: { + 'content-type': 'application/json', + }, + body: { + model: input.modelName || 'unknown-model', + endpoint: input.endpoint || 'chat', + }, + runtime: { + executor: 'default', + modelName: input.modelName || 'unknown-model', + stream: false, + }, + })); + dispatchRuntimeRequestMock.mockImplementation(async (input: { request?: { body?: { model?: string } } }) => { + const modelName = String(input?.request?.body?.model || ''); + if (modelName === 'gpt-ghost') { + return new Response(JSON.stringify({ + error: { + message: 'The model `gpt-ghost` does not exist', + }, + }), { + status: 404, + headers: { + 'content-type': 'application/json', + }, + }); + } + return new Response(JSON.stringify({ + id: 'resp_probe', + model: modelName, + choices: [ + { + message: { + role: 'assistant', + content: 'OK', + }, + }, + ], + }), { + status: 200, + headers: { + 'content-type': 'application/json', + }, + }); + }); + resolveChannelProxyUrlMock.mockReturnValue(null); + withSiteRecordProxyRequestInitMock.mockImplementation((_site: unknown, init: unknown) => init); + + await db.delete(schema.routeChannels).run(); + await db.delete(schema.tokenRoutes).run(); + await db.delete(schema.tokenModelAvailability).run(); + await db.delete(schema.modelAvailability).run(); + await db.delete(schema.accountTokens).run(); + await db.delete(schema.accounts).run(); + await db.delete(schema.sites).run(); + }); + + afterAll(() => { + if (originalDataDir === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = originalDataDir; + } + }); + + it('marks supported rows true and unsupported rows false after probing', async () => { + const site = await db.insert(schema.sites).values({ + name: 'probe-site', + url: 'https://probe.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'probe-user', + accessToken: '', + apiToken: 'sk-account-probe', + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'apikey', + }), + }).returning().get(); + + const token = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: 'default', + token: 'sk-token-probe', + enabled: true, + isDefault: true, + valueStatus: 'ready', + }).returning().get(); + + await db.insert(schema.modelAvailability).values([ + { + accountId: account.id, + modelName: 'gpt-real', + available: true, + checkedAt: '2026-03-20T00:00:00.000Z', + }, + ]).run(); + + await db.insert(schema.tokenModelAvailability).values([ + { + tokenId: token.id, + modelName: 'gpt-ghost', + available: true, + checkedAt: '2026-03-20T00:00:00.000Z', + }, + ]).run(); + + const result = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: false, + }); + + expect(result.summary).toMatchObject({ + totalAccounts: 1, + scanned: 2, + supported: 1, + unsupported: 1, + inconclusive: 0, + skippedModels: 0, + updatedRows: 2, + rebuiltRoutes: false, + }); + + const accountRows = await db.select().from(schema.modelAvailability) + .where(eq(schema.modelAvailability.accountId, account.id)) + .all(); + expect(accountRows).toHaveLength(1); + expect(accountRows[0]?.available).toBe(true); + + const tokenRows = await db.select().from(schema.tokenModelAvailability) + .where(eq(schema.tokenModelAvailability.tokenId, token.id)) + .all(); + expect(tokenRows).toHaveLength(1); + expect(tokenRows[0]?.available).toBe(false); + }); + + it('skips probing obvious non-conversation models', async () => { + const site = await db.insert(schema.sites).values({ + name: 'probe-site', + url: 'https://probe.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'probe-user', + accessToken: '', + apiToken: 'sk-account-probe', + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'apikey', + }), + }).returning().get(); + + await db.insert(schema.modelAvailability).values([ + { + accountId: account.id, + modelName: 'text-embedding-3-large', + available: true, + checkedAt: '2026-03-20T00:00:00.000Z', + }, + ]).run(); + + const result = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: false, + }); + + expect(result.summary).toMatchObject({ + totalAccounts: 1, + scanned: 1, + supported: 0, + unsupported: 0, + inconclusive: 0, + skippedModels: 1, + updatedRows: 0, + }); + expect(dispatchRuntimeRequestMock).not.toHaveBeenCalled(); + }); + + it('does not mark a model unavailable when probe result is inconclusive', async () => { + dispatchRuntimeRequestMock.mockImplementation(async (input: { request?: { body?: { model?: string } } }) => { + const modelName = String(input?.request?.body?.model || ''); + if (modelName === 'gpt-flaky') { + return new Response(JSON.stringify({ + error: { + message: 'upstream temporary failure', + }, + }), { + status: 502, + headers: { + 'content-type': 'application/json', + }, + }); + } + return new Response(JSON.stringify({ + id: 'resp_probe', + model: modelName, + choices: [ + { + message: { + role: 'assistant', + content: 'OK', + }, + }, + ], + }), { + status: 200, + headers: { + 'content-type': 'application/json', + }, + }); + }); + + const site = await db.insert(schema.sites).values({ + name: 'probe-site', + url: 'https://probe.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'probe-user', + accessToken: '', + apiToken: 'sk-account-probe', + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'apikey', + }), + }).returning().get(); + + const checkedAt = '2026-03-20T00:00:00.000Z'; + await db.insert(schema.modelAvailability).values([ + { + accountId: account.id, + modelName: 'gpt-flaky', + available: true, + checkedAt, + }, + ]).run(); + + const result = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: false, + }); + + expect(result.summary).toMatchObject({ + totalAccounts: 1, + scanned: 1, + supported: 0, + unsupported: 0, + inconclusive: 1, + skippedModels: 0, + updatedRows: 0, + }); + + const accountRows = await db.select().from(schema.modelAvailability) + .where(eq(schema.modelAvailability.accountId, account.id)) + .all(); + expect(accountRows).toHaveLength(1); + expect(accountRows[0]?.available).toBe(true); + expect(accountRows[0]?.checkedAt).toBe(checkedAt); + }); + + it('does not rebuild routes when probe only refreshes checkedAt and latency', async () => { + const site = await db.insert(schema.sites).values({ + name: 'stable-site', + url: 'https://stable.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'stable-user', + accessToken: '', + apiToken: 'sk-stable', + status: 'active', + }).returning().get(); + + await db.insert(schema.modelAvailability).values({ + accountId: account.id, + modelName: 'gpt-stable', + available: true, + checkedAt: '2026-03-20T00:00:00.000Z', + latencyMs: 123, + }).run(); + + const result = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: true, + }); + + expect(result.summary.updatedRows).toBe(1); + expect(result.summary.rebuiltRoutes).toBe(false); + expect(rebuildRoutesOnlyMock).not.toHaveBeenCalled(); + }); + + it('rebuilds routes when probe flips availability state', async () => { + const site = await db.insert(schema.sites).values({ + name: 'changed-site', + url: 'https://changed.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'changed-user', + accessToken: '', + apiToken: 'sk-changed', + status: 'active', + }).returning().get(); + + await db.insert(schema.modelAvailability).values({ + accountId: account.id, + modelName: 'gpt-changed', + available: false, + checkedAt: '2026-03-20T00:00:00.000Z', + }).run(); + + const result = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: true, + }); + + expect(result.summary.updatedRows).toBe(1); + expect(result.summary.rebuiltRoutes).toBe(true); + expect(rebuildRoutesOnlyMock).toHaveBeenCalledTimes(1); + }); + + it('does not count inactive accounts in probe results', async () => { + const site = await db.insert(schema.sites).values({ + name: 'inactive-site', + url: 'https://inactive.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'inactive-user', + accessToken: '', + apiToken: 'sk-inactive', + status: 'disabled', + }).returning().get(); + + await db.insert(schema.modelAvailability).values({ + accountId: account.id, + modelName: 'gpt-inactive', + available: true, + checkedAt: '2026-03-20T00:00:00.000Z', + }).run(); + + const result = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: false, + }); + + expect(result.summary.totalAccounts).toBe(0); + expect(result.results).toEqual([]); + expect(dispatchRuntimeRequestMock).not.toHaveBeenCalled(); + }); + + it('skips an account when another probe already holds the execution lease', async () => { + const site = await db.insert(schema.sites).values({ + name: 'leased-site', + url: 'https://leased.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'leased-user', + accessToken: '', + apiToken: 'sk-leased', + status: 'active', + }).returning().get(); + + await db.insert(schema.modelAvailability).values({ + accountId: account.id, + modelName: 'gpt-leased', + available: true, + checkedAt: '2026-03-20T00:00:00.000Z', + }).run(); + + let releaseFirstProbe: (() => void) | null = null; + const firstProbeStarted = new Promise((resolve) => { + dispatchRuntimeRequestMock.mockImplementationOnce(async () => { + resolve(); + await new Promise((release) => { + releaseFirstProbe = release; + }); + return new Response(JSON.stringify({ + id: 'resp_probe', + choices: [{ message: { role: 'assistant', content: 'OK' } }], + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }); + }); + + const firstProbe = executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: false, + }); + await firstProbeStarted; + + const overlappingProbe = await executeModelAvailabilityProbe({ + accountId: account.id, + rebuildRoutes: false, + }); + + releaseFirstProbe?.(); + await firstProbe; + + expect(overlappingProbe.summary).toMatchObject({ + totalAccounts: 1, + skipped: 1, + scanned: 0, + }); + expect(overlappingProbe.results[0]).toMatchObject({ + accountId: account.id, + siteId: site.id, + status: 'skipped', + message: 'model availability probe already running for account', + }); + }); +}); diff --git a/src/server/services/modelAvailabilityProbeService.ts b/src/server/services/modelAvailabilityProbeService.ts new file mode 100644 index 0000000..8a6fdf3 --- /dev/null +++ b/src/server/services/modelAvailabilityProbeService.ts @@ -0,0 +1,447 @@ +import { and, asc, eq } from 'drizzle-orm'; +import { config } from '../config.js'; +import { db, schema } from '../db/index.js'; +import { startBackgroundTask } from './backgroundTaskService.js'; +import { isUsableAccountToken, ACCOUNT_TOKEN_VALUE_STATUS_READY } from './accountTokenService.js'; +import { probeRuntimeModel } from './runtimeModelProbe.js'; +import * as routeRefreshWorkflow from './routeRefreshWorkflow.js'; + +type ProbeStatus = 'supported' | 'unsupported' | 'inconclusive' | 'skipped'; + +type ProbeAccountTarget = { + kind: 'account'; + rowId: number; + modelName: string; + lastKnownAvailable: boolean; + account: typeof schema.accounts.$inferSelect; + site: typeof schema.sites.$inferSelect; +}; + +type ProbeTokenTarget = { + kind: 'token'; + rowId: number; + tokenId: number; + modelName: string; + tokenValue: string; + lastKnownAvailable: boolean; + account: typeof schema.accounts.$inferSelect; + site: typeof schema.sites.$inferSelect; +}; + +type ProbeTarget = ProbeAccountTarget | ProbeTokenTarget; +type ProbeAccountContext = { + account: typeof schema.accounts.$inferSelect; + site: typeof schema.sites.$inferSelect; +}; + +export type ModelAvailabilityProbeAccountResult = { + accountId: number; + siteId: number; + status: 'success' | 'failed' | 'skipped'; + scanned: number; + supported: number; + unsupported: number; + inconclusive: number; + skipped: number; + updatedRows: number; + message: string; +}; + +export type ModelAvailabilityProbeExecutionResult = { + results: ModelAvailabilityProbeAccountResult[]; + summary: { + totalAccounts: number; + success: number; + failed: number; + skipped: number; + scanned: number; + supported: number; + unsupported: number; + inconclusive: number; + skippedModels: number; + updatedRows: number; + rebuiltRoutes: boolean; + }; +}; + +let probeSchedulerTimer: ReturnType | null = null; +const probeAccountLeases = new Set(); + +async function mapWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T, index: number) => Promise, +): Promise { + const safeConcurrency = Math.max(1, Math.min(items.length || 1, Math.trunc(concurrency || 1))); + const results = new Array(items.length); + let nextIndex = 0; + + const runWorker = async () => { + while (true) { + const currentIndex = nextIndex; + nextIndex += 1; + if (currentIndex >= items.length) return; + results[currentIndex] = await worker(items[currentIndex] as T, currentIndex); + } + }; + + await Promise.all(Array.from({ length: safeConcurrency }, () => runWorker())); + return results; +} + +async function probeSingleTarget(target: ProbeTarget): Promise<{ + status: ProbeStatus; + latencyMs: number | null; + reason: string; +}> { + return await probeRuntimeModel({ + site: target.site, + account: target.account, + modelName: target.modelName, + timeoutMs: config.modelAvailabilityProbeTimeoutMs, + tokenValue: target.kind === 'token' ? target.tokenValue : undefined, + }); +} + +async function updateProbeRow(target: ProbeTarget, status: ProbeStatus, latencyMs: number | null): Promise<{ + touched: boolean; + availabilityChanged: boolean; +}> { + if (status === 'inconclusive' || status === 'skipped') { + return { + touched: false, + availabilityChanged: false, + }; + } + const nextAvailable = status === 'supported'; + const patch = { + available: nextAvailable, + latencyMs, + checkedAt: new Date().toISOString(), + }; + + if (target.kind === 'account') { + await db.update(schema.modelAvailability) + .set(patch) + .where(eq(schema.modelAvailability.id, target.rowId)) + .run(); + return { + touched: true, + availabilityChanged: target.lastKnownAvailable !== nextAvailable, + }; + } + + await db.update(schema.tokenModelAvailability) + .set(patch) + .where(eq(schema.tokenModelAvailability.id, target.rowId)) + .run(); + return { + touched: true, + availabilityChanged: target.lastKnownAvailable !== nextAvailable, + }; +} + +async function loadActiveProbeAccountContext(accountId: number): Promise { + const accountRow = await db.select() + .from(schema.accounts) + .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id)) + .where(eq(schema.accounts.id, accountId)) + .get(); + if (!accountRow) return null; + if ((accountRow.accounts.status || 'active') !== 'active') return null; + if ((accountRow.sites.status || 'active') !== 'active') return null; + return { + account: accountRow.accounts, + site: accountRow.sites, + }; +} + +async function loadProbeTargetsForAccount(context: ProbeAccountContext): Promise { + const targets: ProbeTarget[] = []; + const accountModels = await db.select() + .from(schema.modelAvailability) + .where(eq(schema.modelAvailability.accountId, context.account.id)) + .orderBy(asc(schema.modelAvailability.checkedAt)) + .all(); + for (const row of accountModels) { + if (row.isManual) continue; + targets.push({ + kind: 'account', + rowId: row.id, + modelName: row.modelName, + lastKnownAvailable: !!row.available, + account: context.account, + site: context.site, + }); + } + + const tokenRows = await db.select() + .from(schema.tokenModelAvailability) + .innerJoin(schema.accountTokens, eq(schema.tokenModelAvailability.tokenId, schema.accountTokens.id)) + .where(and( + eq(schema.accountTokens.accountId, context.account.id), + eq(schema.accountTokens.enabled, true), + eq(schema.accountTokens.valueStatus, ACCOUNT_TOKEN_VALUE_STATUS_READY), + )) + .orderBy(asc(schema.tokenModelAvailability.checkedAt)) + .all(); + for (const row of tokenRows) { + if (!isUsableAccountToken(row.account_tokens)) continue; + const tokenValue = String(row.account_tokens.token || '').trim(); + if (!tokenValue) continue; + targets.push({ + kind: 'token', + rowId: row.token_model_availability.id, + tokenId: row.account_tokens.id, + modelName: row.token_model_availability.modelName, + tokenValue, + lastKnownAvailable: !!row.token_model_availability.available, + account: context.account, + site: context.site, + }); + } + + return targets; +} + +function tryAcquireProbeAccountLease(accountId: number): boolean { + if (!Number.isFinite(accountId) || accountId <= 0) return false; + if (probeAccountLeases.has(accountId)) return false; + probeAccountLeases.add(accountId); + return true; +} + +function releaseProbeAccountLease(accountId: number): void { + probeAccountLeases.delete(accountId); +} + +function buildSkippedProbeAccountResult(input: { + accountId: number; + siteId: number; + message: string; +}): ModelAvailabilityProbeAccountResult { + return { + accountId: input.accountId, + siteId: input.siteId, + status: 'skipped', + scanned: 0, + supported: 0, + unsupported: 0, + inconclusive: 0, + skipped: 0, + updatedRows: 0, + message: input.message, + }; +} + +function summarizeProbeResults(results: ModelAvailabilityProbeAccountResult[], rebuiltRoutes: boolean): ModelAvailabilityProbeExecutionResult { + return { + results, + summary: { + totalAccounts: results.length, + success: results.filter((item) => item.status === 'success').length, + failed: results.filter((item) => item.status === 'failed').length, + skipped: results.filter((item) => item.status === 'skipped').length, + scanned: results.reduce((sum, item) => sum + item.scanned, 0), + supported: results.reduce((sum, item) => sum + item.supported, 0), + unsupported: results.reduce((sum, item) => sum + item.unsupported, 0), + inconclusive: results.reduce((sum, item) => sum + item.inconclusive, 0), + skippedModels: results.reduce((sum, item) => sum + item.skipped, 0), + updatedRows: results.reduce((sum, item) => sum + item.updatedRows, 0), + rebuiltRoutes, + }, + }; +} + +export async function executeModelAvailabilityProbe(input: { + accountId?: number; + rebuildRoutes?: boolean; +} = {}): Promise { + const accountIds = input.accountId + ? [input.accountId] + : (await db.select({ id: schema.accounts.id }) + .from(schema.accounts) + .where(eq(schema.accounts.status, 'active')) + .all()).map((row) => row.id); + + const results: ModelAvailabilityProbeAccountResult[] = []; + let shouldRebuildRoutes = false; + + for (const accountId of accountIds) { + const context = await loadActiveProbeAccountContext(accountId); + if (!context) { + continue; + } + if (!tryAcquireProbeAccountLease(accountId)) { + results.push(buildSkippedProbeAccountResult({ + accountId, + siteId: context.site.id, + message: 'model availability probe already running for account', + })); + continue; + } + + try { + const targets = await loadProbeTargetsForAccount(context); + if (targets.length <= 0) { + results.push(buildSkippedProbeAccountResult({ + accountId, + siteId: context.site.id, + message: 'no discovered models to probe', + })); + continue; + } + + let supported = 0; + let unsupported = 0; + let inconclusive = 0; + let skipped = 0; + let updatedRows = 0; + let failed = false; + + const probeOutcomes = await mapWithConcurrency( + targets, + config.modelAvailabilityProbeConcurrency, + async (target) => { + try { + const probe = await probeSingleTarget(target); + const update = await updateProbeRow(target, probe.status, probe.latencyMs); + return { + target, + probe, + touched: update.touched, + availabilityChanged: update.availabilityChanged, + failed: false, + }; + } catch (error) { + console.warn(`[model-probe] account ${accountId} model ${target.modelName} probe failed`, error); + return { + target, + probe: { + status: 'inconclusive' as const, + latencyMs: null, + reason: error instanceof Error ? error.message : 'probe failed', + }, + touched: false, + availabilityChanged: false, + failed: true, + }; + } + }, + ); + + for (const outcome of probeOutcomes) { + if (outcome.probe.status === 'supported') supported += 1; + if (outcome.probe.status === 'unsupported') unsupported += 1; + if (outcome.probe.status === 'inconclusive') inconclusive += 1; + if (outcome.probe.status === 'skipped') skipped += 1; + if (outcome.touched) { + updatedRows += 1; + } + if (outcome.availabilityChanged) { + shouldRebuildRoutes = true; + } + if (outcome.failed) { + failed = true; + } + } + + results.push({ + accountId, + siteId: context.site.id, + status: failed ? 'failed' : 'success', + scanned: targets.length, + supported, + unsupported, + inconclusive, + skipped, + updatedRows, + message: failed + ? 'model availability probe finished with partial failures' + : 'model availability probe finished', + }); + } finally { + releaseProbeAccountLease(accountId); + } + } + + let rebuiltRoutes = false; + if (input.rebuildRoutes !== false && shouldRebuildRoutes) { + await routeRefreshWorkflow.rebuildRoutesOnly(); + rebuiltRoutes = true; + } + + return summarizeProbeResults(results, rebuiltRoutes); +} + +export function buildModelAvailabilityProbeTaskDedupeKey(accountId?: number | null): string { + const normalizedAccountId = Number.isFinite(accountId as number) && Number(accountId) > 0 + ? Math.trunc(Number(accountId)) + : null; + return normalizedAccountId + ? `model-availability-probe-${normalizedAccountId}` + : 'model-availability-probe-all'; +} + +export function queueModelAvailabilityProbeTask(input: { + accountId?: number; + title?: string; +}) { + const accountId = Number.isFinite(input.accountId as number) ? Math.trunc(input.accountId as number) : null; + const title = input.title || (accountId + ? `探测模型可用性 #${accountId}` + : '探测模型可用性'); + const dedupeKey = buildModelAvailabilityProbeTaskDedupeKey(accountId); + + return startBackgroundTask( + { + type: 'model-probe', + title, + dedupeKey, + notifyOnFailure: true, + successMessage: (currentTask) => { + const summary = (currentTask.result as ModelAvailabilityProbeExecutionResult | undefined)?.summary; + if (!summary) return `${title}已完成`; + return `${title}完成：探测 ${summary.scanned}，可用 ${summary.supported}，不可用 ${summary.unsupported}，不确定 ${summary.inconclusive}`; + }, + failureMessage: (currentTask) => `${title}失败：${currentTask.error || 'unknown error'}`, + }, + async () => executeModelAvailabilityProbe({ + accountId: accountId ?? undefined, + rebuildRoutes: true, + }), + ); +} + +export function startModelAvailabilityProbeScheduler(intervalMs = config.modelAvailabilityProbeIntervalMs) { + stopModelAvailabilityProbeScheduler(); + if (!config.modelAvailabilityProbeEnabled) { + return { + enabled: false, + intervalMs: 0, + }; + } + + const safeIntervalMs = Math.max(60_000, Math.trunc(intervalMs || 0)); + probeSchedulerTimer = setInterval(() => { + void queueModelAvailabilityProbeTask({ + title: '后台模型可用性探测', + }); + }, safeIntervalMs); + probeSchedulerTimer.unref?.(); + return { + enabled: true, + intervalMs: safeIntervalMs, + }; +} + +export function stopModelAvailabilityProbeScheduler() { + if (probeSchedulerTimer) { + clearInterval(probeSchedulerTimer); + probeSchedulerTimer = null; + } +} + +export function __resetModelAvailabilityProbeExecutionStateForTests(): void { + probeAccountLeases.clear(); +} diff --git a/src/server/services/proxyChannelCoordinator.test.ts b/src/server/services/proxyChannelCoordinator.test.ts index dc9d237..8e89007 100644 --- a/src/server/services/proxyChannelCoordinator.test.ts +++ b/src/server/services/proxyChannelCoordinator.test.ts @@ -162,4 +162,71 @@ describe('proxyChannelCoordinator', () => { second.lease.release(); } }); + + it('exposes the set of currently active leased channels', async () => { + const lease = await proxyChannelCoordinator.acquireChannelLease({ + channelId: 23, + accountExtraConfig: JSON.stringify({ credentialMode: 'session' }), + }); + expect(lease.status).toBe('acquired'); + if (lease.status !== 'acquired') return; + + expect(proxyChannelCoordinator.getActiveChannelIds()).toEqual([23]); + + lease.lease.release(); + expect(proxyChannelCoordinator.getActiveChannelIds()).toEqual([]); + }); + + it('reports active and waiting load for a guarded session channel', async () => { + const first = await proxyChannelCoordinator.acquireChannelLease({ + channelId: 31, + accountExtraConfig: JSON.stringify({ credentialMode: 'session' }), + }); + expect(first.status).toBe('acquired'); + if (first.status !== 'acquired') return; + + const secondPromise = proxyChannelCoordinator.acquireChannelLease({ + channelId: 31, + accountExtraConfig: JSON.stringify({ credentialMode: 'session' }), + }); + await vi.advanceTimersByTimeAsync(0); + + expect(proxyChannelCoordinator.getChannelLoadSnapshot({ + channelId: 31, + accountExtraConfig: JSON.stringify({ credentialMode: 'session' }), + })).toEqual({ + channelId: 31, + sessionScoped: true, + concurrencyLimit: 1, + activeLeaseCount: 1, + waitingCount: 1, + loadRatio: 2, + saturated: true, + }); + + first.lease.release(); + await vi.advanceTimersByTimeAsync(0); + + const second = await secondPromise; + expect(second.status).toBe('acquired'); + if (second.status === 'acquired') { + second.lease.release(); + } + }); + + it('treats structured oauth providers as session-scoped in load snapshots', () => { + expect(proxyChannelCoordinator.getChannelLoadSnapshot({ + channelId: 41, + accountExtraConfig: JSON.stringify({ credentialMode: 'session' }), + accountOauthProvider: 'codex', + })).toEqual({ + channelId: 41, + sessionScoped: true, + concurrencyLimit: 1, + activeLeaseCount: 0, + waitingCount: 0, + loadRatio: 0, + saturated: false, + }); + }); }); diff --git a/src/server/services/proxyChannelCoordinator.ts b/src/server/services/proxyChannelCoordinator.ts index bf3be5f..1a72bc6 100644 --- a/src/server/services/proxyChannelCoordinator.ts +++ b/src/server/services/proxyChannelCoordinator.ts @@ -24,6 +24,16 @@ type ChannelRuntimeState = { queue: ChannelWaiter[]; }; +export type ProxyChannelLoadSnapshot = { + channelId: number; + sessionScoped: boolean; + concurrencyLimit: number; + activeLeaseCount: number; + waitingCount: number; + loadRatio: number; + saturated: boolean; +}; + export type ProxyChannelLease = { channelId: number; isActive(): boolean; @@ -104,9 +114,15 @@ function getOrCreateChannelRuntimeState(channelId: number): ChannelRuntimeState return state; } +function pruneCancelledWaiters(state: ChannelRuntimeState): void { + if (state.queue.length <= 0) return; + state.queue = state.queue.filter((waiter) => !waiter.cancelled); +} + function maybeDeleteChannelRuntimeState(channelId: number): void { const state = channelRuntimeStates.get(channelId); if (!state) return; + pruneCancelledWaiters(state); if (state.activeLeaseIds.size <= 0 && state.queue.every((waiter) => waiter.cancelled)) { channelRuntimeStates.delete(channelId); } @@ -177,6 +193,62 @@ class ProxyChannelCoordinator { stickySessionBindings.delete(normalizedKey); } + getActiveChannelIds(): number[] { + const ids: number[] = []; + for (const [channelId, state] of channelRuntimeStates.entries()) { + pruneCancelledWaiters(state); + if (state.activeLeaseIds.size > 0) { + ids.push(channelId); + } + } + return ids; + } + + getChannelLoadSnapshot(input: { + channelId: number; + accountExtraConfig?: string | null; + accountOauthProvider?: string | null; + }): ProxyChannelLoadSnapshot { + const channelId = Math.trunc(input.channelId || 0); + const sessionScoped = isSessionScopedChannel({ + extraConfig: input.accountExtraConfig, + oauthProvider: input.accountOauthProvider, + }); + const concurrencyLimit = getChannelConcurrencyLimit({ + extraConfig: input.accountExtraConfig, + oauthProvider: input.accountOauthProvider, + }); + const state = channelId > 0 ? channelRuntimeStates.get(channelId) : null; + if (state) { + pruneCancelledWaiters(state); + } + const activeLeaseCount = state?.activeLeaseIds.size ?? 0; + const waitingCount = state?.queue.length ?? 0; + const denominator = concurrencyLimit > 0 ? concurrencyLimit : 1; + return { + channelId, + sessionScoped, + concurrencyLimit, + activeLeaseCount, + waitingCount, + loadRatio: (activeLeaseCount + waitingCount) / denominator, + saturated: concurrencyLimit > 0 && activeLeaseCount >= concurrencyLimit, + }; + } + + getChannelLoadSnapshots(input: Array<{ + channelId: number; + accountExtraConfig?: string | null; + accountOauthProvider?: string | null; + }>): Map { + const snapshots = new Map(); + for (const item of input) { + const snapshot = this.getChannelLoadSnapshot(item); + snapshots.set(snapshot.channelId, snapshot); + } + return snapshots; + } + async acquireChannelLease(input: { channelId: number; accountExtraConfig?: string | null; @@ -202,6 +274,7 @@ class ProxyChannelCoordinator { } const state = getOrCreateChannelRuntimeState(channelId); + pruneCancelledWaiters(state); if (state.activeLeaseIds.size < concurrencyLimit) { return { status: 'acquired', @@ -226,6 +299,7 @@ class ProxyChannelCoordinator { waiter.timer = setTimeout(() => { waiter.cancelled = true; waiter.timer = null; + pruneCancelledWaiters(state); maybeDeleteChannelRuntimeState(channelId); resolve({ status: 'timeout', @@ -285,6 +359,7 @@ class ProxyChannelCoordinator { private drainQueue(channelId: number): void { const state = channelRuntimeStates.get(channelId); if (!state) return; + pruneCancelledWaiters(state); const concurrencyLimit = Math.max(0, Math.trunc(config.proxySessionChannelConcurrencyLimit || 0)); while (state.activeLeaseIds.size < concurrencyLimit && state.queue.length > 0) { const waiter = state.queue.shift(); diff --git a/src/server/services/proxyDebugTraceRuntime.ts b/src/server/services/proxyDebugTraceRuntime.ts index 3f84de7..df3f3f4 100644 --- a/src/server/services/proxyDebugTraceRuntime.ts +++ b/src/server/services/proxyDebugTraceRuntime.ts @@ -1,7 +1,7 @@ import { readRuntimeResponseText } from '../proxy-core/executors/types.js'; import type { EndpointAttemptSuccessContext, -} from '../routes/proxy/endpointFlow.js'; +} from '../proxy-core/orchestration/endpointFlow.js'; import { finalizeProxyDebugTrace, insertProxyDebugAttempt, diff --git a/src/server/services/proxyRetryPolicy.test.ts b/src/server/services/proxyRetryPolicy.test.ts index 6318bdb..e262388 100644 --- a/src/server/services/proxyRetryPolicy.test.ts +++ b/src/server/services/proxyRetryPolicy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { shouldRetryProxyRequest } from './proxyRetryPolicy.js'; +import { shouldAbortSameSiteEndpointFallback, shouldRetryProxyRequest } from './proxyRetryPolicy.js'; describe('proxyRetryPolicy', () => { it('retries on rate limit and server errors', () => { @@ -52,4 +52,16 @@ describe('proxyRetryPolicy', () => { shouldRetryProxyRequest(400, '{"error":{"message":"invalid timeout parameter"}}'), ).toBe(false); }); + + it('aborts same-site endpoint fallback on rate-limit and quota responses', () => { + expect( + shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"rate limit exceeded"}}'), + ).toBe(true); + expect( + shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"quota exceeded"}}'), + ).toBe(true); + expect( + shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"too many requests"}}'), + ).toBe(true); + }); }); diff --git a/src/server/services/proxyRetryPolicy.ts b/src/server/services/proxyRetryPolicy.ts index 6ecf33b..4702d0f 100644 --- a/src/server/services/proxyRetryPolicy.ts +++ b/src/server/services/proxyRetryPolicy.ts @@ -9,6 +9,7 @@ const MODEL_UNSUPPORTED_PATTERNS: RegExp[] = [ /model.*does\s+not\s+exist/i, /no\s+such\s+model/i, /unknown\s+model/i, + /unknown\s+provider\s+for\s+model/i, /invalid\s+model/i, /model[_\s-]?not[_\s-]?found/i, /you\s+do\s+not\s+have\s+access\s+to\s+the\s+model/i, @@ -54,6 +55,23 @@ const NON_RETRYABLE_REQUEST_PATTERNS: RegExp[] = [ /unsupported\s+media\s+type/i, ]; +const SAME_SITE_ENDPOINT_ABORT_PATTERNS: RegExp[] = [ + /\b429\b/i, + /too\s+many\s+requests/i, + /rate\s+limit/i, + /quota(?:\s+exceeded)?/i, + /bad\s+gateway/i, + /gateway\s+time-?out/i, + /service\s+unavailable/i, + /temporar(?:y|ily)\s+unavailable/i, + /cpu\s+overloaded/i, + /connection\s+reset/i, + /connection\s+refused/i, + /econnreset/i, + /econnrefused/i, + ...RETRYABLE_TIMEOUT_PATTERNS, +]; + function isModelUnsupportedErrorMessage(rawMessage?: string | null): boolean { const text = (rawMessage || '').trim(); if (!text) return false; @@ -76,3 +94,10 @@ export function shouldRetryProxyRequest(status: number, upstreamErrorText?: stri if (status === 400 || status === 404 || status === 422) return false; return false; } + +export function shouldAbortSameSiteEndpointFallback(status: number, upstreamErrorText?: string | null): boolean { + if (status < 500 && status !== 408 && status !== 429) { + return false; + } + return matchesAnyPattern(SAME_SITE_ENDPOINT_ABORT_PATTERNS, upstreamErrorText); +} diff --git a/src/server/services/runtimeDispatch.ts b/src/server/services/runtimeDispatch.ts new file mode 100644 index 0000000..01995dc --- /dev/null +++ b/src/server/services/runtimeDispatch.ts @@ -0,0 +1 @@ +export { dispatchRuntimeRequest } from '../routes/proxy/runtimeExecutor.js'; diff --git a/src/server/services/runtimeModelProbe.test.ts b/src/server/services/runtimeModelProbe.test.ts new file mode 100644 index 0000000..436129c --- /dev/null +++ b/src/server/services/runtimeModelProbe.test.ts @@ -0,0 +1,116 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const resolveUpstreamEndpointCandidatesMock = vi.fn(); +const buildUpstreamEndpointRequestMock = vi.fn(); +const dispatchRuntimeRequestMock = vi.fn(); +const resolveChannelProxyUrlMock = vi.fn(); +const withSiteRecordProxyRequestInitMock = vi.fn(); + +vi.mock('../routes/proxy/upstreamEndpoint.js', () => ({ + resolveUpstreamEndpointCandidates: (...args: unknown[]) => resolveUpstreamEndpointCandidatesMock(...args), + buildUpstreamEndpointRequest: (...args: unknown[]) => buildUpstreamEndpointRequestMock(...args), +})); + +vi.mock('../routes/proxy/runtimeExecutor.js', () => ({ + dispatchRuntimeRequest: (...args: unknown[]) => dispatchRuntimeRequestMock(...args), +})); + +vi.mock('./siteProxy.js', () => ({ + resolveChannelProxyUrl: (...args: unknown[]) => resolveChannelProxyUrlMock(...args), + withSiteRecordProxyRequestInit: (...args: unknown[]) => withSiteRecordProxyRequestInitMock(...args), +})); + +describe('probeRuntimeModel', () => { + const site = { + id: 1, + name: 'probe-site', + url: 'https://probe.example.com', + platform: 'new-api', + status: 'active', + } as any; + + const account = { + id: 1, + siteId: 1, + username: 'probe-user', + accessToken: '', + apiToken: 'sk-probe', + status: 'active', + extraConfig: null, + } as any; + + beforeEach(() => { + vi.resetModules(); + resolveUpstreamEndpointCandidatesMock.mockReset(); + buildUpstreamEndpointRequestMock.mockReset(); + dispatchRuntimeRequestMock.mockReset(); + resolveChannelProxyUrlMock.mockReset(); + withSiteRecordProxyRequestInitMock.mockReset(); + + buildUpstreamEndpointRequestMock.mockReturnValue({ + path: '/v1/chat/completions', + headers: { 'content-type': 'application/json' }, + body: { model: 'gpt-5.4' }, + runtime: { + executor: 'default', + modelName: 'gpt-5.4', + stream: false, + }, + }); + resolveChannelProxyUrlMock.mockReturnValue(null); + withSiteRecordProxyRequestInitMock.mockImplementation(async (_site: unknown, init: RequestInit) => init); + }); + + it('returns an inconclusive result instead of throwing when endpoint resolution fails', async () => { + resolveUpstreamEndpointCandidatesMock.mockRejectedValue(new Error('resolution failed')); + + const { probeRuntimeModel } = await import('./runtimeModelProbe.js'); + const result = await probeRuntimeModel({ + site, + account, + modelName: 'gpt-5.4', + timeoutMs: 10, + }); + + expect(result.status).toBe('inconclusive'); + expect(result.reason).toContain('resolution failed'); + expect(result.latencyMs).not.toBeNull(); + }); + + it('uses the remaining timeout budget for the runtime request phase', async () => { + resolveUpstreamEndpointCandidatesMock.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 15)); + return ['chat']; + }); + dispatchRuntimeRequestMock.mockImplementation(async (input: { + buildInit: (requestUrl: string, request: Record) => Promise; + request: Record; + targetUrl?: string; + }) => { + const init = await input.buildInit(input.targetUrl || 'https://probe.example.com/v1/chat/completions', input.request); + const signal = init.signal as AbortSignal | undefined; + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 40); + signal?.addEventListener('abort', () => { + clearTimeout(timer); + reject(signal.reason ?? new Error('aborted')); + }, { once: true }); + }); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }); + + const { probeRuntimeModel } = await import('./runtimeModelProbe.js'); + const startedAt = Date.now(); + const result = await probeRuntimeModel({ + site, + account, + modelName: 'gpt-5.4', + timeoutMs: 30, + }); + const elapsedMs = Date.now() - startedAt; + + expect(result.status).toBe('inconclusive'); + expect(result.latencyMs).not.toBeNull(); + expect(elapsedMs).toBeLessThan(200); + }); +}); diff --git a/src/server/services/runtimeModelProbe.ts b/src/server/services/runtimeModelProbe.ts new file mode 100644 index 0000000..fe01ea5 --- /dev/null +++ b/src/server/services/runtimeModelProbe.ts @@ -0,0 +1,250 @@ +import { getOauthInfoFromAccount } from './oauth/oauthAccount.js'; +import { buildOauthProviderHeaders } from './oauth/service.js'; +import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from './siteProxy.js'; +import { dispatchRuntimeRequest } from './runtimeDispatch.js'; +import { + buildUpstreamEndpointRequest, + resolveUpstreamEndpointCandidates, + type UpstreamEndpoint, +} from './upstreamEndpointRuntime.js'; +import { executeEndpointFlow, type BuiltEndpointRequest } from '../proxy-core/orchestration/endpointFlow.js'; +import type { schema } from '../db/index.js'; + +export type RuntimeModelProbeStatus = 'supported' | 'unsupported' | 'inconclusive' | 'skipped'; + +export type RuntimeModelProbeResult = { + status: RuntimeModelProbeStatus; + latencyMs: number | null; + reason: string; +}; + +const NON_CONVERSATION_MODEL_PATTERNS = [ + /(^|[-_/])embedding(s)?($|[-_/])/i, + /(^|[-_/])rerank($|[-_/])/i, + /(^|[-_/])moderation($|[-_/])/i, + /(^|[-_/])whisper($|[-_/])/i, + /(^|[-_/])tts($|[-_/])/i, + /(^|[-_/])transcribe|transcription/i, + /(^|[-_/])dall-e($|[-_/])/i, + /(^|[-_/])imagen($|[-_/])/i, + /(^|[-_/])veo($|[-_/])/i, + /(^|[-_/])cogvideo($|[-_/])/i, +]; + +const DEFINITE_UNSUPPORTED_PATTERNS = [ + /no such model/i, + /unknown model/i, + /unsupported model/i, + /invalid model/i, + /model[^]{0,80}(does not exist|not found|not available|unavailable|unsupported|invalid|disabled)/i, + /(does not exist|not found|not available|unavailable|unsupported|invalid|disabled)[^]{0,40}model/i, + /模型[^]{0,40}(不存在|不可用|不支持|无效|禁用|未开通|未开放)/, + /(不存在|不可用|不支持|无效|禁用)[^]{0,20}模型/, + /model[^]{0,80}(access denied|permission|forbidden|not allowed)/i, + /模型[^]{0,40}(无权限|未授权|禁止访问)/, +]; + +function isLikelyConversationModel(modelName: string): boolean { + const normalized = String(modelName || '').trim(); + if (!normalized) return false; + if (normalized.startsWith('__')) return false; + return !NON_CONVERSATION_MODEL_PATTERNS.some((pattern) => pattern.test(normalized)); +} + +function classifyUnsupportedFailure(status: number, rawErrorText: string): boolean { + if (![400, 403, 404, 422].includes(status)) return false; + const normalized = String(rawErrorText || '').trim(); + if (!normalized) return false; + return DEFINITE_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalized)); +} + +function buildProbeBody(modelName: string): Record { + return { + model: modelName, + messages: [ + { + role: 'user', + content: 'Reply with OK.', + }, + ], + max_tokens: 8, + stream: false, + }; +} + +async function withTimeout(fn: () => Promise, timeoutMs: number, timeoutMessage: string): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + fn(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(timeoutMessage)), timeoutMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function resolveRemainingTimeoutMs(deadlineAtMs: number, timeoutLabel: string): number { + const remainingMs = deadlineAtMs - Date.now(); + if (remainingMs <= 0) { + throw new Error(timeoutLabel); + } + return remainingMs; +} + +export async function probeRuntimeModel(input: { + site: typeof schema.sites.$inferSelect; + account: typeof schema.accounts.$inferSelect; + modelName: string; + timeoutMs: number; + tokenValue?: string | null; +}): Promise { + if (!isLikelyConversationModel(input.modelName)) { + return { + status: 'skipped', + latencyMs: null, + reason: 'skipped non-conversation model probe', + }; + } + + const oauth = getOauthInfoFromAccount(input.account); + const tokenValue = String( + input.tokenValue + || (oauth ? input.account.accessToken : input.account.apiToken) + || '', + ).trim(); + if (!tokenValue) { + return { + status: 'inconclusive', + latencyMs: null, + reason: 'missing credential for probe', + }; + } + + const startedAt = Date.now(); + const deadlineAtMs = startedAt + Math.max(1, input.timeoutMs); + try { + const endpointCandidates = await withTimeout( + () => resolveUpstreamEndpointCandidates( + { + site: input.site, + account: input.account, + }, + input.modelName, + 'openai', + input.modelName, + ), + resolveRemainingTimeoutMs( + deadlineAtMs, + `runtime model probe candidate resolution timeout (${Math.round(input.timeoutMs / 1000)}s)`, + ), + `runtime model probe candidate resolution timeout (${Math.round(input.timeoutMs / 1000)}s)`, + ); + if (endpointCandidates.length <= 0) { + return { + status: 'inconclusive', + latencyMs: Date.now() - startedAt, + reason: 'no compatible probe endpoint candidates', + }; + } + + const providerHeaders = buildOauthProviderHeaders({ + account: input.account, + downstreamHeaders: {}, + }); + const openaiBody = buildProbeBody(input.modelName); + const channelProxyUrl = resolveChannelProxyUrl(input.site, input.account.extraConfig); + const abortController = new AbortController(); + const remainingExecutionTimeoutMs = resolveRemainingTimeoutMs( + deadlineAtMs, + `runtime model probe timeout (${Math.round(input.timeoutMs / 1000)}s)`, + ); + const abortTimer = setTimeout(() => { + abortController.abort(new Error(`runtime model probe timeout (${Math.round(input.timeoutMs / 1000)}s)`)); + }, remainingExecutionTimeoutMs); + abortTimer.unref?.(); + + const buildRequest = (endpoint: UpstreamEndpoint): BuiltEndpointRequest => { + const request = buildUpstreamEndpointRequest({ + endpoint, + modelName: input.modelName, + stream: false, + tokenValue, + oauthProvider: oauth?.provider, + oauthProjectId: oauth?.projectId, + sitePlatform: input.site.platform, + siteUrl: input.site.url, + openaiBody, + downstreamFormat: 'openai', + downstreamHeaders: {}, + providerHeaders, + }); + return { + endpoint, + path: request.path, + headers: request.headers, + body: request.body as Record, + runtime: request.runtime, + }; + }; + const dispatchRequest = async ( + request: BuiltEndpointRequest, + targetUrl: string, + ) => ( + dispatchRuntimeRequest({ + siteUrl: input.site.url, + targetUrl, + request, + buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit( + input.site, + { + method: 'POST', + headers: requestForFetch.headers, + body: JSON.stringify(requestForFetch.body), + signal: abortController.signal, + }, + channelProxyUrl, + ), + }) + ); + + let result: Awaited>; + try { + result = await executeEndpointFlow({ + siteUrl: input.site.url, + proxyUrl: channelProxyUrl, + endpointCandidates, + buildRequest, + dispatchRequest, + }); + } finally { + clearTimeout(abortTimer); + } + const latencyMs = Date.now() - startedAt; + + if (result.ok) { + await result.upstream.text().catch(() => undefined); + return { + status: 'supported', + latencyMs, + reason: 'probe succeeded', + }; + } + + const rawErrorText = String(result.rawErrText || result.errText || '').trim(); + return { + status: classifyUnsupportedFailure(result.status || 0, rawErrorText) ? 'unsupported' : 'inconclusive', + latencyMs, + reason: rawErrorText || `probe failed with status ${result.status || 0}`, + }; + } catch (error) { + return { + status: 'inconclusive', + latencyMs: Date.now() - startedAt, + reason: error instanceof Error ? error.message : 'probe failed', + }; + } +} diff --git a/src/server/services/tokenRouter.cache.test.ts b/src/server/services/tokenRouter.cache.test.ts index b66ff17..1c934dd 100644 --- a/src/server/services/tokenRouter.cache.test.ts +++ b/src/server/services/tokenRouter.cache.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { eq } from 'drizzle-orm'; +import { eq, inArray } from 'drizzle-orm'; type DbModule = typeof import('../db/index.js'); type TokenRouterModule = typeof import('./tokenRouter.js'); @@ -615,6 +615,90 @@ describe('TokenRouter runtime cache', () => { expect(await router.selectChannel('gpt-4o-mini')).toBeNull(); }); + it('clears short-window cooldown on sibling channels after a successful recovery probe', async () => { + const site = await db.insert(schema.sites).values({ + name: 'shared-credential-recovery-site', + url: 'https://shared-credential-recovery.example.com', + platform: 'codex', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'shared-credential-recovery-user', + accessToken: 'shared-credential-recovery-access-token', + status: 'active', + oauthProvider: 'codex', + extraConfig: JSON.stringify({ + credentialMode: 'session', + oauth: { + provider: 'codex', + }, + }), + }).returning().get(); + + const primaryRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4', + routingStrategy: 'weighted', + enabled: true, + }).returning().get(); + + const siblingRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-4o-mini', + routingStrategy: 'weighted', + enabled: true, + }).returning().get(); + + const primaryChannel = await db.insert(schema.routeChannels).values({ + routeId: primaryRoute.id, + accountId: account.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const siblingChannel = await db.insert(schema.routeChannels).values({ + routeId: siblingRoute.id, + accountId: account.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + await router.recordFailure(primaryChannel.id, { + status: 429, + errorText: JSON.stringify({ + error: { + type: 'usage_limit_reached', + message: 'The usage limit has been reached', + }, + }), + modelName: 'gpt-5.4', + }); + + await router.recordProbeSuccess(primaryChannel.id, 180, 'gpt-5.4'); + + const refreshedChannels = await db.select().from(schema.routeChannels) + .where(inArray(schema.routeChannels.id, [primaryChannel.id, siblingChannel.id])) + .all(); + + expect(refreshedChannels).toEqual(expect.arrayContaining([ + expect.objectContaining({ + id: primaryChannel.id, + cooldownUntil: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + }), + expect.objectContaining({ + id: siblingChannel.id, + cooldownUntil: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + }), + ])); + }); + it('round robins across all available channels regardless of priority', async () => { const site = await db.insert(schema.sites).values({ name: 'round-robin-site', diff --git a/src/server/services/tokenRouter.selection.test.ts b/src/server/services/tokenRouter.selection.test.ts index 038d2c3..6e54072 100644 --- a/src/server/services/tokenRouter.selection.test.ts +++ b/src/server/services/tokenRouter.selection.test.ts @@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm'; type DbModule = typeof import('../db/index.js'); type TokenRouterModule = typeof import('./tokenRouter.js'); type ConfigModule = typeof import('../config.js'); +type ProxyChannelCoordinatorModule = typeof import('./proxyChannelCoordinator.js'); const mockedCatalogRoutingCost = vi.fn<( input: { siteId: number; accountId: number; modelName: string } @@ -24,14 +25,20 @@ describe('TokenRouter selection scoring', () => { let db: DbModule['db']; let schema: DbModule['schema']; let TokenRouter: TokenRouterModule['TokenRouter']; + let tokenRouterTestUtils: TokenRouterModule['__tokenRouterTestUtils']; let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache']; let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState']; let flushSiteRuntimeHealthPersistence: TokenRouterModule['flushSiteRuntimeHealthPersistence']; + let filterRecentlyFailedCandidates: TokenRouterModule['filterRecentlyFailedCandidates']; let config: ConfigModule['config']; + let proxyChannelCoordinator: ProxyChannelCoordinatorModule['proxyChannelCoordinator']; + let resetProxyChannelCoordinatorState: ProxyChannelCoordinatorModule['resetProxyChannelCoordinatorState']; let dataDir = ''; let idSeed = 0; let originalRoutingWeights: typeof config.routingWeights; let originalRoutingFallbackUnitCost: number; + let originalProxySessionChannelConcurrencyLimit: number; + let originalProxySessionChannelQueueWaitMs: number; const nextId = () => { idSeed += 1; @@ -46,21 +53,30 @@ describe('TokenRouter selection scoring', () => { const dbModule = await import('../db/index.js'); const tokenRouterModule = await import('./tokenRouter.js'); const configModule = await import('../config.js'); + const coordinatorModule = await import('./proxyChannelCoordinator.js'); db = dbModule.db; schema = dbModule.schema; TokenRouter = tokenRouterModule.TokenRouter; + tokenRouterTestUtils = tokenRouterModule.__tokenRouterTestUtils; invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache; resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState; flushSiteRuntimeHealthPersistence = tokenRouterModule.flushSiteRuntimeHealthPersistence; + filterRecentlyFailedCandidates = tokenRouterModule.filterRecentlyFailedCandidates; config = configModule.config; + proxyChannelCoordinator = coordinatorModule.proxyChannelCoordinator; + resetProxyChannelCoordinatorState = coordinatorModule.resetProxyChannelCoordinatorState; originalRoutingWeights = { ...config.routingWeights }; originalRoutingFallbackUnitCost = config.routingFallbackUnitCost; + originalProxySessionChannelConcurrencyLimit = config.proxySessionChannelConcurrencyLimit; + originalProxySessionChannelQueueWaitMs = config.proxySessionChannelQueueWaitMs; }); beforeEach(async () => { idSeed = 0; mockedCatalogRoutingCost.mockReset(); mockedCatalogRoutingCost.mockReturnValue(null); + config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit; + config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs; await db.delete(schema.routeChannels).run(); await db.delete(schema.tokenRoutes).run(); await db.delete(schema.settings).run(); @@ -69,13 +85,17 @@ describe('TokenRouter selection scoring', () => { await db.delete(schema.sites).run(); invalidateTokenRouterCache(); resetSiteRuntimeHealthState(); + resetProxyChannelCoordinatorState(); }); afterAll(() => { config.routingWeights = { ...originalRoutingWeights }; config.routingFallbackUnitCost = originalRoutingFallbackUnitCost; + config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit; + config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs; invalidateTokenRouterCache(); resetSiteRuntimeHealthState(); + resetProxyChannelCoordinatorState(); delete process.env.DATA_DIR; }); @@ -96,7 +116,7 @@ describe('TokenRouter selection scoring', () => { }).returning().get(); } - async function createAccount(siteId: number, usernamePrefix: string) { + async function createAccount(siteId: number, usernamePrefix: string, options: { extraConfig?: string | null } = {}) { const id = nextId(); return await db.insert(schema.accounts).values({ siteId, @@ -104,6 +124,7 @@ describe('TokenRouter selection scoring', () => { accessToken: `access-${id}`, apiToken: `sk-${id}`, status: 'active', + extraConfig: options.extraConfig ?? null, }).returning().get(); } @@ -126,7 +147,11 @@ describe('TokenRouter selection scoring', () => { usageWeight: 0, }; - const route = await createRoute('gpt-5.2'); + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.2', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); const site = await createSite('sticky-site'); const account = await createAccount(site.id, 'sticky-user'); const tokenA = await createToken(account.id, 'sticky-a'); @@ -165,6 +190,27 @@ describe('TokenRouter selection scoring', () => { await expect(router.selectPreferredChannel('gpt-5.2', preferredChannel.id)).resolves.toBeNull(); }); + it('avoids recently failed candidates by default when healthy alternatives exist', () => { + const nowMs = Date.now(); + const filtered = filterRecentlyFailedCandidates([ + { + channel: { + failCount: 3, + lastFailAt: new Date(nowMs).toISOString(), + }, + }, + { + channel: { + failCount: 0, + lastFailAt: null, + }, + }, + ], nowMs); + + expect(filtered).toHaveLength(1); + expect(filtered[0]?.channel.failCount).toBe(0); + }); + it('normalizes probability across channels on the same site', async () => { config.routingWeights = { baseWeightFactor: 1, @@ -790,6 +836,66 @@ describe('TokenRouter selection scoring', () => { expect(weakCandidate?.reason || '').toContain('成功率=40.0%'); }); + it('stable_first ranks recent and fallback success rate ahead of balance-heavy weak sites', async () => { + config.routingWeights = { + baseWeightFactor: 0, + valueScoreFactor: 1, + costWeight: 0.1, + balanceWeight: 10, + usageWeight: 0, + }; + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); + + const siteStable = await createSite('stable-rate-front'); + const accountStable = await createAccount(siteStable.id, 'stable-rate-user-front'); + const tokenStable = await createToken(accountStable.id, 'stable-rate-token-front'); + const stableChannel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountStable.id, + tokenId: tokenStable.id, + priority: 0, + weight: 10, + enabled: true, + successCount: 80, + failCount: 4, + totalLatencyMs: 80 * 380, + }).returning().get(); + + const siteWeak = await createSite('stable-rate-back'); + const accountWeak = await createAccount(siteWeak.id, 'stable-rate-user-back'); + await db.update(schema.accounts).set({ + balance: 999999, + }).where(eq(schema.accounts.id, accountWeak.id)).run(); + const tokenWeak = await createToken(accountWeak.id, 'stable-rate-token-back'); + const weakChannel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountWeak.id, + tokenId: tokenWeak.id, + priority: 0, + weight: 10, + enabled: true, + successCount: 5, + failCount: 21, + totalLatencyMs: 5 * 1600, + }).returning().get(); + + const decision = await new TokenRouter().explainSelection('gpt-5.4'); + const stableCandidate = decision.candidates.find((candidate) => candidate.channelId === stableChannel.id); + const weakCandidate = decision.candidates.find((candidate) => candidate.channelId === weakChannel.id); + + expect(stableCandidate).toBeTruthy(); + expect(weakCandidate).toBeTruthy(); + expect(decision.selectedChannelId).toBe(stableChannel.id); + expect((stableCandidate?.probability || 0)).toBeGreaterThan(weakCandidate?.probability || 0); + expect(stableCandidate?.reason || '').toContain('近期成功率='); + expect(weakCandidate?.reason || '').toContain('综合近期成功率='); + }); + it('reloads persisted runtime health after in-memory reset', async () => { config.routingWeights = { baseWeightFactor: 1, @@ -856,6 +962,79 @@ describe('TokenRouter selection scoring', () => { expect(candidateA?.reason || '').toContain('运行时健康='); }); + it('keeps a recovered stable_first site behind healthier peers until recent success rebuilds', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.3', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); + + const siteRecovered = await createSite('stable-recovery-a'); + const accountRecovered = await createAccount(siteRecovered.id, 'stable-recovery-user-a'); + const tokenRecovered = await createToken(accountRecovered.id, 'stable-recovery-token-a'); + const recoveredChannel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountRecovered.id, + tokenId: tokenRecovered.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const siteHealthy = await createSite('stable-recovery-b'); + const accountHealthy = await createAccount(siteHealthy.id, 'stable-recovery-user-b'); + const tokenHealthy = await createToken(accountHealthy.id, 'stable-recovery-token-b'); + const healthyChannel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountHealthy.id, + tokenId: tokenHealthy.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + for (let index = 0; index < 3; index += 1) { + await router.recordFailure(recoveredChannel.id, { + status: 502, + errorText: 'Gateway timeout', + modelName: 'gpt-5.3', + }); + } + await db.update(schema.routeChannels).set({ + cooldownUntil: null, + lastFailAt: null, + failCount: 0, + consecutiveFailCount: 0, + cooldownLevel: 0, + }).where(eq(schema.routeChannels.id, recoveredChannel.id)).run(); + + await router.recordSuccess(recoveredChannel.id, 900, 0, 'gpt-5.3'); + for (let index = 0; index < 4; index += 1) { + await router.recordSuccess(healthyChannel.id, 320, 0, 'gpt-5.3'); + } + invalidateTokenRouterCache(); + + const preview = await router.previewSelectedChannel('gpt-5.3'); + const decision = await router.explainSelection('gpt-5.3'); + const recoveredCandidate = decision.candidates.find((candidate) => candidate.channelId === recoveredChannel.id); + const healthyCandidate = decision.candidates.find((candidate) => candidate.channelId === healthyChannel.id); + + expect(preview?.channel.id).toBe(healthyChannel.id); + expect(decision.selectedChannelId).toBe(healthyChannel.id); + expect((recoveredCandidate?.probability || 0)).toBeLessThan(healthyCandidate?.probability || 0); + expect(recoveredCandidate?.reason || '').toContain('近期成功率='); + expect(healthyCandidate?.reason || '').toContain('近期成功率='); + }); + it('penalizes the failed model more than unrelated models on the same site', async () => { config.routingWeights = { baseWeightFactor: 1, @@ -934,6 +1113,90 @@ describe('TokenRouter selection scoring', () => { expect(gptCandidateA?.reason || '').toContain('模型='); }); + it('treats unknown provider for model as model-scoped degradation instead of opening a site breaker', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + + const gptRoute = await createRoute('gpt-5.4'); + const claudeRoute = await createRoute('claude-sonnet-4-6'); + + const siteA = await createSite('unknown-provider-a'); + const accountA = await createAccount(siteA.id, 'unknown-provider-user-a'); + const tokenA = await createToken(accountA.id, 'unknown-provider-token-a'); + const gptChannelA = await db.insert(schema.routeChannels).values({ + routeId: gptRoute.id, + accountId: accountA.id, + tokenId: tokenA.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + await db.insert(schema.routeChannels).values({ + routeId: claudeRoute.id, + accountId: accountA.id, + tokenId: tokenA.id, + priority: 0, + weight: 10, + enabled: true, + }).run(); + + const siteB = await createSite('unknown-provider-b'); + const accountB = await createAccount(siteB.id, 'unknown-provider-user-b'); + const tokenB = await createToken(accountB.id, 'unknown-provider-token-b'); + await db.insert(schema.routeChannels).values([ + { + routeId: gptRoute.id, + accountId: accountB.id, + tokenId: tokenB.id, + priority: 0, + weight: 10, + enabled: true, + }, + { + routeId: claudeRoute.id, + accountId: accountB.id, + tokenId: tokenB.id, + priority: 0, + weight: 10, + enabled: true, + }, + ]).run(); + + const router = new TokenRouter(); + for (let index = 0; index < 3; index += 1) { + await router.recordFailure(gptChannelA.id, { + status: 502, + errorText: 'unknown provider for model gpt-5.4', + modelName: 'gpt-5.4', + }); + } + await db.update(schema.routeChannels).set({ + cooldownUntil: null, + lastFailAt: null, + failCount: 0, + }).where(eq(schema.routeChannels.id, gptChannelA.id)).run(); + invalidateTokenRouterCache(); + + const gptDecision = await router.explainSelection('gpt-5.4'); + const claudeDecision = await router.explainSelection('claude-sonnet-4-6'); + const gptCandidateA = gptDecision.candidates.find((candidate) => candidate.siteName.startsWith('unknown-provider-a')); + const claudeCandidateA = claudeDecision.candidates.find((candidate) => candidate.siteName.startsWith('unknown-provider-a')); + + expect(gptCandidateA).toBeTruthy(); + expect(claudeCandidateA).toBeTruthy(); + expect(gptDecision.summary.join(' ')).not.toContain('站点熔断避让'); + expect(claudeDecision.summary.join(' ')).not.toContain('站点熔断避让'); + expect(gptCandidateA?.reason || '').not.toContain('站点熔断'); + expect(claudeCandidateA?.reason || '').not.toContain('站点熔断'); + expect((gptCandidateA?.probability || 0)).toBeLessThan((claudeCandidateA?.probability || 0)); + expect(gptCandidateA?.reason || '').toContain('模型='); + }); + it('stable_first deterministically chooses the healthiest candidate', async () => { config.routingWeights = { baseWeightFactor: 1, @@ -993,4 +1256,282 @@ describe('TokenRouter selection scoring', () => { expect(decision.summary.join(' ')).toContain('稳定优先'); expect(decision.selectedChannelId).toBe(channelB.id); }); + + it('stable_first rotates across sites that remain inside the stable pool', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); + + const siteA = await createSite('stable-pool-a'); + const accountA = await createAccount(siteA.id, 'stable-pool-user-a'); + const tokenA = await createToken(accountA.id, 'stable-pool-token-a'); + const channelA = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountA.id, + tokenId: tokenA.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const siteB = await createSite('stable-pool-b'); + const accountB = await createAccount(siteB.id, 'stable-pool-user-b'); + const tokenB = await createToken(accountB.id, 'stable-pool-token-b'); + const channelB = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountB.id, + tokenId: tokenB.id, + priority: 0, + weight: 9, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + const first = await router.selectChannel('gpt-5.4'); + const second = await router.selectChannel('gpt-5.4'); + const third = await router.selectChannel('gpt-5.4'); + const decision = await router.explainSelection('gpt-5.4'); + + expect(first?.channel.id).toBe(channelA.id); + expect(second?.channel.id).toBe(channelB.id); + expect(third?.channel.id).toBe(channelA.id); + expect(decision.summary.join(' ')).toContain('主池站点 2'); + }); + + it('stable_first rotates across healthy sites in configured priority order instead of stopping at the first layer', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-4.1', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); + + const siteA = await createSite('ordered-stable-a'); + const accountA = await createAccount(siteA.id, 'ordered-stable-user-a'); + const tokenA = await createToken(accountA.id, 'ordered-stable-token-a'); + const channelA = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountA.id, + tokenId: tokenA.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const siteB = await createSite('ordered-stable-b'); + const accountB = await createAccount(siteB.id, 'ordered-stable-user-b'); + const tokenB = await createToken(accountB.id, 'ordered-stable-token-b'); + const channelB = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountB.id, + tokenId: tokenB.id, + priority: 4, + weight: 10, + enabled: true, + }).returning().get(); + + const siteC = await createSite('ordered-stable-c'); + const accountC = await createAccount(siteC.id, 'ordered-stable-user-c'); + const tokenC = await createToken(accountC.id, 'ordered-stable-token-c'); + const channelC = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountC.id, + tokenId: tokenC.id, + priority: 8, + weight: 10, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + const first = await router.selectChannel('gpt-4.1'); + const second = await router.selectChannel('gpt-4.1'); + const third = await router.selectChannel('gpt-4.1'); + const fourth = await router.selectChannel('gpt-4.1'); + const decision = await router.explainSelection('gpt-4.1'); + + expect(first?.channel.id).toBe(channelA.id); + expect(second?.channel.id).toBe(channelB.id); + expect(third?.channel.id).toBe(channelC.id); + expect(fourth?.channel.id).toBe(channelA.id); + expect(decision.summary.join(' ')).toContain('按配置顺序轮询站点'); + }); + + it('stable_first gives observation-pool sites occasional real traffic without background probing', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4-probe-free', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); + + const sitePrimary = await createSite('observation-primary'); + const accountPrimary = await createAccount(sitePrimary.id, 'observation-user-primary'); + const tokenPrimary = await createToken(accountPrimary.id, 'observation-token-primary'); + const primaryChannel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountPrimary.id, + tokenId: tokenPrimary.id, + priority: 0, + weight: 10, + enabled: true, + successCount: 48, + failCount: 0, + totalLatencyMs: 48 * 320, + }).returning().get(); + + const siteObservation = await createSite('observation-candidate'); + const accountObservation = await createAccount(siteObservation.id, 'observation-user-candidate'); + const tokenObservation = await createToken(accountObservation.id, 'observation-token-candidate'); + const observationChannel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountObservation.id, + tokenId: tokenObservation.id, + priority: 1, + weight: 10, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + const selectedChannelIds: number[] = []; + for (let index = 0; index < 23; index += 1) { + const selected = await router.selectChannel('gpt-5.4-probe-free'); + selectedChannelIds.push(selected?.channel.id ?? 0); + } + let decision = await router.explainSelection('gpt-5.4-probe-free'); + let observationCandidate = decision.candidates.find((candidate) => candidate.channelId === observationChannel.id); + let primaryCandidate = decision.candidates.find((candidate) => candidate.channelId === primaryChannel.id); + + expect(observationCandidate?.probability).toBe(100); + expect(primaryCandidate?.probability).toBe(0); + expect(decision.summary.join(' ')).toContain('本次命中观察池灰度流量'); + + const observationSelected = await router.selectChannel('gpt-5.4-probe-free'); + selectedChannelIds.push(observationSelected?.channel.id ?? 0); + + decision = await router.explainSelection('gpt-5.4-probe-free'); + observationCandidate = decision.candidates.find((candidate) => candidate.channelId === observationChannel.id); + primaryCandidate = decision.candidates.find((candidate) => candidate.channelId === primaryChannel.id); + + expect(selectedChannelIds.filter((channelId) => channelId === observationChannel.id)).toHaveLength(1); + expect(selectedChannelIds.filter((channelId) => channelId === primaryChannel.id).length).toBeGreaterThan(20); + expect(decision.summary.join(' ')).toContain('观察池站点 1'); + expect(decision.summary.join(' ')).toContain('还需 23 次主池请求'); + expect(observationCandidate?.reason || '').toContain('观察池'); + expect(observationCandidate?.probability).toBe(0); + expect(primaryCandidate?.reason || '').toContain('主池'); + expect((primaryCandidate?.probability || 0)).toBeGreaterThan(0); + }); + + it('caps the stable_first rotation cache size', () => { + invalidateTokenRouterCache(); + + for (let index = 0; index < 1200; index += 1) { + tokenRouterTestUtils.rememberStableFirstSiteSelectionForKey(`route:${index}`, (index % 7) + 1); + } + + expect(tokenRouterTestUtils.getStableFirstRotationCacheSize()).toBeLessThanOrEqual(1024); + }); + + it('penalizes saturated session-scoped channels using runtime load snapshots', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + config.proxySessionChannelConcurrencyLimit = 1; + config.proxySessionChannelQueueWaitMs = 5_000; + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.2', + routingStrategy: 'stable_first', + enabled: true, + }).returning().get(); + const sessionExtraConfig = JSON.stringify({ credentialMode: 'session' }); + + const siteBusy = await createSite('runtime-load-busy'); + const accountBusy = await createAccount(siteBusy.id, 'runtime-load-user-busy', { + extraConfig: sessionExtraConfig, + }); + const tokenBusy = await createToken(accountBusy.id, 'runtime-load-token-busy'); + const channelBusy = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountBusy.id, + tokenId: tokenBusy.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const siteFree = await createSite('runtime-load-free'); + const accountFree = await createAccount(siteFree.id, 'runtime-load-user-free', { + extraConfig: sessionExtraConfig, + }); + const tokenFree = await createToken(accountFree.id, 'runtime-load-token-free'); + const channelFree = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountFree.id, + tokenId: tokenFree.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const activeLease = await proxyChannelCoordinator.acquireChannelLease({ + channelId: channelBusy.id, + accountExtraConfig: accountBusy.extraConfig, + }); + expect(activeLease.status).toBe('acquired'); + if (activeLease.status !== 'acquired') return; + + const queuedLeasePromise = proxyChannelCoordinator.acquireChannelLease({ + channelId: channelBusy.id, + accountExtraConfig: accountBusy.extraConfig, + }); + await Promise.resolve(); + + const router = new TokenRouter(); + const preview = await router.previewSelectedChannel('gpt-5.2'); + const decision = await router.explainSelection('gpt-5.2'); + const busyCandidate = decision.candidates.find((candidate) => candidate.channelId === channelBusy.id); + const freeCandidate = decision.candidates.find((candidate) => candidate.channelId === channelFree.id); + + expect(preview?.channel.id).toBe(channelFree.id); + expect(busyCandidate?.reason || '').toContain('会话负载='); + expect(busyCandidate?.reason || '').toContain('活跃=1/1'); + expect(busyCandidate?.reason || '').toContain('等待=1'); + expect((busyCandidate?.probability || 0)).toBeLessThan((freeCandidate?.probability || 0)); + + activeLease.lease.release(); + const queuedLease = await queuedLeasePromise; + expect(queuedLease.status).toBe('acquired'); + if (queuedLease.status === 'acquired') { + queuedLease.lease.release(); + } + }); }); diff --git a/src/server/services/tokenRouter.test.ts b/src/server/services/tokenRouter.test.ts index 087630d..b7b8a64 100644 --- a/src/server/services/tokenRouter.test.ts +++ b/src/server/services/tokenRouter.test.ts @@ -81,7 +81,7 @@ describe('filterRecentlyFailedCandidates', () => { expect(result.map((c) => c.id).sort()).toEqual(['a', 'b']); }); - it('keeps candidates unchanged when avoidSec is omitted', () => { + it('uses the default recent-failure backoff when avoidSec is omitted', () => { const nowMs = Date.now(); const candidates: Candidate[] = [ { @@ -101,7 +101,7 @@ describe('filterRecentlyFailedCandidates', () => { ]; const result = filterRecentlyFailedCandidates(candidates, nowMs); - expect(result).toEqual(candidates); + expect(result.map((candidate) => candidate.id)).toEqual(['healthy']); }); it('does not penalize stale failures outside the avoidance window', () => { diff --git a/src/server/services/tokenRouter.ts b/src/server/services/tokenRouter.ts index 6872992..58ed3cb 100644 --- a/src/server/services/tokenRouter.ts +++ b/src/server/services/tokenRouter.ts @@ -7,6 +7,7 @@ import { TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING, } from '../config.js'; import { getCachedModelRoutingReferenceCost, refreshModelPricingCatalog } from './modelPricingService.js'; +import { proxyChannelCoordinator, type ProxyChannelLoadSnapshot } from './proxyChannelCoordinator.js'; import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js'; import { normalizeRouteRoutingStrategy, @@ -67,6 +68,9 @@ type SiteRuntimeHealthState = { latencyEmaMs: number | null; transientFailureStreak: number; lastTransientFailureAtMs: number | null; + recentSuccessCount: number; + recentFailureCount: number; + recentWindowUpdatedAtMs: number; breakerLevel: number; breakerUntilMs: number | null; lastUpdatedAtMs: number; @@ -81,6 +85,7 @@ const MAX_FAILURE_BACKOFF_SEC = 30 * 24 * 60 * 60; const MIN_EFFECTIVE_UNIT_COST = 1e-6; const ROUND_ROBIN_FAILURE_THRESHOLD = 3; const ROUND_ROBIN_COOLDOWN_LEVELS_SEC = [0, 10 * 60, 60 * 60, 24 * 60 * 60] as const; +const STABLE_FIRST_SITE_SCORE_RATIO = 0.92; const SITE_RUNTIME_HEALTH_DECAY_HALF_LIFE_MS = 10 * 60 * 1000; const SITE_RUNTIME_MIN_MULTIPLIER = 0.08; const SITE_RUNTIME_LATENCY_BASELINE_MS = 2_500; @@ -90,6 +95,12 @@ const SITE_RUNTIME_LATENCY_EMA_ALPHA = 0.3; const SITE_RUNTIME_BREAKER_STREAK_THRESHOLD = 3; const SITE_RUNTIME_BREAKER_LEVELS_MS = [0, 60_000, 5 * 60_000, 30 * 60 * 1000] as const; const SITE_TRANSIENT_STREAK_WINDOW_MS = 5 * 60 * 1000; +const SITE_RECENT_OUTCOME_HALF_LIFE_MS = 30 * 60 * 1000; +const SITE_RECENT_SUCCESS_CONFIDENCE_SAMPLES = 12; +const SITE_RECENT_SUCCESS_PRIOR_SUCCESSES = 1; +const SITE_RECENT_SUCCESS_PRIOR_FAILURES = 1; +const SITE_RECENT_SUCCESS_FALLBACK_RATE = 0.5; +const SITE_RECENT_MODEL_WEIGHT = 0.65; const SITE_HISTORICAL_HEALTH_MIN_MULTIPLIER = 0.45; const SITE_HISTORICAL_HEALTH_MAX_SAMPLE = 24; const SITE_HISTORICAL_LATENCY_BASELINE_MS = 2_000; @@ -120,6 +131,7 @@ const SITE_MODEL_FAILURE_PATTERNS: RegExp[] = [ /does\s+not\s+support(?:\s+the)?\s+model/i, /no\s+such\s+model/i, /unknown\s+model/i, + /unknown\s+provider\s+for\s+model/i, /invalid\s+model/i, /model.*does\s+not\s+exist/i, /当前\s*api\s*不支持所选模型/i, @@ -175,17 +187,113 @@ type SiteRuntimeHealthDetails = { globalBreakerOpen: boolean; modelBreakerOpen: boolean; modelKey: string; + recentSuccessRate: number; + recentSampleCount: number; + recentConfidence: number; }; type WeightedSelectionMode = 'weighted' | 'stable_first'; +type WeightedSelectionResult = { + selected: RouteChannelCandidate | null; + details: Array<{ candidate: RouteChannelCandidate; probability: number; reason: string }>; + stableSiteCount: number; +}; + +type RecentOutcomeSnapshot = { + successCount: number; + failureCount: number; + sampleCount: number; + successRate: number; + confidence: number; +}; + +type StableFirstSitePoolState = { + siteId: number; + leader: RouteChannelCandidate; + effectiveSuccessRate: number; + trusted: boolean; + observationReason: string | null; +}; + +type StableFirstPoolPlan = { + primaryCandidates: RouteChannelCandidate[]; + observationCandidates: RouteChannelCandidate[]; + primarySiteIds: Set; + observationSiteIds: Set; + siteStateById: Map; +}; + +type StableFirstObservationProgressState = { + requestCount: number; + lastObservationAtMs: number | null; +}; const siteRuntimeHealthStates = new Map(); const siteModelRuntimeHealthStates = new Map>(); +const stableFirstLastSelectedSiteByKey = new Map(); +const MAX_STABLE_FIRST_ROTATION_KEYS = 1024; +const stableFirstObservationProgressByKey = new Map(); +const stableFirstObservationSiteCooldownByKey = new Map(); +const MAX_STABLE_FIRST_OBSERVATION_PROGRESS_KEYS = 1024; +const MAX_STABLE_FIRST_OBSERVATION_SITE_COOLDOWN_KEYS = 4096; let siteRuntimeHealthLoaded = false; let siteRuntimeHealthLoadPromise: Promise | null = null; let siteRuntimeHealthSaveTimer: ReturnType | null = null; let siteRuntimeHealthPersistInFlight: Promise | null = null; +const STABLE_FIRST_PRIMARY_SUCCESS_RATE_RATIO = 0.92; +const STABLE_FIRST_TRUSTED_RECENT_CONFIDENCE = 0.5; +const STABLE_FIRST_TRUSTED_HISTORICAL_CALLS = 8; +const STABLE_FIRST_OBSERVATION_REQUEST_INTERVAL = 24; +const STABLE_FIRST_OBSERVATION_SITE_COOLDOWN_MS = 30 * 60 * 1000; + +function rememberStableFirstSiteSelectionForKey(rotationKey: string, siteId: number): void { + if (!rotationKey || !Number.isFinite(siteId) || siteId <= 0) return; + if (stableFirstLastSelectedSiteByKey.has(rotationKey)) { + stableFirstLastSelectedSiteByKey.delete(rotationKey); + } + stableFirstLastSelectedSiteByKey.set(rotationKey, siteId); + while (stableFirstLastSelectedSiteByKey.size > MAX_STABLE_FIRST_ROTATION_KEYS) { + const oldestKey = stableFirstLastSelectedSiteByKey.keys().next().value; + if (!oldestKey) break; + stableFirstLastSelectedSiteByKey.delete(oldestKey); + } +} + +function rememberStableFirstObservationProgressForKey( + rotationKey: string, + state: StableFirstObservationProgressState, +): void { + if (!rotationKey) return; + if (stableFirstObservationProgressByKey.has(rotationKey)) { + stableFirstObservationProgressByKey.delete(rotationKey); + } + stableFirstObservationProgressByKey.set(rotationKey, state); + while (stableFirstObservationProgressByKey.size > MAX_STABLE_FIRST_OBSERVATION_PROGRESS_KEYS) { + const oldestKey = stableFirstObservationProgressByKey.keys().next().value; + if (!oldestKey) break; + stableFirstObservationProgressByKey.delete(oldestKey); + } +} + +function rememberStableFirstObservationSiteCooldown( + rotationKey: string, + siteId: number, + observedAtMs: number, +): void { + if (!rotationKey || !Number.isFinite(siteId) || siteId <= 0) return; + const scopedKey = `${rotationKey}:${siteId}`; + if (stableFirstObservationSiteCooldownByKey.has(scopedKey)) { + stableFirstObservationSiteCooldownByKey.delete(scopedKey); + } + stableFirstObservationSiteCooldownByKey.set(scopedKey, observedAtMs); + while (stableFirstObservationSiteCooldownByKey.size > MAX_STABLE_FIRST_OBSERVATION_SITE_COOLDOWN_KEYS) { + const oldestKey = stableFirstObservationSiteCooldownByKey.keys().next().value; + if (!oldestKey) break; + stableFirstObservationSiteCooldownByKey.delete(oldestKey); + } +} + function fibonacciNumber(index: number): number { if (index <= 2) return 1; let prev = 1; @@ -244,10 +352,27 @@ function isUsageLimitRateLimitFailure(context: SiteRuntimeFailureContext = {}): return matchesAnyPattern(USAGE_LIMIT_RATE_LIMIT_PATTERNS, context.errorText); } +function isModelScopedRuntimeFailure(context: SiteRuntimeFailureContext = {}): boolean { + return matchesAnyPattern(SITE_MODEL_FAILURE_PATTERNS, context.errorText); +} + +function isProtocolRuntimeFailure(context: SiteRuntimeFailureContext = {}): boolean { + return matchesAnyPattern(SITE_PROTOCOL_FAILURE_PATTERNS, context.errorText); +} + +function isValidationRuntimeFailure(context: SiteRuntimeFailureContext = {}): boolean { + return matchesAnyPattern(SITE_VALIDATION_FAILURE_PATTERNS, context.errorText); +} + function clampNumber(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } +function isContributionCloseToBest(value: number, bestValue: number, ratio = STABLE_FIRST_SITE_SCORE_RATIO): boolean { + if (bestValue <= 0) return true; + return value >= (bestValue * ratio); +} + function isRecord(value: unknown): value is Record { return !!value && typeof value === 'object' && !Array.isArray(value); } @@ -267,6 +392,79 @@ function readNullableTimestamp(value: unknown): number | null { return normalized; } +function decayRecentOutcomeCount(value: number, elapsedMs: number): number { + if (!Number.isFinite(value) || value <= 0) return 0; + if (elapsedMs <= 0) return value; + const decayFactor = Math.pow(0.5, elapsedMs / SITE_RECENT_OUTCOME_HALF_LIFE_MS); + return value * decayFactor; +} + +function buildRecentOutcomeSnapshot( + successCount: number, + failureCount: number, +): RecentOutcomeSnapshot { + const normalizedSuccessCount = Math.max(0, successCount); + const normalizedFailureCount = Math.max(0, failureCount); + const sampleCount = normalizedSuccessCount + normalizedFailureCount; + const successRate = ( + normalizedSuccessCount + SITE_RECENT_SUCCESS_PRIOR_SUCCESSES + ) / ( + sampleCount + SITE_RECENT_SUCCESS_PRIOR_SUCCESSES + SITE_RECENT_SUCCESS_PRIOR_FAILURES + ); + return { + successCount: normalizedSuccessCount, + failureCount: normalizedFailureCount, + sampleCount, + successRate, + confidence: clampNumber(sampleCount / SITE_RECENT_SUCCESS_CONFIDENCE_SAMPLES, 0, 1), + }; +} + +function getRecentOutcomeSnapshot(state: SiteRuntimeHealthState | null | undefined, nowMs = Date.now()): RecentOutcomeSnapshot { + if (!state) { + return buildRecentOutcomeSnapshot(0, 0); + } + const updatedAtMs = Math.max(0, readFiniteInteger(state.recentWindowUpdatedAtMs) ?? state.lastUpdatedAtMs ?? nowMs); + const elapsedMs = Math.max(0, nowMs - updatedAtMs); + return buildRecentOutcomeSnapshot( + decayRecentOutcomeCount(state.recentSuccessCount, elapsedMs), + decayRecentOutcomeCount(state.recentFailureCount, elapsedMs), + ); +} + +function refreshRecentOutcomeWindow(state: SiteRuntimeHealthState, nowMs = Date.now()): void { + const snapshot = getRecentOutcomeSnapshot(state, nowMs); + state.recentSuccessCount = snapshot.successCount; + state.recentFailureCount = snapshot.failureCount; + state.recentWindowUpdatedAtMs = nowMs; +} + +function blendRecentOutcomeSnapshots( + globalSnapshot: RecentOutcomeSnapshot, + modelSnapshot: RecentOutcomeSnapshot | null, +): RecentOutcomeSnapshot { + if (!modelSnapshot || modelSnapshot.sampleCount <= 0) { + return globalSnapshot; + } + const modelWeight = SITE_RECENT_MODEL_WEIGHT; + const globalWeight = 1 - modelWeight; + return buildRecentOutcomeSnapshot( + (globalSnapshot.successCount * globalWeight) + (modelSnapshot.successCount * modelWeight), + (globalSnapshot.failureCount * globalWeight) + (modelSnapshot.failureCount * modelWeight), + ); +} + +function resolveStableFirstSuccessRate( + details: SiteRuntimeHealthDetails, + historicalSuccessRate: number | null | undefined, +): number { + const fallbackRate = historicalSuccessRate ?? SITE_RECENT_SUCCESS_FALLBACK_RATE; + return ( + (details.recentSuccessRate * details.recentConfidence) + + (fallbackRate * (1 - details.recentConfidence)) + ); +} + function resolveSiteRuntimeFailurePenalty(context: SiteRuntimeFailureContext = {}): number { const status = typeof context.status === 'number' ? context.status : 0; const errorText = (context.errorText || '').trim(); @@ -275,6 +473,18 @@ function resolveSiteRuntimeFailurePenalty(context: SiteRuntimeFailureContext = { return 0.4; } + if (isModelScopedRuntimeFailure({ status, errorText })) { + return 0.9; + } + + if (isProtocolRuntimeFailure({ status, errorText })) { + return 0.6; + } + + if (isValidationRuntimeFailure({ status, errorText })) { + return 0.25; + } + if (status >= 500 || matchesAnyPattern(SITE_TRANSIENT_FAILURE_PATTERNS, errorText)) { return 2.5; } @@ -287,18 +497,6 @@ function resolveSiteRuntimeFailurePenalty(context: SiteRuntimeFailureContext = { return 1.8; } - if (matchesAnyPattern(SITE_PROTOCOL_FAILURE_PATTERNS, errorText)) { - return 0.6; - } - - if (matchesAnyPattern(SITE_MODEL_FAILURE_PATTERNS, errorText)) { - return 0.9; - } - - if (matchesAnyPattern(SITE_VALIDATION_FAILURE_PATTERNS, errorText)) { - return 0.25; - } - if (status >= 400 && status < 500) { return 0.9; } @@ -312,6 +510,15 @@ function isTransientSiteRuntimeFailure(context: SiteRuntimeFailureContext = {}): if (isUsageLimitRateLimitFailure({ status, errorText })) { return false; } + if (isModelScopedRuntimeFailure({ status, errorText })) { + return false; + } + if (isProtocolRuntimeFailure({ status, errorText })) { + return false; + } + if (isValidationRuntimeFailure({ status, errorText })) { + return false; + } return status >= 500 || status === 429 || matchesAnyPattern(SITE_TRANSIENT_FAILURE_PATTERNS, errorText); } @@ -378,11 +585,15 @@ function hydrateSiteRuntimeHealthState(raw: unknown): SiteRuntimeHealthState | n if (!isRecord(raw)) return null; const lastUpdatedAtMs = readFiniteInteger(raw.lastUpdatedAtMs) ?? Date.now(); + const recentWindowUpdatedAtMs = readFiniteInteger(raw.recentWindowUpdatedAtMs) ?? lastUpdatedAtMs; return { penaltyScore: Math.max(0, readFiniteNumber(raw.penaltyScore) ?? 0), latencyEmaMs: readFiniteNumber(raw.latencyEmaMs), transientFailureStreak: Math.max(0, readFiniteInteger(raw.transientFailureStreak) ?? 0), lastTransientFailureAtMs: readNullableTimestamp(raw.lastTransientFailureAtMs), + recentSuccessCount: Math.max(0, readFiniteNumber(raw.recentSuccessCount) ?? 0), + recentFailureCount: Math.max(0, readFiniteNumber(raw.recentFailureCount) ?? 0), + recentWindowUpdatedAtMs: Math.max(0, recentWindowUpdatedAtMs), breakerLevel: Math.max(0, readFiniteInteger(raw.breakerLevel) ?? 0), breakerUntilMs: readNullableTimestamp(raw.breakerUntilMs), lastUpdatedAtMs: Math.max(0, lastUpdatedAtMs), @@ -397,6 +608,9 @@ function cloneSiteRuntimeHealthState(state: SiteRuntimeHealthState): SiteRuntime latencyEmaMs: state.latencyEmaMs, transientFailureStreak: state.transientFailureStreak, lastTransientFailureAtMs: state.lastTransientFailureAtMs, + recentSuccessCount: state.recentSuccessCount, + recentFailureCount: state.recentFailureCount, + recentWindowUpdatedAtMs: state.recentWindowUpdatedAtMs, breakerLevel: state.breakerLevel, breakerUntilMs: state.breakerUntilMs, lastUpdatedAtMs: state.lastUpdatedAtMs, @@ -413,6 +627,9 @@ function getOrCreateRuntimeHealthState(states: Map latencyEmaMs: null, transientFailureStreak: 0, lastTransientFailureAtMs: null, + recentSuccessCount: 0, + recentFailureCount: 0, + recentWindowUpdatedAtMs: nowMs, breakerLevel: 0, breakerUntilMs: null, lastUpdatedAtMs: nowMs, @@ -485,6 +702,9 @@ function getSiteRuntimeHealthDetails(siteId: number, modelName?: string | null, const modelState = modelKey ? getSiteModelRuntimeHealthState(siteId, modelKey) : null; const globalMultiplier = getRuntimeHealthMultiplier(globalState, nowMs); const modelMultiplier = modelState ? getRuntimeHealthMultiplier(modelState, nowMs) : 1; + const globalRecentSnapshot = getRecentOutcomeSnapshot(globalState, nowMs); + const modelRecentSnapshot = modelState ? getRecentOutcomeSnapshot(modelState, nowMs) : null; + const recentSnapshot = blendRecentOutcomeSnapshots(globalRecentSnapshot, modelRecentSnapshot); return { globalMultiplier, modelMultiplier, @@ -496,10 +716,15 @@ function getSiteRuntimeHealthDetails(siteId: number, modelName?: string | null, globalBreakerOpen: isRuntimeHealthBreakerOpen(globalState, nowMs), modelBreakerOpen: isRuntimeHealthBreakerOpen(modelState, nowMs), modelKey, + recentSuccessRate: recentSnapshot.successRate, + recentSampleCount: recentSnapshot.sampleCount, + recentConfidence: recentSnapshot.confidence, }; } function applyRuntimeHealthFailure(state: SiteRuntimeHealthState, context: SiteRuntimeFailureContext = {}, nowMs = Date.now()): void { + refreshRecentOutcomeWindow(state, nowMs); + state.recentFailureCount += 1; state.penaltyScore += resolveSiteRuntimeFailurePenalty(context); if (isTransientSiteRuntimeFailure(context)) { const lastTransientFailureAtMs = state.lastTransientFailureAtMs; @@ -525,6 +750,8 @@ function applyRuntimeHealthFailure(state: SiteRuntimeHealthState, context: SiteR } function applyRuntimeHealthSuccess(state: SiteRuntimeHealthState, latencyMs: number, nowMs = Date.now()): void { + refreshRecentOutcomeWindow(state, nowMs); + state.recentSuccessCount += 1; state.penaltyScore = Math.max(0, state.penaltyScore * 0.2 - 0.3); state.transientFailureStreak = 0; state.lastTransientFailureAtMs = null; @@ -551,6 +778,7 @@ function shouldPersistSiteRuntimeHealthState(state: SiteRuntimeHealthState, nowM if (isRuntimeHealthBreakerOpen(state, nowMs)) return true; if (getDecayedSiteRuntimePenalty(state, nowMs) >= SITE_RUNTIME_HEALTH_PERSIST_MIN_PENALTY) return true; + if (getRecentOutcomeSnapshot(state, nowMs).sampleCount > 0.01) return true; if ((state.latencyEmaMs ?? 0) > 0) return true; return (nowMs - lastTouchedAtMs) <= SITE_RUNTIME_HEALTH_PERSIST_IDLE_TTL_MS; } @@ -693,6 +921,8 @@ function recordSiteRuntimeSuccess(siteId: number, latencyMs: number, modelName?: export function resetSiteRuntimeHealthState(): void { siteRuntimeHealthStates.clear(); siteModelRuntimeHealthStates.clear(); + stableFirstObservationProgressByKey.clear(); + stableFirstObservationSiteCooldownByKey.clear(); siteRuntimeHealthLoaded = false; siteRuntimeHealthLoadPromise = null; if (siteRuntimeHealthSaveTimer) { @@ -960,13 +1190,16 @@ function patchCachedChannel(channelId: number, apply: (channel: ChannelRow) => v } } -export function invalidateTokenRouterCache(): void { - routeCacheSnapshot = { - loadedAt: 0, - routes: [], - }; - routeMatchCache.clear(); -} +export function invalidateTokenRouterCache(): void { + routeCacheSnapshot = { + loadedAt: 0, + routes: [], + }; + routeMatchCache.clear(); + stableFirstLastSelectedSiteByKey.clear(); + stableFirstObservationProgressByKey.clear(); + stableFirstObservationSiteCooldownByKey.clear(); +} function isSiteDisabled(status?: string | null): boolean { return (status || 'active') === 'disabled'; @@ -994,7 +1227,7 @@ export function filterRecentlyFailedCandidates !isChannelRecentlyFailed(candidate.channel, nowMs, avoidSec)); // If all channels failed recently, keep them all and let weight/random decide. @@ -1230,6 +1463,38 @@ function compareNullableTimeAsc(left?: string | null, right?: string | null): nu if (rightMs == null) return 1; return leftMs - rightMs; } + +function compareStableFirstCandidateOrder(left: RouteChannelCandidate, right: RouteChannelCandidate): number { + const selectionOrder = compareNullableTimeAsc( + left.channel.lastSelectedAt || left.channel.lastUsedAt, + right.channel.lastSelectedAt || right.channel.lastUsedAt, + ); + if (selectionOrder !== 0) return selectionOrder; + + const usedOrder = compareNullableTimeAsc(left.channel.lastUsedAt, right.channel.lastUsedAt); + if (usedOrder !== 0) return usedOrder; + + return (left.channel.id ?? 0) - (right.channel.id ?? 0); +} + +function resolveChannelRuntimeLoadMultiplier(snapshot: ProxyChannelLoadSnapshot): number { + if (!snapshot.sessionScoped || snapshot.concurrencyLimit <= 0) return 1; + + const activeRatio = clampNumber(snapshot.activeLeaseCount / Math.max(1, snapshot.concurrencyLimit), 0, 1.5); + const waitingRatio = clampNumber(snapshot.waitingCount / Math.max(1, snapshot.concurrencyLimit), 0, 3); + const activePenalty = activeRatio * 0.28; + const waitingPenalty = waitingRatio * 0.32; + const saturationPenalty = snapshot.saturated ? 0.12 : 0; + return clampNumber(1 - activePenalty - waitingPenalty - saturationPenalty, 0.18, 1); +} + +function formatChannelRuntimeLoad(snapshot: ProxyChannelLoadSnapshot): string { + if (!snapshot.sessionScoped || snapshot.concurrencyLimit <= 0) { + return '未限流'; + } + const multiplier = resolveChannelRuntimeLoadMultiplier(snapshot); + return `${multiplier.toFixed(2)}（活跃=${snapshot.activeLeaseCount}/${snapshot.concurrencyLimit}，等待=${snapshot.waitingCount}）`; +} function resolveEffectiveUnitCost(candidate: RouteChannelCandidate, modelName: string): CostSignal { const successCount = Math.max(0, candidate.channel.successCount ?? 0); @@ -1348,6 +1613,149 @@ function buildSiteHistoricalHealthMetrics(candidates: RouteChannelCandidate[]): return metrics; } +function buildStableFirstPoolPlan( + candidates: RouteChannelCandidate[], + modelName: string | ((candidate: RouteChannelCandidate) => string), + nowMs = Date.now(), +): StableFirstPoolPlan { + if (candidates.length <= 0) { + return { + primaryCandidates: [], + observationCandidates: [], + primarySiteIds: new Set(), + observationSiteIds: new Set(), + siteStateById: new Map(), + }; + } + + const resolveModelName = typeof modelName === 'function' + ? modelName + : (() => modelName); + const historicalBySiteId = buildSiteHistoricalHealthMetrics(candidates); + const leaderBySiteId = new Map(); + const siteStateById = new Map(); + + for (const candidate of candidates) { + const siteId = candidate.site.id; + const currentLeader = leaderBySiteId.get(siteId); + if (!currentLeader || compareStableFirstCandidateOrder(candidate, currentLeader) < 0) { + leaderBySiteId.set(siteId, candidate); + } + } + + for (const [siteId, leader] of leaderBySiteId.entries()) { + const healthDetails = getSiteRuntimeHealthDetails(siteId, resolveModelName(leader), nowMs); + const historical = historicalBySiteId.get(siteId); + const historicalTotalCalls = historical?.totalCalls ?? 0; + const effectiveSuccessRate = resolveStableFirstSuccessRate(healthDetails, historical?.successRate); + const trusted = ( + healthDetails.recentConfidence >= STABLE_FIRST_TRUSTED_RECENT_CONFIDENCE + || historicalTotalCalls >= STABLE_FIRST_TRUSTED_HISTORICAL_CALLS + ); + siteStateById.set(siteId, { + siteId, + leader, + effectiveSuccessRate, + trusted, + observationReason: null, + }); + } + + const allSiteStates = Array.from(siteStateById.values()).sort((left, right) => { + const rateDiff = right.effectiveSuccessRate - left.effectiveSuccessRate; + if (Math.abs(rateDiff) > 1e-9) return rateDiff > 0 ? 1 : -1; + return compareStableFirstCandidateOrder(left.leader, right.leader); + }); + const trustedSiteStates = allSiteStates.filter((state) => state.trusted); + const leaderPool = trustedSiteStates.length > 0 ? trustedSiteStates : allSiteStates; + + const primarySiteIds = new Set(); + const observationSiteIds = new Set(); + const bestRate = leaderPool[0]?.effectiveSuccessRate ?? 0; + const thresholdRate = bestRate > 0 + ? (bestRate * STABLE_FIRST_PRIMARY_SUCCESS_RATE_RATIO) + : 0; + + for (const state of allSiteStates) { + const inPrimary = leaderPool.length === 0 + ? true + : ( + leaderPool.some((leaderState) => leaderState.siteId === state.siteId) + && state.effectiveSuccessRate >= thresholdRate + ); + if (inPrimary) { + primarySiteIds.add(state.siteId); + continue; + } + observationSiteIds.add(state.siteId); + state.observationReason = state.trusted + ? '观察池：近期成功率暂时落后，仅灰度真实流量会命中' + : '观察池：近期样本不足，仅灰度真实流量会命中'; + } + + if (primarySiteIds.size <= 0 && allSiteStates.length > 0) { + primarySiteIds.add(allSiteStates[0].siteId); + observationSiteIds.delete(allSiteStates[0].siteId); + } + + return { + primaryCandidates: candidates.filter((candidate) => primarySiteIds.has(candidate.site.id)), + observationCandidates: candidates.filter((candidate) => observationSiteIds.has(candidate.site.id)), + primarySiteIds, + observationSiteIds, + siteStateById, + }; +} + +function shouldUseStableFirstObservationCandidate( + rotationKey: string, + observationCandidates: RouteChannelCandidate[], + nowMs = Date.now(), +): boolean { + if (!rotationKey || observationCandidates.length <= 0) return false; + const state = stableFirstObservationProgressByKey.get(rotationKey) ?? { + requestCount: 0, + lastObservationAtMs: null, + }; + if ((state.requestCount + 1) < STABLE_FIRST_OBSERVATION_REQUEST_INTERVAL) { + return false; + } + return observationCandidates.some((candidate) => { + const observedAtMs = stableFirstObservationSiteCooldownByKey.get(`${rotationKey}:${candidate.site.id}`) ?? null; + return observedAtMs == null || (nowMs - observedAtMs) >= STABLE_FIRST_OBSERVATION_SITE_COOLDOWN_MS; + }); +} + +function updateStableFirstObservationProgress( + rotationKey: string, + input: { + usedObservation: boolean; + selectedSiteId?: number | null; + nowMs?: number; + }, +): void { + if (!rotationKey) return; + const nowMs = input.nowMs ?? Date.now(); + const previous = stableFirstObservationProgressByKey.get(rotationKey) ?? { + requestCount: 0, + lastObservationAtMs: null, + }; + if (input.usedObservation) { + rememberStableFirstObservationProgressForKey(rotationKey, { + requestCount: 0, + lastObservationAtMs: nowMs, + }); + if (typeof input.selectedSiteId === 'number' && input.selectedSiteId > 0) { + rememberStableFirstObservationSiteCooldown(rotationKey, input.selectedSiteId, nowMs); + } + return; + } + rememberStableFirstObservationProgressForKey(rotationKey, { + requestCount: Math.max(0, previous.requestCount) + 1, + lastObservationAtMs: previous.lastObservationAtMs, + }); +} + function isExplicitTokenChannel(candidate: RouteChannelCandidate): boolean { return typeof candidate.channel.tokenId === 'number' && candidate.channel.tokenId > 0; } @@ -1518,9 +1926,9 @@ export class TokenRouter { summary.push(`按显示名命中：${normalizeRouteDisplayName(match.route.displayName)}`); summary.push('显示名仅用于聚合展示，实际转发模型按选中通道来源模型决定'); } - const availableByPriority = new Map(); - const candidates: RouteDecisionCandidate[] = []; - const candidateMap = new Map(); + const available: RouteChannelCandidate[] = []; + const candidates: RouteDecisionCandidate[] = []; + const candidateMap = new Map(); for (const row of match.channels) { const reasonParts = this.getCandidateEligibilityReasons(row, { @@ -1548,17 +1956,15 @@ export class TokenRouter { probability: 0, reason: eligible ? '可用' : reasonParts.join('、'), }; - candidates.push(candidate); - candidateMap.set(candidate.channelId, candidate); - - if (eligible) { - const priority = row.channel.priority ?? 0; - if (!availableByPriority.has(priority)) availableByPriority.set(priority, []); - availableByPriority.get(priority)!.push(row); - } - } - - if (availableByPriority.size === 0) { + candidates.push(candidate); + candidateMap.set(candidate.channelId, candidate); + + if (eligible) { + available.push(row); + } + } + + if (available.length === 0) { summary.push('没有可用通道（全部被禁用、站点不可用、冷却或令牌不可用）'); return { requestedModel, @@ -1572,10 +1978,7 @@ export class TokenRouter { } if (routeStrategy === 'round_robin') { - const rawOrdered = this.getRoundRobinCandidates(match.channels.filter((row) => { - const target = candidateMap.get(row.channel.id); - return !!target?.eligible; - })); + const rawOrdered = this.getRoundRobinCandidates(available); const breakerFiltered = filterSiteRuntimeBrokenCandidatesByModel(rawOrdered, runtimeModelResolver, nowMs); if (breakerFiltered.avoided.length > 0) { for (const item of breakerFiltered.avoided) { @@ -1646,6 +2049,182 @@ export class TokenRouter { }; } + if (routeStrategy === 'stable_first') { + const breakerFiltered = filterSiteRuntimeBrokenCandidatesByModel(available, runtimeModelResolver, nowMs); + if (breakerFiltered.avoided.length > 0) { + for (const item of breakerFiltered.avoided) { + const target = candidateMap.get(item.candidate.channel.id); + if (!target) continue; + target.reason = item.reason; + } + } + + const filteredCandidates = filterRecentlyFailedCandidates(breakerFiltered.candidates, nowMs); + const avoided = breakerFiltered.candidates.filter((row) => !filteredCandidates.some((item) => item.channel.id === row.channel.id)); + if (avoided.length > 0) { + for (const row of avoided) { + const target = candidateMap.get(row.channel.id); + if (!target) continue; + target.avoidedByRecentFailure = true; + target.reason = `最近失败，优先避让（${resolveFailureBackoffSec(row.channel.failCount)} 秒窗口）`; + } + } + + const rotationKey = this.buildStableFirstRotationKey(match.route.id, requestedModel); + const poolPlan = buildStableFirstPoolPlan( + filteredCandidates, + useChannelSourceModelForCost ? runtimeModelResolver : mappedModel, + nowMs, + ); + const observationDueNow = poolPlan.observationCandidates.length > 0 + && shouldUseStableFirstObservationCandidate(rotationKey, poolPlan.observationCandidates, nowMs); + const useObservationNow = poolPlan.observationCandidates.length > 0 + && (poolPlan.primaryCandidates.length <= 0 || observationDueNow); + const observationProgressState = stableFirstObservationProgressByKey.get(rotationKey) ?? { + requestCount: 0, + lastObservationAtMs: null, + }; + const remainingPrimaryRequestsBeforeObservation = poolPlan.primaryCandidates.length > 0 + ? Math.max(0, STABLE_FIRST_OBSERVATION_REQUEST_INTERVAL - (observationProgressState.requestCount + 1)) + : 0; + const observationBlockedByCooldown = poolPlan.primaryCandidates.length > 0 + && poolPlan.observationCandidates.length > 0 + && remainingPrimaryRequestsBeforeObservation === 0 + && !observationDueNow; + const primaryWeighted = this.calculateWeightedSelection( + poolPlan.primaryCandidates, + useChannelSourceModelForCost ? runtimeModelResolver : mappedModel, + downstreamPolicy, + nowMs, + 'stable_first', + rotationKey, + ); + const observationWeighted = poolPlan.observationCandidates.length > 0 + ? this.calculateWeightedSelection( + poolPlan.observationCandidates, + useChannelSourceModelForCost ? runtimeModelResolver : mappedModel, + downstreamPolicy, + nowMs, + 'stable_first', + `${rotationKey}:observe`, + ) + : { + selected: null, + details: [], + stableSiteCount: 0, + }; + + for (const detail of primaryWeighted.details) { + const target = candidateMap.get(detail.candidate.channel.id); + if (!target) continue; + target.probability = Number((detail.probability * (useObservationNow ? 0 : 100)).toFixed(2)); + if (target.eligible && !target.avoidedByRecentFailure) { + target.reason = useObservationNow + ? `主池：本次让位给观察池灰度请求；${detail.reason}` + : `主池：${detail.reason}`; + } + } + for (const detail of observationWeighted.details) { + const target = candidateMap.get(detail.candidate.channel.id); + if (!target) continue; + target.probability = Number((detail.probability * (useObservationNow ? 100 : 0)).toFixed(2)); + if (target.eligible && !target.avoidedByRecentFailure) { + const siteState = poolPlan.siteStateById.get(detail.candidate.site.id); + const observationWindowPrefix = useObservationNow + ? (poolPlan.primaryCandidates.length > 0 + ? '本次命中灰度真实请求' + : '当前主池为空，改由观察池承接') + : (observationBlockedByCooldown + ? '当前已到灰度窗口，但观察站点仍在冷却' + : `当前还需 ${remainingPrimaryRequestsBeforeObservation} 次主池请求`); + target.reason = poolPlan.observationSiteIds.has(detail.candidate.site.id) + ? `${siteState?.observationReason || '观察池'}；${observationWindowPrefix}；${detail.reason}` + : `观察池：${observationWindowPrefix}；${detail.reason}`; + } + } + + const weighted = useObservationNow + ? observationWeighted + : (primaryWeighted.selected ? primaryWeighted : observationWeighted); + if (!weighted.selected) { + summary.push('本次未选出通道'); + return { + requestedModel, + actualModel: mappedModel, + matched: true, + routeId: match.route.id, + modelPattern: match.route.modelPattern, + summary, + candidates, + }; + } + + const summaryParts = [`稳定优先：可用 ${available.length}`]; + if (poolPlan.primarySiteIds.size > 0) { + summaryParts.push(`主池站点 ${poolPlan.primarySiteIds.size}`); + } + if (poolPlan.observationSiteIds.size > 0) { + summaryParts.push(`观察池站点 ${poolPlan.observationSiteIds.size}`); + } + summaryParts.push('按近期成功率分层后按配置顺序轮询站点'); + if (poolPlan.observationSiteIds.size > 0) { + if (useObservationNow) { + summaryParts.push('本次命中观察池灰度流量'); + } else if (observationBlockedByCooldown) { + summaryParts.push('观察池已到灰度窗口，但候选站点仍在观察冷却'); + } else if (poolPlan.primaryCandidates.length <= 0) { + summaryParts.push('当前主池为空，由观察池承接流量'); + } else { + summaryParts.push(`观察池仅消耗少量真实请求灰度流量（当前还需 ${remainingPrimaryRequestsBeforeObservation} 次主池请求）`); + } + } + if (breakerFiltered.avoided.length > 0) { + const breakerSummaryLabel = breakerFiltered.avoided.some((item) => item.reason.includes('模型熔断')) + ? '运行时熔断避让' + : '站点熔断避让'; + summaryParts.push(`${breakerSummaryLabel} ${breakerFiltered.avoided.length}`); + } + if (avoided.length > 0) { + summaryParts.push(`最近失败避让 ${avoided.length}`); + } + summary.push(summaryParts.join('，')); + + const selectedChannel = candidateMap.get(weighted.selected.channel.id); + const selectedLabel = selectedChannel + ? `${selectedChannel.username} @ ${selectedChannel.siteName} / ${selectedChannel.tokenName}` + : `channel-${weighted.selected.channel.id}`; + const actualModel = resolveActualModelForSelectedChannel( + requestedModel, + match.route, + mappedModel, + weighted.selected.channel.sourceModel, + ); + summary.push(`最终选择：${selectedLabel}（P${weighted.selected.channel.priority ?? 0}）`); + if (actualModel !== mappedModel) { + summary.push(`实际转发模型：${actualModel}`); + } + + return { + requestedModel, + actualModel, + matched: true, + routeId: match.route.id, + modelPattern: match.route.modelPattern, + selectedChannelId: weighted.selected.channel.id, + selectedAccountId: weighted.selected.account.id, + selectedLabel, + summary, + candidates, + }; + } + + const availableByPriority = new Map(); + for (const row of available) { + const priority = row.channel.priority ?? 0; + if (!availableByPriority.has(priority)) availableByPriority.set(priority, []); + availableByPriority.get(priority)!.push(row); + } + const sortedPriorities = Array.from(availableByPriority.keys()).sort((a, b) => a - b); let selected: RouteChannelCandidate | null = null; let selectedPriority = 0; @@ -1679,7 +2258,7 @@ export class TokenRouter { useChannelSourceModelForCost ? runtimeModelResolver : mappedModel, downstreamPolicy, nowMs, - routeStrategy === 'stable_first' ? 'stable_first' : 'weighted', + 'weighted', ); for (const detail of weighted.details) { const target = candidateMap.get(detail.candidate.channel.id); @@ -1830,6 +2409,81 @@ export class TokenRouter { recordSiteRuntimeSuccess(account.siteId, latencyMs, modelName); } + async recordProbeSuccess(channelId: number, latencyMs: number, modelName?: string | null) { + await ensureSiteRuntimeHealthStateLoaded(); + const row = await db.select() + .from(schema.routeChannels) + .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id)) + .where(eq(schema.routeChannels.id, channelId)) + .get(); + if (!row) return; + + const ch = row.route_channels; + const account = row.accounts; + const affectedChannelIds = await loadCredentialScopedChannelIds(ch, account.id); + const needsChannelReset = !!ch.cooldownUntil + || !!ch.lastFailAt + || (ch.consecutiveFailCount ?? 0) > 0 + || (ch.cooldownLevel ?? 0) > 0; + + if (needsChannelReset) { + await db.update(schema.routeChannels).set({ + cooldownUntil: null, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + }).where(inArray(schema.routeChannels.id, affectedChannelIds)).run(); + + for (const affectedChannelId of affectedChannelIds) { + patchCachedChannel(affectedChannelId, (channel) => { + channel.cooldownUntil = null; + channel.lastFailAt = null; + channel.consecutiveFailCount = 0; + channel.cooldownLevel = 0; + }); + } + } else if (affectedChannelIds.length > 1) { + const scopedRows = await db.select({ + id: schema.routeChannels.id, + cooldownUntil: schema.routeChannels.cooldownUntil, + lastFailAt: schema.routeChannels.lastFailAt, + consecutiveFailCount: schema.routeChannels.consecutiveFailCount, + cooldownLevel: schema.routeChannels.cooldownLevel, + }) + .from(schema.routeChannels) + .where(inArray(schema.routeChannels.id, affectedChannelIds)) + .all(); + const siblingIdsToReset = scopedRows + .filter((candidate) => candidate.id !== channelId && ( + !!candidate.cooldownUntil + || !!candidate.lastFailAt + || (candidate.consecutiveFailCount ?? 0) > 0 + || (candidate.cooldownLevel ?? 0) > 0 + )) + .map((candidate) => candidate.id); + + if (siblingIdsToReset.length > 0) { + await db.update(schema.routeChannels).set({ + cooldownUntil: null, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + }).where(inArray(schema.routeChannels.id, siblingIdsToReset)).run(); + + for (const siblingId of siblingIdsToReset) { + patchCachedChannel(siblingId, (channel) => { + channel.cooldownUntil = null; + channel.lastFailAt = null; + channel.consecutiveFailCount = 0; + channel.cooldownLevel = 0; + }); + } + } + } + + recordSiteRuntimeSuccess(account.siteId, latencyMs, modelName); + } + /** * Clear persisted failure and cooldown state for the given channels. */ @@ -2006,6 +2660,64 @@ export class TokenRouter { }; } + if (routeStrategy === 'stable_first') { + const breakerFiltered = filterSiteRuntimeBrokenCandidatesByModel(available, runtimeModelResolver, nowMs); + const candidates = filterRecentlyFailedCandidates(breakerFiltered.candidates, nowMs); + const rotationKey = this.buildStableFirstRotationKey(match.route.id, requestedModel); + const poolPlan = buildStableFirstPoolPlan( + candidates, + requestedByDisplayName ? runtimeModelResolver : mappedModel, + nowMs, + ); + const shouldUseObservation = ( + poolPlan.observationCandidates.length > 0 + && ( + poolPlan.primaryCandidates.length <= 0 + || ( + recordSelection + && shouldUseStableFirstObservationCandidate(rotationKey, poolPlan.observationCandidates, nowMs) + ) + ) + ); + const selectionPool = shouldUseObservation + ? poolPlan.observationCandidates + : (poolPlan.primaryCandidates.length > 0 ? poolPlan.primaryCandidates : poolPlan.observationCandidates); + const selected = this.stableFirstSelect( + selectionPool, + requestedByDisplayName ? runtimeModelResolver : mappedModel, + downstreamPolicy, + nowMs, + shouldUseObservation ? `${rotationKey}:observe` : rotationKey, + ); + if (!selected) return null; + + const tokenValue = this.resolveChannelTokenValue(selected); + if (!tokenValue) return null; + if (recordSelection) { + rememberStableFirstSiteSelectionForKey(shouldUseObservation ? `${rotationKey}:observe` : rotationKey, selected.site.id); + updateStableFirstObservationProgress(rotationKey, { + usedObservation: shouldUseObservation, + selectedSiteId: selected.site.id, + nowMs, + }); + await this.recordChannelSelection(selected.channel.id); + } + + const actualModel = resolveActualModelForSelectedChannel( + requestedModel, + match.route, + mappedModel, + selected.channel.sourceModel, + ); + + return { + ...selected, + tokenValue, + tokenName: selected.token?.name || 'default', + actualModel, + }; + } + const layers = new Map(); for (const candidate of available) { const priority = candidate.channel.priority ?? 0; @@ -2018,14 +2730,7 @@ export class TokenRouter { const rawLayer = layers.get(priority) ?? []; const breakerFiltered = filterSiteRuntimeBrokenCandidatesByModel(rawLayer, runtimeModelResolver, nowMs); const candidates = filterRecentlyFailedCandidates(breakerFiltered.candidates, nowMs); - const selected = routeStrategy === 'stable_first' - ? this.stableFirstSelect( - candidates, - requestedByDisplayName ? runtimeModelResolver : mappedModel, - downstreamPolicy, - nowMs, - ) - : this.weightedRandomSelect( + const selected = this.weightedRandomSelect( candidates, requestedByDisplayName ? runtimeModelResolver : mappedModel, downstreamPolicy, @@ -2035,9 +2740,6 @@ export class TokenRouter { const tokenValue = this.resolveChannelTokenValue(selected); if (!tokenValue) continue; - if (routeStrategy === 'stable_first' && recordSelection) { - await this.recordChannelSelection(selected.channel.id); - } const actualModel = resolveActualModelForSelectedChannel( requestedModel, @@ -2098,6 +2800,12 @@ export class TokenRouter { const tokenValue = this.resolveChannelTokenValue(selected); if (!tokenValue) return null; if (recordSelection && (routeStrategy === 'round_robin' || routeStrategy === 'stable_first')) { + if (routeStrategy === 'stable_first') { + rememberStableFirstSiteSelectionForKey( + this.buildStableFirstRotationKey(match.route.id, requestedModel), + selected.site.id, + ); + } await this.recordChannelSelection(selected.channel.id); } @@ -2244,16 +2952,37 @@ export class TokenRouter { } private compareStableFirstCandidates(left: RouteChannelCandidate, right: RouteChannelCandidate): number { - const selectionOrder = compareNullableTimeAsc( - left.channel.lastSelectedAt || left.channel.lastUsedAt, - right.channel.lastSelectedAt || right.channel.lastUsedAt, - ); - if (selectionOrder !== 0) return selectionOrder; + return compareStableFirstCandidateOrder(left, right); + } - const usedOrder = compareNullableTimeAsc(left.channel.lastUsedAt, right.channel.lastUsedAt); - if (usedOrder !== 0) return usedOrder; + private buildStableFirstRotationKey(routeId: number, requestedModel: string): string { + const normalizedModel = normalizeModelAlias(requestedModel) + || normalizeRouteDisplayName(requestedModel).toLowerCase() + || String(routeId); + return `${routeId}:${normalizedModel}`; + } - return (left.channel.id ?? 0) - (right.channel.id ?? 0); + private getStableFirstSiteOrder(candidates: RouteChannelCandidate[], siteId: number): number { + let order = Number.POSITIVE_INFINITY; + for (const candidate of candidates) { + if (candidate.site.id !== siteId) continue; + order = Math.min(order, candidate.channel.priority ?? 0); + } + return Number.isFinite(order) ? order : 0; + } + + private getStableFirstOrderedSiteLeaderIndices( + candidates: RouteChannelCandidate[], + stableSiteLeaderIndices: number[], + ): number[] { + return [...stableSiteLeaderIndices].sort((leftIndex, rightIndex) => { + const leftSiteId = candidates[leftIndex]?.site.id ?? 0; + const rightSiteId = candidates[rightIndex]?.site.id ?? 0; + const orderDiff = this.getStableFirstSiteOrder(candidates, leftSiteId) + - this.getStableFirstSiteOrder(candidates, rightSiteId); + if (orderDiff !== 0) return orderDiff; + return (candidates[leftIndex]?.channel.id ?? 0) - (candidates[rightIndex]?.channel.id ?? 0); + }); } private async recordChannelSelection(channelId: number): Promise { @@ -2281,8 +3010,16 @@ export class TokenRouter { modelName: string | ((candidate: RouteChannelCandidate) => string), downstreamPolicy: DownstreamRoutingPolicy, nowMs = Date.now(), + stableFirstRotationKey?: string, ) { - return this.calculateWeightedSelection(candidates, modelName, downstreamPolicy, nowMs, 'stable_first').selected; + return this.calculateWeightedSelection( + candidates, + modelName, + downstreamPolicy, + nowMs, + 'stable_first', + stableFirstRotationKey, + ).selected; } private calculateWeightedSelection( @@ -2291,26 +3028,17 @@ export class TokenRouter { downstreamPolicy: DownstreamRoutingPolicy, nowMs = Date.now(), selectionMode: WeightedSelectionMode = 'weighted', - ) { + stableFirstRotationKey?: string, + ): WeightedSelectionResult { if (candidates.length === 0) { return { selected: null as RouteChannelCandidate | null, details: [] as Array<{ candidate: RouteChannelCandidate; probability: number; reason: string }>, - }; - } - - if (candidates.length === 1) { - return { - selected: candidates[0], - details: [{ - candidate: candidates[0], - probability: 1, - reason: selectionMode === 'stable_first' ? '稳定优先（唯一可用候选）' : '唯一可用候选', - }], + stableSiteCount: 0, }; } - - const { baseWeightFactor, valueScoreFactor, costWeight, balanceWeight, usageWeight } = config.routingWeights; + + const { baseWeightFactor, valueScoreFactor, costWeight, balanceWeight, usageWeight } = config.routingWeights; const resolveModelName = typeof modelName === 'function' ? modelName : (() => modelName); @@ -2318,6 +3046,13 @@ export class TokenRouter { const runtimeHealthDetails = candidates.map((candidate) => ( getSiteRuntimeHealthDetails(candidate.site.id, resolveModelName(candidate), nowMs) )); + const channelLoadSnapshots = candidates.map((candidate) => ( + proxyChannelCoordinator.getChannelLoadSnapshot({ + channelId: candidate.channel.id, + accountExtraConfig: candidate.account.extraConfig, + accountOauthProvider: candidate.account.oauthProvider, + }) + )); const valueScores = candidates.map((c, i) => { const unitCost = effectiveCosts[i]?.unitCost || 1; @@ -2347,32 +3082,46 @@ export class TokenRouter { const contributions = candidates.map((candidate, i) => { const siteChannels = Math.max(1, siteChannelCounts.get(candidate.site.id) || 1); + const runtimeMultiplier = runtimeHealthDetails[i]?.combinedMultiplier ?? 1; + const runtimeLoadMultiplier = resolveChannelRuntimeLoadMultiplier(channelLoadSnapshots[i]); + if (selectionMode === 'stable_first') { + const recentSuccessRate = resolveStableFirstSuccessRate( + runtimeHealthDetails[i], + siteHistoricalHealthMetrics.get(candidate.site.id)?.successRate, + ); + let contribution = Math.max(1e-4, recentSuccessRate ** 2); + contribution *= runtimeMultiplier; + contribution *= runtimeLoadMultiplier; + return contribution / siteChannels; + } + let contribution = baseContributions[i] / siteChannels; - const downstreamSiteMultiplier = downstreamPolicy.siteWeightMultipliers[candidate.site.id] ?? 1; - const normalizedDownstreamSiteMultiplier = - (Number.isFinite(downstreamSiteMultiplier) && downstreamSiteMultiplier > 0) - ? downstreamSiteMultiplier - : 1; - const siteGlobalWeight = - (Number.isFinite(candidate.site.globalWeight) && (candidate.site.globalWeight || 0) > 0) - ? (candidate.site.globalWeight as number) - : 1; + const downstreamSiteMultiplier = downstreamPolicy.siteWeightMultipliers[candidate.site.id] ?? 1; + const normalizedDownstreamSiteMultiplier = + (Number.isFinite(downstreamSiteMultiplier) && downstreamSiteMultiplier > 0) + ? downstreamSiteMultiplier + : 1; + const siteGlobalWeight = + (Number.isFinite(candidate.site.globalWeight) && (candidate.site.globalWeight || 0) > 0) + ? (candidate.site.globalWeight as number) + : 1; const combinedSiteWeight = siteGlobalWeight * normalizedDownstreamSiteMultiplier; if (combinedSiteWeight > 0 && Number.isFinite(combinedSiteWeight)) { contribution *= combinedSiteWeight; } - contribution *= runtimeHealthDetails[i]?.combinedMultiplier ?? 1; + contribution *= runtimeMultiplier; contribution *= siteHistoricalHealthMetrics.get(candidate.site.id)?.multiplier ?? 1; + contribution *= runtimeLoadMultiplier; // If upstream price is unknown and we are using fallback unit cost, // apply an explicit penalty so raising fallback cost meaningfully lowers probability. if (effectiveCosts[i]?.source === 'fallback') { contribution *= 1 / Math.max(1, effectiveCosts[i]?.unitCost || 1); - } - - return contribution; - }); + } + + return contribution; + }); const totalContribution = contributions.reduce((a, b) => a + b, 0); const rankedIndices = candidates.map((_, index) => index) @@ -2387,6 +3136,10 @@ export class TokenRouter { rankedIndices.forEach((candidateIndex, rank) => { rankByIndex.set(candidateIndex, rank + 1); }); + const stableSiteLeaderIndices = selectionMode === 'stable_first' + ? this.getStableFirstSiteLeaderIndices(candidates, contributions, rankedIndices) + : []; + const stableSiteIds = new Set(stableSiteLeaderIndices.map((index) => candidates[index]?.site.id).filter((siteId) => typeof siteId === 'number')); const details = candidates.map((candidate, i) => { const probability = totalContribution > 0 ? contributions[i] / totalContribution : 0; const weight = candidate.channel.weight ?? 10; @@ -2414,18 +3167,39 @@ export class TokenRouter { const historicalLatencyText = siteHistoricalHealth?.avgLatencyMs == null ? '—' : `${siteHistoricalHealth.avgLatencyMs}ms`; + const channelRuntimeLoad = channelLoadSnapshots[i]; const runtimeHealthText = siteRuntimeDetail.modelKey ? `${siteRuntimeDetail.combinedMultiplier.toFixed(2)}（站点=${siteRuntimeDetail.globalMultiplier.toFixed(2)}，模型=${siteRuntimeDetail.modelMultiplier.toFixed(2)}）` : `${siteRuntimeDetail.globalMultiplier.toFixed(2)}`; + const runtimeLoadText = formatChannelRuntimeLoad(channelRuntimeLoad); + const recentSuccessRateText = `${(siteRuntimeDetail.recentSuccessRate * 100).toFixed(1)}%`; + const stableFirstSuccessRate = resolveStableFirstSuccessRate(siteRuntimeDetail, siteHistoricalHealth?.successRate); + const stableFirstSuccessRateText = `${(stableFirstSuccessRate * 100).toFixed(1)}%`; + const stableSiteOrder = this.getStableFirstSiteOrder(candidates, candidate.site.id); const reasonPrefix = selectionMode === 'stable_first' - ? `稳定优先（综合评分第 ${rankByIndex.get(i) ?? 1} / ${candidates.length}` - : '按权重随机'; + ? ( + candidates.length === 1 + ? '稳定优先（唯一可用候选' + : `稳定优先（综合评分第 ${rankByIndex.get(i) ?? 1} / ${candidates.length}` + ) + : ( + candidates.length === 1 + ? '按权重随机（唯一可用候选' + : '按权重随机' + ); + const stablePoolText = selectionMode === 'stable_first' + ? `，轮询顺位=P${stableSiteOrder}` + : ''; return { candidate, probability, reason: selectionMode === 'stable_first' - ? `${reasonPrefix}，W=${weight}，成本=${costSourceText}:${(cost?.unitCost || 1).toFixed(6)}，站点权重=${siteGlobalWeight.toFixed(2)}x下游倍率=${normalizedDownstreamSiteMultiplier.toFixed(2)}=${combinedSiteWeight.toFixed(2)}，运行时健康=${runtimeHealthText}，历史健康=${siteHistoricalMultiplier.toFixed(2)}（成功率=${historicalSuccessRateText}，均延迟=${historicalLatencyText}，样本=${siteHistoricalHealth?.totalCalls ?? 0}），同站点通道=${siteChannels}，评分占比≈${(probability * 100).toFixed(1)}%）` - : `按权重随机（W=${weight}，成本=${costSourceText}:${(cost?.unitCost || 1).toFixed(6)}，站点权重=${siteGlobalWeight.toFixed(2)}x下游倍率=${normalizedDownstreamSiteMultiplier.toFixed(2)}=${combinedSiteWeight.toFixed(2)}，运行时健康=${runtimeHealthText}，历史健康=${siteHistoricalMultiplier.toFixed(2)}（成功率=${historicalSuccessRateText}，均延迟=${historicalLatencyText}，样本=${siteHistoricalHealth?.totalCalls ?? 0}），同站点通道=${siteChannels}，概率≈${(probability * 100).toFixed(1)}%）`, + ? `${reasonPrefix}，近期成功率=${recentSuccessRateText}（样本=${siteRuntimeDetail.recentSampleCount.toFixed(2)}，置信=${siteRuntimeDetail.recentConfidence.toFixed(2)}），回退成功率=${historicalSuccessRateText}，综合近期成功率=${stableFirstSuccessRateText}，运行时健康=${runtimeHealthText}，会话负载=${runtimeLoadText}，同站点通道=${siteChannels}${stablePoolText}，评分占比≈${(probability * 100).toFixed(1)}%）` + : ( + candidates.length === 1 + ? `${reasonPrefix}，W=${weight}，成本=${costSourceText}:${(cost?.unitCost || 1).toFixed(6)}，站点权重=${siteGlobalWeight.toFixed(2)}x下游倍率=${normalizedDownstreamSiteMultiplier.toFixed(2)}=${combinedSiteWeight.toFixed(2)}，运行时健康=${runtimeHealthText}，会话负载=${runtimeLoadText}，历史健康=${siteHistoricalMultiplier.toFixed(2)}（成功率=${historicalSuccessRateText}，均延迟=${historicalLatencyText}，样本=${siteHistoricalHealth?.totalCalls ?? 0}），同站点通道=${siteChannels}，概率≈${(probability * 100).toFixed(1)}%）` + : `按权重随机（W=${weight}，成本=${costSourceText}:${(cost?.unitCost || 1).toFixed(6)}，站点权重=${siteGlobalWeight.toFixed(2)}x下游倍率=${normalizedDownstreamSiteMultiplier.toFixed(2)}=${combinedSiteWeight.toFixed(2)}，运行时健康=${runtimeHealthText}，会话负载=${runtimeLoadText}，历史健康=${siteHistoricalMultiplier.toFixed(2)}（成功率=${historicalSuccessRateText}，均延迟=${historicalLatencyText}，样本=${siteHistoricalHealth?.totalCalls ?? 0}），同站点通道=${siteChannels}，概率≈${(probability * 100).toFixed(1)}%）` + ), }; }); @@ -2440,15 +3214,80 @@ export class TokenRouter { break; } } + } else { + selected = this.selectStableFirstCandidate( + candidates, + contributions, + rankedIndices, + stableFirstRotationKey, + ) ?? selected; } - return { selected, details }; + return { + selected, + details, + stableSiteCount: stableSiteIds.size, + }; } -} + + private getStableFirstSiteLeaderIndices( + candidates: RouteChannelCandidate[], + contributions: number[], + rankedIndices: number[], + ): number[] { + if (rankedIndices.length <= 1) return rankedIndices; + + const siteLeaderIndices: number[] = []; + const seenSiteIds = new Set(); + for (const index of rankedIndices) { + const siteId = candidates[index]?.site.id; + if (!Number.isFinite(siteId) || seenSiteIds.has(siteId)) continue; + seenSiteIds.add(siteId); + siteLeaderIndices.push(index); + } + + if (siteLeaderIndices.length <= 1) return siteLeaderIndices; + + const bestContribution = contributions[siteLeaderIndices[0] ?? rankedIndices[0] ?? 0] ?? 0; + const stableSiteLeaderIndices = siteLeaderIndices.filter((index) => ( + isContributionCloseToBest(contributions[index] ?? 0, bestContribution) + )); + + return stableSiteLeaderIndices.length > 0 ? stableSiteLeaderIndices : siteLeaderIndices; + } + + private selectStableFirstCandidate( + candidates: RouteChannelCandidate[], + contributions: number[], + rankedIndices: number[], + stableFirstRotationKey?: string, + ): RouteChannelCandidate | null { + const stableSiteLeaderIndices = this.getStableFirstSiteLeaderIndices(candidates, contributions, rankedIndices); + if (stableSiteLeaderIndices.length <= 0) return candidates[rankedIndices[0] ?? 0] ?? null; + + const orderedSiteLeaderIndices = this.getStableFirstOrderedSiteLeaderIndices(candidates, stableSiteLeaderIndices); + const lastSelectedSiteId = stableFirstRotationKey + ? stableFirstLastSelectedSiteByKey.get(stableFirstRotationKey) + : undefined; + const lastSelectedIndex = typeof lastSelectedSiteId === 'number' + ? orderedSiteLeaderIndices.findIndex((index) => candidates[index]?.site.id === lastSelectedSiteId) + : -1; + const selectedSiteLeader = orderedSiteLeaderIndices[lastSelectedIndex >= 0 + ? ((lastSelectedIndex + 1) % orderedSiteLeaderIndices.length) + : 0]; + if (selectedSiteLeader == null) return candidates[rankedIndices[0] ?? 0] ?? null; + + const selectedSiteId = candidates[selectedSiteLeader]?.site.id; + const topSiteCandidateIndex = rankedIndices.find((index) => candidates[index]?.site.id === selectedSiteId); + return topSiteCandidateIndex == null ? (candidates[selectedSiteLeader] ?? null) : (candidates[topSiteCandidateIndex] ?? null); + } +} export const tokenRouter = new TokenRouter(); export const __tokenRouterTestUtils = { resolveMappedModel, + getStableFirstRotationCacheSize: () => stableFirstLastSelectedSiteByKey.size, + rememberStableFirstSiteSelectionForKey, }; diff --git a/src/server/services/upstreamEndpointRuntime.ts b/src/server/services/upstreamEndpointRuntime.ts new file mode 100644 index 0000000..48aa10d --- /dev/null +++ b/src/server/services/upstreamEndpointRuntime.ts @@ -0,0 +1,6 @@ +export { + buildClaudeCountTokensUpstreamRequest, + buildUpstreamEndpointRequest, + resolveUpstreamEndpointCandidates, + type UpstreamEndpoint, +} from '../routes/proxy/upstreamEndpoint.js'; diff --git a/src/server/services/upstreamEndpointRuntimeMemory.ts b/src/server/services/upstreamEndpointRuntimeMemory.ts index ade692e..c81cc6f 100644 --- a/src/server/services/upstreamEndpointRuntimeMemory.ts +++ b/src/server/services/upstreamEndpointRuntimeMemory.ts @@ -33,6 +33,7 @@ export type EndpointCapabilityProfile = { hasNonImageFileInput: boolean; hasRemoteDocumentUrl: boolean; wantsNativeResponsesReasoning: boolean; + wantsContinuationAwareResponses: boolean; }; type EndpointRuntimeState = { @@ -88,6 +89,7 @@ export function buildEndpointCapabilityProfile(input?: { hasNonImageFileInput?: boolean; conversationFileSummary?: ConversationFileInputSummary; wantsNativeResponsesReasoning?: boolean; + wantsContinuationAwareResponses?: boolean; }; }): EndpointCapabilityProfile { const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary; @@ -107,6 +109,7 @@ export function buildEndpointCapabilityProfile(input?: { conversationFileSummary?.hasRemoteDocumentUrl === true ), wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true, + wantsContinuationAwareResponses: input?.requestCapabilities?.wantsContinuationAwareResponses === true, }; } @@ -131,6 +134,7 @@ function buildEndpointRuntimeStateKey(input: { capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles', capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl', capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning', + capabilityProfile.wantsContinuationAwareResponses ? 'continuation' : 'nocontinuation', ].join(':'); } @@ -247,6 +251,7 @@ export function getUpstreamEndpointRuntimeStateSnapshot(input: { hasNonImageFileInput?: boolean; conversationFileSummary?: ConversationFileInputSummary; wantsNativeResponsesReasoning?: boolean; + wantsContinuationAwareResponses?: boolean; }; }) { const capabilityProfile = buildEndpointCapabilityProfile({ @@ -349,6 +354,7 @@ export function recordUpstreamEndpointSuccess(input: { hasNonImageFileInput?: boolean; conversationFileSummary?: ConversationFileInputSummary; wantsNativeResponsesReasoning?: boolean; + wantsContinuationAwareResponses?: boolean; }; }): UpstreamEndpointRuntimeMemoryWrite | null { const capabilityProfile = buildEndpointCapabilityProfile({ @@ -390,6 +396,7 @@ export function recordUpstreamEndpointFailure(input: { hasNonImageFileInput?: boolean; conversationFileSummary?: ConversationFileInputSummary; wantsNativeResponsesReasoning?: boolean; + wantsContinuationAwareResponses?: boolean; }; }): UpstreamEndpointRuntimeMemoryWrite | null { const capabilityProfile = buildEndpointCapabilityProfile({ diff --git a/src/server/transformers/anthropic/messages/compatibility.ts b/src/server/transformers/anthropic/messages/compatibility.ts index 55a17ac..6dff710 100644 --- a/src/server/transformers/anthropic/messages/compatibility.ts +++ b/src/server/transformers/anthropic/messages/compatibility.ts @@ -1,5 +1,57 @@ import { type DownstreamFormat } from '../../shared/normalized.js'; +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function asTrimmedString(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +function hasAnthropicContinuationHint(body: Record): boolean { + if (asTrimmedString(body.previous_response_id)) return true; + if (asTrimmedString(body.prompt_cache_key)) return true; + if (isRecord(body.metadata) && asTrimmedString(body.metadata.user_id)) return true; + return false; +} + +function hasOrphanAnthropicToolResult(body: Record): boolean { + const messages = Array.isArray(body.messages) ? body.messages : []; + const seenToolUseIds = new Set(); + + for (const message of messages) { + if (!isRecord(message) || !Array.isArray(message.content)) continue; + + const role = asTrimmedString(message.role).toLowerCase(); + for (const block of message.content) { + if (!isRecord(block)) continue; + const type = asTrimmedString(block.type).toLowerCase(); + if (!type) continue; + + if (role === 'assistant' && type === 'tool_use') { + const toolUseId = asTrimmedString(block.id); + if (toolUseId) seenToolUseIds.add(toolUseId); + continue; + } + + if (type !== 'tool_result') continue; + const toolUseId = asTrimmedString(block.tool_use_id ?? block.toolUseId); + if (toolUseId && !seenToolUseIds.has(toolUseId)) { + return true; + } + } + } + + return false; +} + +export function shouldPreferResponsesForAnthropicContinuation( + body: Record | null | undefined, +): boolean { + if (!body) return false; + return hasAnthropicContinuationHint(body) && hasOrphanAnthropicToolResult(body); +} + export function shouldRetryNormalizedMessagesBody(input: { downstreamFormat: DownstreamFormat; endpointPath: string; diff --git a/src/server/transformers/anthropic/messages/conversion.test.ts b/src/server/transformers/anthropic/messages/conversion.test.ts index 215f3aa..f13c323 100644 --- a/src/server/transformers/anthropic/messages/conversion.test.ts +++ b/src/server/transformers/anthropic/messages/conversion.test.ts @@ -62,6 +62,30 @@ describe('sanitizeAnthropicMessagesBody', () => { ]); }); + it('normalizes primitive content arrays into text blocks and drops unsupported entries', () => { + const result = sanitizeAnthropicMessagesBody({ + model: 'claude-opus-4-6', + messages: [ + { + role: 'user', + content: ['hello', true, 42, null, { type: 'text', text: 'done' }], + }, + ], + }); + + expect(result.messages).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'hello' }, + { type: 'text', text: 'true' }, + { type: 'text', text: '42' }, + { type: 'text', text: 'done', cache_control: { type: 'ephemeral' } }, + ], + }, + ]); + }); + it('drops output_config.effort unless thinking stays adaptive', () => { const enabled = sanitizeAnthropicMessagesBody({ model: 'claude-opus-4-6', diff --git a/src/server/transformers/anthropic/messages/conversion.ts b/src/server/transformers/anthropic/messages/conversion.ts index 89dbade..751d9de 100644 --- a/src/server/transformers/anthropic/messages/conversion.ts +++ b/src/server/transformers/anthropic/messages/conversion.ts @@ -425,6 +425,43 @@ function sanitizeAnthropicContentBlock(item: Record): Record): Record | null { + const next: Record = { ...message }; + const rawContent = next.content; + + if (Array.isArray(rawContent)) { + const content = rawContent + .map((item) => { + if (isRecord(item)) return sanitizeAnthropicContentBlock(item); + if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') { + return sanitizeAnthropicContentBlock(toAnthropicTextBlock(String(item)) ?? {}); + } + return null; + }) + .filter((item): item is Record => item !== null); + if (content.length <= 0) return null; + next.content = content; + return next; + } + + if (rawContent === null || rawContent === undefined) { + return null; + } + + if (isRecord(rawContent)) { + const block = sanitizeAnthropicContentBlock(rawContent); + if (!block) return null; + next.content = [block]; + return next; + } + + if (typeof rawContent === 'number' || typeof rawContent === 'boolean') { + next.content = String(rawContent); + } + + return typeof next.content === 'string' ? next : null; +} + function isCacheableAnthropicMessageBlock(item: Record): boolean { const type = asTrimmedString(item.type).toLowerCase(); if (type === 'thinking' || type === 'redacted_thinking') return false; @@ -710,15 +747,9 @@ export function sanitizeAnthropicMessagesBody( const rawMessages = Array.isArray(sanitized.messages) ? sanitized.messages : []; if (rawMessages.length > 0) { sanitized.messages = rawMessages.map((message) => { - if (!isRecord(message)) return message; - const next = { ...message }; - if (Array.isArray(next.content)) { - next.content = next.content - .map((item) => (isRecord(item) ? sanitizeAnthropicContentBlock(item) : item)) - .filter((item) => item !== null); - } - return next; - }); + if (!isRecord(message)) return null; + return sanitizeAnthropicMessage(message); + }).filter((message): message is Record => message !== null); } if (autoOptimizeCacheControls) { diff --git a/src/server/transformers/anthropic/messages/inbound.test.ts b/src/server/transformers/anthropic/messages/inbound.test.ts index fa2ba0a..ea13529 100644 --- a/src/server/transformers/anthropic/messages/inbound.test.ts +++ b/src/server/transformers/anthropic/messages/inbound.test.ts @@ -43,6 +43,66 @@ describe('anthropicMessagesInbound', () => { }); }); + it('drops null-content turns instead of rejecting the whole Claude request', () => { + const result = anthropicMessagesInbound.parse({ + model: 'claude-opus-4-6', + max_tokens: 256, + messages: [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: null }, + ], + }); + + expect(result.error).toBeUndefined(); + expect(result.value?.parsed.upstreamBody.messages).toEqual([ + { role: 'user', content: 'hello' }, + ]); + expect(result.value?.parsed.claudeOriginalBody).toMatchObject({ + messages: [ + { role: 'user', content: 'hello' }, + ], + }); + }); + + it('preserves continuation hints for tool_result-only follow-up turns', () => { + const result = anthropicMessagesInbound.parse({ + model: 'claude-opus-4-6', + max_tokens: 256, + previous_response_id: 'resp_prev_1', + prompt_cache_key: 'cache-key-1', + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_missing', + content: [{ type: 'text', text: '{"matches":1}' }], + }, + { type: 'text', text: 'continue' }, + ], + }, + ], + }); + + expect(result.error).toBeUndefined(); + expect(result.value?.parsed.upstreamBody).toMatchObject({ + previous_response_id: 'resp_prev_1', + prompt_cache_key: 'cache-key-1', + messages: [ + { + role: 'tool', + tool_call_id: 'toolu_missing', + content: '{"matches":1}', + }, + { + role: 'user', + content: 'continue', + }, + ], + }); + }); + it('normalizes adaptive effort and tool choice at the inbound boundary', () => { const result = anthropicMessagesInbound.parse({ model: 'claude-opus-4-6', diff --git a/src/server/transformers/openai/responses/continuation.test.ts b/src/server/transformers/openai/responses/continuation.test.ts new file mode 100644 index 0000000..d2f9575 --- /dev/null +++ b/src/server/transformers/openai/responses/continuation.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from 'vitest'; + +import { + extractResponsesTerminalResponseId, + isResponsesPreviousResponseNotFoundError, + shouldInferResponsesPreviousResponseId, + stripResponsesPreviousResponseId, + withResponsesPreviousResponseId, +} from './continuation.js'; + +describe('responses continuation helpers', () => { + it('infers previous_response_id only for tool-output follow-up turns that omit it', () => { + expect(shouldInferResponsesPreviousResponseId({ + input: [{ + type: 'function_call_output', + call_id: 'call_1', + output: '{"ok":true}', + }], + }, 'resp_prev_1')).toBe(true); + + expect(shouldInferResponsesPreviousResponseId({ + previous_response_id: 'resp_existing', + input: [{ + type: 'function_call_output', + call_id: 'call_1', + output: '{"ok":true}', + }], + }, 'resp_prev_1')).toBe(false); + + expect(shouldInferResponsesPreviousResponseId({ + input: [{ + type: 'message', + role: 'user', + content: [{ type: 'input_text', text: 'hello' }], + }], + }, 'resp_prev_1')).toBe(false); + }); + + it('adds and removes previous_response_id without mutating other fields', () => { + const body = { + model: 'gpt-5.4', + input: [{ + type: 'function_call_output', + call_id: 'call_1', + output: '{"ok":true}', + }], + }; + + const withPrevious = withResponsesPreviousResponseId(body, 'resp_prev_1'); + expect(withPrevious).toMatchObject({ + model: 'gpt-5.4', + previous_response_id: 'resp_prev_1', + }); + + const stripped = stripResponsesPreviousResponseId(withPrevious); + expect(stripped.removed).toBe(true); + expect(stripped.body.previous_response_id).toBeUndefined(); + expect(stripped.body.input).toEqual(body.input); + }); + + it('detects previous_response_not_found from either raw text or structured payloads', () => { + expect(isResponsesPreviousResponseNotFoundError({ + rawErrText: JSON.stringify({ + error: { + code: 'previous_response_not_found', + message: 'previous_response_not_found', + }, + }), + })).toBe(true); + + expect(isResponsesPreviousResponseNotFoundError({ + payload: { + type: 'response.failed', + response: { + error: { + message: 'Previous response id not found', + type: 'invalid_request_error', + }, + }, + }, + })).toBe(true); + + expect(isResponsesPreviousResponseNotFoundError({ + rawErrText: JSON.stringify({ + error: { + code: 'rate_limit_exceeded', + message: 'too many requests', + }, + }), + })).toBe(false); + }); + + it('extracts terminal response ids from responses payloads and websocket terminal events', () => { + expect(extractResponsesTerminalResponseId({ + id: 'resp_1', + object: 'response', + status: 'completed', + })).toBe('resp_1'); + + expect(extractResponsesTerminalResponseId({ + type: 'response.failed', + response: { + id: 'resp_2', + status: 'failed', + error: { + message: 'tool failed', + }, + }, + })).toBe('resp_2'); + + expect(extractResponsesTerminalResponseId({ + type: 'response.created', + response: { + id: 'resp_3', + status: 'in_progress', + }, + })).toBeNull(); + + expect(extractResponsesTerminalResponseId({ + id: 'resp_in_progress', + object: 'response', + status: 'in_progress', + })).toBeNull(); + }); +}); diff --git a/src/server/transformers/openai/responses/continuation.ts b/src/server/transformers/openai/responses/continuation.ts new file mode 100644 index 0000000..79a7e31 --- /dev/null +++ b/src/server/transformers/openai/responses/continuation.ts @@ -0,0 +1,143 @@ +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function asTrimmedString(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +const RESPONSES_TOOL_OUTPUT_TYPES = new Set([ + 'function_call_output', + 'custom_tool_call_output', +]); + +const RESPONSES_TERMINAL_STATUSES = new Set([ + 'completed', + 'failed', + 'incomplete', +]); + +function collectResponsesErrorFragments(value: unknown): string[] { + if (!value) return []; + if (typeof value === 'string') return [value]; + if (!isRecord(value)) return []; + + const fragments = [ + value.type, + value.code, + value.message, + value.reason, + ] + .map((entry) => asTrimmedString(entry)) + .filter(Boolean); + + if (isRecord(value.error)) { + fragments.push(...collectResponsesErrorFragments(value.error)); + } + + if (isRecord(value.response)) { + fragments.push(...collectResponsesErrorFragments(value.response)); + } + + if (isRecord(value.incomplete_details)) { + fragments.push(...collectResponsesErrorFragments(value.incomplete_details)); + } + + return fragments; +} + +function hasResponsesToolOutput(input: unknown): boolean { + if (!Array.isArray(input)) return false; + return input.some((item) => { + if (!isRecord(item)) return false; + const type = asTrimmedString(item.type).toLowerCase(); + if (!RESPONSES_TOOL_OUTPUT_TYPES.has(type)) return false; + return asTrimmedString(item.call_id ?? item.id).length > 0; + }); +} + +export function shouldInferResponsesPreviousResponseId( + body: Record | null | undefined, + candidatePreviousResponseId: unknown, +): candidatePreviousResponseId is string { + if (!body) return false; + if (asTrimmedString(body.previous_response_id)) return false; + const candidate = asTrimmedString(candidatePreviousResponseId); + if (!candidate) return false; + return hasResponsesToolOutput(body.input); +} + +export function withResponsesPreviousResponseId( + body: Record, + previousResponseId: string, +): Record { + return { + ...body, + previous_response_id: previousResponseId.trim(), + }; +} + +export function stripResponsesPreviousResponseId( + body: Record, +): { body: Record; removed: boolean } { + if (!Object.prototype.hasOwnProperty.call(body, 'previous_response_id')) { + return { body, removed: false }; + } + const next = { ...body }; + delete next.previous_response_id; + return { body: next, removed: true }; +} + +export function isResponsesPreviousResponseNotFoundError(input: { + rawErrText?: string | null; + payload?: unknown; +}): boolean { + const fragments = [ + ...collectResponsesErrorFragments(input.payload), + ]; + const rawErrText = asTrimmedString(input.rawErrText); + if (rawErrText) fragments.push(rawErrText); + const combined = fragments.join(' ').toLowerCase(); + if (!combined) return false; + return ( + combined.includes('previous_response_not_found') + || /previous[\s_-]*response(?:[\s_-]*(?:id|identifier))?[\s_-]*not[\s_-]*found/i.test(combined) + ); +} + +export function extractResponsesTerminalResponseId(payload: unknown): string | null { + if (!isRecord(payload)) return null; + + const type = asTrimmedString(payload.type).toLowerCase(); + if ( + type === 'response.completed' + || type === 'response.failed' + || type === 'response.incomplete' + ) { + if (isRecord(payload.response)) { + const responseId = asTrimmedString(payload.response.id); + return responseId || null; + } + return null; + } + + const object = asTrimmedString(payload.object).toLowerCase(); + const status = asTrimmedString(payload.status).toLowerCase(); + if ( + (object === 'response' && RESPONSES_TERMINAL_STATUSES.has(status)) + || RESPONSES_TERMINAL_STATUSES.has(status) + ) { + const responseId = asTrimmedString(payload.id); + return responseId || null; + } + + if (isRecord(payload.response)) { + const nestedStatus = asTrimmedString(payload.response.status).toLowerCase(); + if (RESPONSES_TERMINAL_STATUSES.has(nestedStatus)) { + const responseId = asTrimmedString(payload.response.id); + return responseId || null; + } + } + + return null; +} diff --git a/src/server/transformers/shared/chatFormatsCore.ts b/src/server/transformers/shared/chatFormatsCore.ts index 50c68e4..ca1273e 100644 --- a/src/server/transformers/shared/chatFormatsCore.ts +++ b/src/server/transformers/shared/chatFormatsCore.ts @@ -1234,6 +1234,12 @@ export function convertClaudeRequestToOpenAiBody(body: Record): if (body.tools !== undefined) payload.tools = convertClaudeToolsToOpenAiChat(body.tools); if (body.tool_choice !== undefined) payload.tool_choice = convertClaudeToolChoiceToOpenAiChat(body.tool_choice); + const promptCacheKey = asTrimmedString(body.prompt_cache_key); + if (promptCacheKey) payload.prompt_cache_key = promptCacheKey; + + const previousResponseId = asTrimmedString(body.previous_response_id); + if (previousResponseId) payload.previous_response_id = previousResponseId; + return { model, stream, messages, payload }; } diff --git a/src/web/api.ts b/src/web/api.ts index e607791..9350984 100644 --- a/src/web/api.ts +++ b/src/web/api.ts @@ -342,6 +342,7 @@ export type RuntimeRoutingWeightsPayload = { export type RuntimeSettingsPayload = { proxyToken?: string; systemProxyUrl?: string; + modelAvailabilityProbeEnabled?: boolean; codexUpstreamWebsocketEnabled?: boolean; disableCrossProtocolFallback?: boolean; proxySessionChannelConcurrencyLimit?: number; diff --git a/src/web/i18n.supplement.ts b/src/web/i18n.supplement.ts index 8e7b4c0..0ce210d 100644 --- a/src/web/i18n.supplement.ts +++ b/src/web/i18n.supplement.ts @@ -41,6 +41,7 @@ export const zhToEnSupplemental: Record = { '按成本、延迟、成功率自动选择最优通道，故障自动转移': 'Automatically select the optimal channel based on cost, delay, and success rate, and automatically transfer failures', '按模型过滤': 'Filter by model', '按权重随机（W=${weight}，成本=${costSourceText}:${(cost?.unitCost || 1).toFixed(6)}，同站点通道=${siteChannels}，概率≈${(probability * 100).toFixed(1)}%）': 'Randomly by weight (W=${weight}, cost=${costSourceText}:${(cost?.unitCost || 1).toFixed(6)}, same-site channel=${siteChannels}, probability≈${(probability * 100).toFixed(1)}%)', + 'P 值是硬优先级，只会在当前最高可用优先级内结合权重、成本和健康度随机选择': 'P value is a hard priority. Selection stays within the highest available priority tier, then chooses randomly using weight, cost, and health signals.', '保存': 'save', '保存 Cookie': 'Save Cookie', '保存 Cookie 失败': 'Failed to save cookie', @@ -573,6 +574,7 @@ export const zhToEnSupplemental: Record = { '系统会根据模型可用性自动生成路由。优先级按 P0/P1 等桶管理，同一桶内可有多个通道；拖动通道或灰色分隔线即可调整。精确模型路由会自动过滤只支持该模型的账号和令牌。群组路由中的优先级调整会直接回写来源通道。选中概率表示请求到达时该通道被选中的概率。成本来源优先级为：实测成本 → 账号配置成本 → 目录参考价 → 默认回退单价。': 'The system automatically generates routes based on model availability. Priorities are managed as P0/P1 buckets, and multiple channels can share the same bucket; drag channels or the gray separators to adjust them. Exact model routes automatically filter accounts and tokens that only support that model. Priority edits inside group routes write back to the source channels directly. The selection probability indicates the probability of the channel being selected when the request arrives. The priority of cost sources is: measured cost → account configuration cost → catalog reference price → default fallback unit price.', '该群组会将多个来源模型聚合为一个对外模型名；这里调整优先级桶时会直接回写来源通道。若某个来源模型被其他群组复用，保存前会提示影响范围。': 'This group aggregates multiple source models into one public model name. Priority bucket edits here write back to the source channels directly. If a source model is reused by other groups, the impact is shown before saving.', '通配符路由按请求实时决策；下方优先级桶在整条路由内全局生效，来源模型只作为通道标签展示。': 'Wildcard routes decide per request. The priority buckets below apply globally across the whole route, while source models are shown only as channel labels.', + '系统会根据模型可用性自动生成路由；精确模型路由会自动过滤只支持该模型的账号和令牌。': 'Routes are auto-generated by model availability; exact model routes auto-filter accounts and tokens that support that model.', '系统设置': 'System settings', '显示调试': 'show debugging', '项目链接': 'Project link', @@ -588,10 +590,16 @@ export const zhToEnSupplemental: Record = { '选择站点': 'Select site', '选择账号': 'Select account', '选择账号后同步站点令牌': 'Synchronize site token after selecting account', + '选中概率用于解释当前策略下这一次请求更可能落到哪里；轮询和稳定优先更适合把它当作顺序参考。': 'Selection probability explains where this request is more likely to land under the current strategy; for round robin and stable first it is better treated as an order hint.', + '先避开最近失败或不健康站点，再在稳定池里按顺序轮询；P 值表示轮询顺位': 'Avoid recently failed or unhealthy sites first, then rotate through the stable pool in order; P value means rotation order.', '讯飞星火': 'iFlytek Spark', '延迟': 'Delay', '验证': 'verify', '验证 Token': 'Verify Token', + '只要更高优先级还有可用通道，后面的通道本次就不会参与选择。': 'As long as a higher-priority tier still has available channels, later channels will not participate in this selection.', + '忽略 P 值，按全局顺序依次调用；连续失败 3 次后进入分级冷却': 'Ignore P value and call in global order; after 3 consecutive failures the channel enters staged cooldown.', + '当前策略不看 P 值；如果之后切回其他策略，拖拽保存的顺序仍会保留。': 'This strategy does not use P value; if you switch back to other strategies later, the saved drag order is still kept.', + '当前策略下，稳定站点会按 P 顺序轮换；不稳定站点会被自动降权或临时避让。': 'Under this strategy, stable sites rotate by P order; unstable sites are automatically downweighted or temporarily avoided.', '验证成功: ${result.userInfo?.username || \'未知用户\'}': 'Verification successful: ${result.userInfo?.username || \'未知用户\'}', '验证失败': 'Authentication failed', '验证中...': 'Verifying...', diff --git a/src/web/i18n.tsx b/src/web/i18n.tsx index 52885a2..9e24fdb 100644 --- a/src/web/i18n.tsx +++ b/src/web/i18n.tsx @@ -197,6 +197,14 @@ const zhToEn: Record = { '系统会根据模型可用性自动生成路由。优先级按 P0/P1 等桶管理，同一桶内可有多个通道；拖动通道或灰色分隔线即可调整。精确模型路由会自动过滤只支持该模型的账号和令牌。群组路由中的优先级调整会直接回写来源通道。选中概率表示请求到达时该通道被选中的概率。成本来源优先级为：实测成本 → 账号配置成本 → 目录参考价 → 默认回退单价。': 'Routes are auto-generated by model availability. Priorities are managed as P0/P1 buckets, and multiple channels can share the same bucket; drag channels or the gray separators to adjust them. Exact model routes auto-filter accounts and tokens that support that model. Priority edits inside group routes write back to the source channels directly. Selection probability is the chance a channel is chosen. Cost priority: measured cost -> account configured cost -> catalog reference price -> default fallback unit price.', '该群组会将多个来源模型聚合为一个对外模型名；这里调整优先级桶时会直接回写来源通道。若某个来源模型被其他群组复用，保存前会提示影响范围。': 'This group aggregates multiple source models into one public model name. Priority bucket edits here write back to the source channels directly. If a source model is reused by other groups, you will be warned about the impact before saving.', '通配符路由按请求实时决策；下方优先级桶在整条路由内全局生效，来源模型只作为通道标签展示。': 'Wildcard routes decide per request. The priority buckets below apply globally across the whole route, while source models are shown only as channel labels.', + '系统会根据模型可用性自动生成路由；精确模型路由会自动过滤只支持该模型的账号和令牌。': 'Routes are auto-generated by model availability; exact model routes auto-filter accounts and tokens that support that model.', + 'P 值是硬优先级，只会在当前最高可用优先级内结合权重、成本和健康度随机选择': 'P value is a hard priority. Selection stays within the highest available priority tier, then chooses randomly using weight, cost, and health signals.', + '忽略 P 值，按全局顺序依次调用；连续失败 3 次后进入分级冷却': 'Ignore P value and call in global order; after 3 consecutive failures the channel enters staged cooldown.', + '先避开最近失败或不健康站点，再在稳定池里按顺序轮询；P 值表示轮询顺位': 'Avoid recently failed or unhealthy sites first, then rotate through the stable pool in order; P value means rotation order.', + '当前策略不看 P 值；如果之后切回其他策略，拖拽保存的顺序仍会保留。': 'This strategy does not use P value; if you switch back to other strategies later, the saved drag order is still kept.', + '当前策略下，稳定站点会按 P 顺序轮换；不稳定站点会被自动降权或临时避让。': 'Under this strategy, stable sites rotate by P order; unstable sites are automatically downweighted or temporarily avoided.', + '只要更高优先级还有可用通道，后面的通道本次就不会参与选择。': 'As long as a higher-priority tier still has available channels, later channels will not participate in this selection.', + '选中概率用于解释当前策略下这一次请求更可能落到哪里；轮询和稳定优先更适合把它当作顺序参考。': 'Selection probability explains where this request is more likely to land under the current strategy; for round robin and stable first it is better treated as an order hint.', '代理端点': 'Proxy Endpoints', '路由行为': 'Routing Behavior', '指标口径': 'Metric Notes', diff --git a/src/web/pages/Settings.tsx b/src/web/pages/Settings.tsx index 83a3510..6b0e458 100644 --- a/src/web/pages/Settings.tsx +++ b/src/web/pages/Settings.tsx @@ -8,6 +8,7 @@ import { useAnimatedVisibility } from '../components/useAnimatedVisibility.js'; import ModernSelect from '../components/ModernSelect.js'; import DownstreamApiKeyModal from './settings/DownstreamApiKeyModal.js'; import FactoryResetModal from './settings/FactoryResetModal.js'; +import ModelAvailabilityProbeConfirmModal from './settings/ModelAvailabilityProbeConfirmModal.js'; import RouteSelectorModal from './settings/RouteSelectorModal.js'; import UpdateCenterSection from './settings/UpdateCenterSection.js'; import { @@ -28,6 +29,7 @@ import { generateDownstreamSkKey } from './helpers/generateDownstreamSkKey.js'; const PROXY_TOKEN_PREFIX = 'sk-'; const FACTORY_RESET_ADMIN_TOKEN = 'change-me-admin-token'; const FACTORY_RESET_CONFIRM_SECONDS = 3; +const MODEL_AVAILABILITY_PROBE_CONFIRM_TEXT = '我确认我使用的中转站全部允许批量测活，如因开启此功能被中转站封号，自行负责。'; const SECONDS_PER_DAY = 24 * 60 * 60; const ROUTE_COOLDOWN_UNIT_OPTIONS = [ { value: 'second', label: '秒', multiplierSec: 1 }, @@ -58,6 +60,7 @@ type RuntimeSettings = { logCleanupUsageLogsEnabled: boolean; logCleanupProgramLogsEnabled: boolean; logCleanupRetentionDays: number; + modelAvailabilityProbeEnabled: boolean; codexUpstreamWebsocketEnabled: boolean; disableCrossProtocolFallback: boolean; proxySessionChannelConcurrencyLimit: number; @@ -235,6 +238,7 @@ export default function Settings() { logCleanupUsageLogsEnabled: false, logCleanupProgramLogsEnabled: false, logCleanupRetentionDays: 30, + modelAvailabilityProbeEnabled: false, codexUpstreamWebsocketEnabled: false, disableCrossProtocolFallback: false, proxySessionChannelConcurrencyLimit: 2, @@ -255,6 +259,7 @@ export default function Settings() { const [testingCheckin, setTestingCheckin] = useState(false); const [savingToken, setSavingToken] = useState(false); const [savingSystemProxy, setSavingSystemProxy] = useState(false); + const [savingModelAvailabilityProbe, setSavingModelAvailabilityProbe] = useState(false); const [savingProxyTransport, setSavingProxyTransport] = useState(false); const [testingSystemProxy, setTestingSystemProxy] = useState(false); const [systemProxyTestState, setSystemProxyTestState] = useState(null); @@ -300,6 +305,10 @@ export default function Settings() { const downstreamModalPresence = useAnimatedVisibility(downstreamModalOpen, 220); const [selectorOpen, setSelectorOpen] = useState(false); const selectorModalPresence = useAnimatedVisibility(selectorOpen, 220); + const [modelAvailabilityProbeConfirmOpen, setModelAvailabilityProbeConfirmOpen] = useState(false); + const modelAvailabilityProbeConfirmPresence = useAnimatedVisibility(modelAvailabilityProbeConfirmOpen, 220); + const [modelAvailabilityProbeConfirmationInput, setModelAvailabilityProbeConfirmationInput] = useState(''); + const [savedModelAvailabilityProbeEnabled, setSavedModelAvailabilityProbeEnabled] = useState(false); const [factoryResetOpen, setFactoryResetOpen] = useState(false); const factoryResetPresence = useAnimatedVisibility(factoryResetOpen, 220); const [factoryResetting, setFactoryResetting] = useState(false); @@ -377,6 +386,12 @@ export default function Settings() { })); }, [migrationDialect]); + useEffect(() => { + if (!modelAvailabilityProbeConfirmOpen) { + setModelAvailabilityProbeConfirmationInput(''); + } + }, [modelAvailabilityProbeConfirmOpen]); + useEffect(() => { if (!factoryResetOpen) { setFactoryResetSecondsLeft(FACTORY_RESET_CONFIRM_SECONDS); @@ -469,6 +484,7 @@ export default function Settings() { logCleanupRetentionDays: Number(runtimeInfo.logCleanupRetentionDays) >= 1 ? Math.trunc(Number(runtimeInfo.logCleanupRetentionDays)) : 30, + modelAvailabilityProbeEnabled: !!runtimeInfo.modelAvailabilityProbeEnabled, codexUpstreamWebsocketEnabled: !!runtimeInfo.codexUpstreamWebsocketEnabled, disableCrossProtocolFallback: !!runtimeInfo.disableCrossProtocolFallback, proxySessionChannelConcurrencyLimit: Number(runtimeInfo.proxySessionChannelConcurrencyLimit) >= 0 @@ -499,6 +515,7 @@ export default function Settings() { globalBlockedBrands: Array.isArray(runtimeInfo.globalBlockedBrands) ? runtimeInfo.globalBlockedBrands : [], globalAllowedModels: Array.isArray(runtimeInfo.globalAllowedModels) ? runtimeInfo.globalAllowedModels : [], }); + setSavedModelAvailabilityProbeEnabled(!!runtimeInfo.modelAvailabilityProbeEnabled); setBlockedBrands(Array.isArray(runtimeInfo.globalBlockedBrands) ? runtimeInfo.globalBlockedBrands : []); setAllowedModels(Array.isArray(runtimeInfo.globalAllowedModels) ? runtimeInfo.globalAllowedModels : []); setProxyErrorKeywordsText( @@ -646,6 +663,42 @@ export default function Settings() { } }; + const persistModelAvailabilityProbeSetting = async (enabled: boolean) => { + setSavingModelAvailabilityProbe(true); + try { + const res = await api.updateRuntimeSettings({ + modelAvailabilityProbeEnabled: enabled, + }); + const nextEnabled = typeof res?.modelAvailabilityProbeEnabled === 'boolean' + ? res.modelAvailabilityProbeEnabled + : enabled; + setRuntime((prev) => ({ + ...prev, + modelAvailabilityProbeEnabled: nextEnabled, + })); + setSavedModelAvailabilityProbeEnabled(nextEnabled); + setModelAvailabilityProbeConfirmOpen(false); + setModelAvailabilityProbeConfirmationInput(''); + toast.success(nextEnabled ? '批量测活已开启' : '批量测活已关闭'); + } catch (err: any) { + toast.error(err?.message || '保存失败'); + } finally { + setSavingModelAvailabilityProbe(false); + } + }; + + const saveModelAvailabilityProbeSettings = async () => { + if (runtime.modelAvailabilityProbeEnabled === savedModelAvailabilityProbeEnabled) { + toast.info('批量测活设置未变化'); + return; + } + if (runtime.modelAvailabilityProbeEnabled) { + setModelAvailabilityProbeConfirmOpen(true); + return; + } + await persistModelAvailabilityProbeSetting(false); + }; + const saveProxyTransportSettings = async () => { setSavingProxyTransport(true); try { @@ -1009,6 +1062,16 @@ export default function Settings() { setFactoryResetOpen(false); }; + const closeModelAvailabilityProbeConfirmModal = () => { + if (savingModelAvailabilityProbe) return; + setModelAvailabilityProbeConfirmOpen(false); + }; + + const handleConfirmModelAvailabilityProbe = async () => { + if (modelAvailabilityProbeConfirmationInput.trim() !== MODEL_AVAILABILITY_PROBE_CONFIRM_TEXT) return; + await persistModelAvailabilityProbeSetting(true); + }; + const handleFactoryReset = async () => { if (factoryResetSecondsLeft > 0 || factoryResetting) return; setFactoryResetting(true); @@ -1417,6 +1480,32 @@ export default function Settings() { +
+
批量测活
+
+ 默认关闭。开启后，metapi 会在后台定时对活跃账号模型发送最小化探测请求，用来校正“/models 能看到但实际不可用”的假阳性。 +
+
+ 只有在你确认自己使用的中转站明确允许批量测活时才应该开启。若上游不允许，这类探测可能带来封号或风控风险。 +
+ +
+ 当前状态：{savedModelAvailabilityProbeEnabled ? '已启用' : '已关闭'}。首次开启必须手动输入确认语句。 +
+
+ +
+
+
下游访问令牌（PROXY_TOKEN）
@@ -2108,6 +2197,15 @@ export default function Settings() { onClose={closeFactoryResetModal} onConfirm={handleFactoryReset} /> + {/* Info tip */} -
- {tr('系统会根据模型可用性自动生成路由。优先级按 P0/P1 等桶管理，同一桶内可有多个通道；拖动通道或灰色分隔线即可调整。精确模型路由会自动过滤只支持该模型的账号和令牌。群组路由中的优先级调整会直接回写来源通道。选中概率表示请求到达时该通道被选中的概率。成本来源优先级为：实测成本 → 账号配置成本 → 目录参考价 → 默认回退单价。')} +
+
{tr('系统会根据模型可用性自动生成路由。优先级按 P0/P1 等桶管理，同一桶内可有多个通道；拖动通道或灰色分隔线即可调整。精确模型路由会自动过滤只支持该模型的账号和令牌。群组路由中的优先级调整会直接回写来源通道。选中概率表示请求到达时该通道被选中的概率。成本来源优先级为：实测成本 → 账号配置成本 → 目录参考价 → 默认回退单价。')}
+
{`${getRouteRoutingStrategyLabel('weighted')}：${getRouteRoutingStrategyDescription('weighted')}`}
+
{`${getRouteRoutingStrategyLabel('round_robin')}：${getRouteRoutingStrategyDescription('round_robin')}`}
+
{`${getRouteRoutingStrategyLabel('stable_first')}：${getRouteRoutingStrategyDescription('stable_first')}`}
+
{tr('选中概率用于解释当前策略下这一次请求更可能落到哪里；轮询和稳定优先更适合把它当作顺序参考。')}
+
{tr('成本来源优先级：实测成本 → 账号配置成本 → 目录参考价 → 默认回退单价。')}
{/* Manual route panel */} diff --git a/src/web/pages/settings.model-availability-probe.test.tsx b/src/web/pages/settings.model-availability-probe.test.tsx new file mode 100644 index 0000000..48b4b93 --- /dev/null +++ b/src/web/pages/settings.model-availability-probe.test.tsx @@ -0,0 +1,157 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { act, create, type ReactTestInstance, type ReactTestRenderer } from 'react-test-renderer'; +import { MemoryRouter } from 'react-router-dom'; +import { ToastProvider } from '../components/Toast.js'; +import Settings from './Settings.js'; + +const MODEL_AVAILABILITY_PROBE_CONFIRM_TEXT = '我确认我使用的中转站全部允许批量测活，如因开启此功能被中转站封号，自行负责。'; + +const { apiMock } = vi.hoisted(() => ({ + apiMock: { + getAuthInfo: vi.fn(), + getRuntimeSettings: vi.fn(), + getDownstreamApiKeys: vi.fn(), + getRoutesLite: vi.fn(), + getRuntimeDatabaseConfig: vi.fn(), + getBrandList: vi.fn(), + updateRuntimeSettings: vi.fn(), + getModelTokenCandidates: vi.fn(), + }, +})); + +vi.mock('../api.js', () => ({ + api: apiMock, +})); + +vi.mock('../components/BrandIcon.js', () => ({ + BrandGlyph: () => null, + InlineBrandIcon: () => null, + getBrand: () => null, + normalizeBrandIconKey: (icon: string) => icon, +})); + +function collectText(node: ReactTestInstance): string { + return (node.children || []).map((child) => { + if (typeof child === 'string') return child; + return collectText(child); + }).join(''); +} + +async function flushMicrotasks() { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +describe('Settings model availability probe confirmation', () => { + beforeEach(() => { + vi.clearAllMocks(); + apiMock.getAuthInfo.mockResolvedValue({ masked: 'sk-****' }); + apiMock.getRuntimeSettings.mockResolvedValue({ + checkinCron: '0 8 * * *', + checkinScheduleMode: 'interval', + checkinIntervalHours: 6, + balanceRefreshCron: '0 * * * *', + logCleanupCron: '15 4 * * *', + logCleanupUsageLogsEnabled: true, + logCleanupProgramLogsEnabled: true, + logCleanupRetentionDays: 14, + modelAvailabilityProbeEnabled: false, + routingFallbackUnitCost: 1, + routingWeights: {}, + adminIpAllowlist: [], + systemProxyUrl: '', + }); + apiMock.getDownstreamApiKeys.mockResolvedValue({ items: [] }); + apiMock.getRoutesLite.mockResolvedValue([]); + apiMock.getBrandList.mockResolvedValue({ brands: [] }); + apiMock.getRuntimeDatabaseConfig.mockResolvedValue({ + active: { dialect: 'sqlite', connection: '(default sqlite path)', ssl: false }, + saved: null, + restartRequired: false, + }); + apiMock.updateRuntimeSettings.mockResolvedValue({ + success: true, + modelAvailabilityProbeEnabled: true, + }); + apiMock.getModelTokenCandidates.mockResolvedValue({ models: {} }); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('requires the exact confirmation text before enabling model availability probing', async () => { + let root!: ReactTestRenderer; + try { + await act(async () => { + root = create( + + + + + , + ); + }); + await flushMicrotasks(); + + const toggleLabel = root.root.find((node) => ( + node.type === 'label' + && collectText(node).includes('允许 metapi 后台主动批量测活') + )); + const toggle = toggleLabel.findByType('input'); + expect(toggle.props.checked).toBe(false); + + await act(async () => { + toggle.props.onChange({ target: { checked: true } }); + }); + + const saveButton = root.root.find((node) => ( + node.type === 'button' + && typeof node.props.onClick === 'function' + && collectText(node).trim() === '保存批量测活设置' + )); + await act(async () => { + saveButton.props.onClick(); + }); + await flushMicrotasks(); + + expect(JSON.stringify(root.toJSON())).toContain(MODEL_AVAILABILITY_PROBE_CONFIRM_TEXT); + expect(apiMock.updateRuntimeSettings).not.toHaveBeenCalled(); + + const confirmButtonBeforeTyping = root.root.find((node) => ( + node.type === 'button' + && collectText(node).trim() === '确认开启批量测活' + && node.props.className === 'btn btn-danger' + )); + expect(confirmButtonBeforeTyping.props.disabled).toBe(true); + + const confirmInput = root.root.find((node) => ( + node.type === 'textarea' + && node.props.placeholder === '请输入上方确认语句' + )); + await act(async () => { + confirmInput.props.onChange({ target: { value: MODEL_AVAILABILITY_PROBE_CONFIRM_TEXT } }); + }); + + const confirmButton = root.root.find((node) => ( + node.type === 'button' + && collectText(node).trim() === '确认开启批量测活' + && node.props.className === 'btn btn-danger' + )); + expect(confirmButton.props.disabled).toBe(false); + + await act(async () => { + confirmButton.props.onClick(); + }); + await flushMicrotasks(); + + expect(apiMock.updateRuntimeSettings).toHaveBeenCalledWith({ + modelAvailabilityProbeEnabled: true, + }); + } finally { + root?.unmount(); + } + }); +}); diff --git a/src/web/pages/settings/ModelAvailabilityProbeConfirmModal.tsx b/src/web/pages/settings/ModelAvailabilityProbeConfirmModal.tsx new file mode 100644 index 0000000..834e785 --- /dev/null +++ b/src/web/pages/settings/ModelAvailabilityProbeConfirmModal.tsx @@ -0,0 +1,105 @@ +import React from 'react'; +import { createPortal } from 'react-dom'; + +type ModalPresence = { + shouldRender: boolean; + isVisible: boolean; +}; + +type ModelAvailabilityProbeConfirmModalProps = { + presence: ModalPresence; + confirmText: string; + confirmationInput: string; + saving: boolean; + onConfirmationInputChange: (value: string) => void; + onClose: () => void; + onConfirm: () => void; +}; + +export default function ModelAvailabilityProbeConfirmModal({ + presence, + confirmText, + confirmationInput, + saving, + onConfirmationInputChange, + onClose, + onConfirm, +}: ModelAvailabilityProbeConfirmModalProps) { + if (!presence.shouldRender) { + return null; + } + + const canConfirm = confirmationInput.trim() === confirmText; + const titleId = 'model-availability-probe-confirm-modal-title'; + const descriptionId = 'model-availability-probe-confirm-modal-description'; + const handleRequestClose = () => { + if (saving) return; + onClose(); + }; + + const modal = ( +
+
e.stopPropagation()} + > +
确认开启批量测活
+
+
+ 开启后，metapi 会在后台对活跃账号模型做最小化探测请求。这可能被部分中转站视为批量测活或异常行为，请务必先确认你的中转站明确允许此类探测。 +
+
+ 请手动输入以下整句后再开启： +
+
+ {confirmText} +
+