From e1b4452c829f325c48d97aa7b5eb89bf0b306f2e Mon Sep 17 00:00:00 2001 From: cita Date: Tue, 10 Mar 2026 16:02:45 +0800 Subject: [PATCH] feat: classify model refresh failures and update health --- .../services/modelService.discovery.test.ts | 2 + src/server/services/modelService.ts | 180 +++++++++++------- 2 files changed, 117 insertions(+), 65 deletions(-) diff --git a/src/server/services/modelService.discovery.test.ts b/src/server/services/modelService.discovery.test.ts index f3934f6..8fd5350 100644 --- a/src/server/services/modelService.discovery.test.ts +++ b/src/server/services/modelService.discovery.test.ts @@ -79,7 +79,9 @@ describe('refreshModelsForAccount credential discovery', () => { expect(result).toMatchObject({ accountId: account.id, refreshed: true, + status: 'success', modelCount: 2, + modelsPreview: ['claude-sonnet-4-5-20250929', 'claude-opus-4-6'], tokenScanned: 0, discoveredByCredential: true, }); diff --git a/src/server/services/modelService.ts b/src/server/services/modelService.ts index 1c25190..5f6a16a 100644 --- a/src/server/services/modelService.ts +++ b/src/server/services/modelService.ts @@ -7,10 +7,28 @@ import { invalidateTokenRouterCache } from './tokenRouter.js'; import { setAccountRuntimeHealth } from './accountHealthService.js'; import { clearAllRouteDecisionSnapshots } from './routeDecisionSnapshotStore.js'; -const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000; -const MODEL_DISCOVERY_TIMEOUT_MS = 12_000; -const MODEL_REFRESH_BATCH_SIZE = 3; - +const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000; +const MODEL_DISCOVERY_TIMEOUT_MS = 12_000; +const MODEL_REFRESH_BATCH_SIZE = 3; + +type ModelRefreshErrorCode = 'timeout' | 'unauthorized' | 'empty_models' | 'unknown'; + +function classifyModelDiscoveryError(message: string): ModelRefreshErrorCode { + const lowered = message.toLowerCase(); + if (lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('请求超时')) return 'timeout'; + if (lowered.includes('http 401') || lowered.includes('http 403') + || lowered.includes('unauthorized') || lowered.includes('invalid') + || lowered.includes('无权') || lowered.includes('未提供令牌')) return 'unauthorized'; + return 'unknown'; +} + +function buildModelFailureMessage(code: ModelRefreshErrorCode, fallback?: string) { + if (code === 'timeout') return '模型获取失败（请求超时）'; + if (code === 'unauthorized') return '模型获取失败，API Key 已无效'; + if (code === 'empty_models') return '模型获取失败：未获取到可用模型'; + return fallback || '模型获取失败'; +} + function isSiteDisabled(status?: string | null): boolean { return (status || 'active') === 'disabled'; } @@ -120,11 +138,16 @@ export async function refreshModelsForAccount(accountId: number) { } } - const accountModels = new Set(); - const modelLatency = new Map(); - let scannedTokenCount = 0; - let discoveredByCredential = false; - const attemptedCredentials = new Set(); + const accountModels = new Set(); + const modelLatency = new Map(); + let scannedTokenCount = 0; + let discoveredByCredential = false; + const attemptedCredentials = new Set(); + const failureMessages: string[] = []; + const recordFailure = (err: unknown) => { + const message = (err as { message?: string })?.message || String(err || ''); + if (message) failureMessages.push(message); + }; const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => { for (const modelName of models) { @@ -146,19 +169,20 @@ export async function refreshModelsForAccount(accountId: number) { attemptedCredentials.add(credential); const startedAt = Date.now(); - let models: string[] = []; - try { - models = normalizeModels( - await withTimeout( - () => adapter.getModels(site.url, credential, platformUserId), - MODEL_DISCOVERY_TIMEOUT_MS, - `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, - ), - ); - } catch { - models = []; - } - if (models.length === 0) return; + let models: string[] = []; + try { + models = normalizeModels( + await withTimeout( + () => adapter.getModels(site.url, credential, platformUserId), + MODEL_DISCOVERY_TIMEOUT_MS, + `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, + ), + ); + } catch (err) { + recordFailure(err); + models = []; + } + if (models.length === 0) return; discoveredByCredential = true; const latencyMs = Date.now() - startedAt; mergeDiscoveredModels(models, latencyMs); @@ -171,19 +195,20 @@ export async function refreshModelsForAccount(accountId: number) { for (const token of enabledTokens) { const startedAt = Date.now(); - let models: string[] = []; - - try { - models = normalizeModels( - await withTimeout( - () => adapter.getModels(site.url, token.token, platformUserId), - MODEL_DISCOVERY_TIMEOUT_MS, - `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, - ), - ); - } catch { - models = []; - } + let models: string[] = []; + + try { + models = normalizeModels( + await withTimeout( + () => adapter.getModels(site.url, token.token, platformUserId), + MODEL_DISCOVERY_TIMEOUT_MS, + `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, + ), + ); + } catch (err) { + recordFailure(err); + models = []; + } if (models.length === 0) continue; @@ -204,37 +229,62 @@ export async function refreshModelsForAccount(accountId: number) { mergeDiscoveredModels(models, latencyMs); } - if (accountModels.size > 0) { - const checkedAt = new Date().toISOString(); - await db.insert(schema.modelAvailability).values( - Array.from(accountModels).map((modelName) => ({ - accountId: account.id, - modelName, - available: true, - latencyMs: modelLatency.get(modelName) ?? null, - checkedAt, - })), - ).run(); - - if (isApiKeyConnection(account)) { - await setAccountRuntimeHealth(account.id, { - state: 'healthy', - reason: '模型探测成功', - source: 'model-discovery', - checkedAt, - }); - } + if (accountModels.size === 0) { + const firstMessage = failureMessages[0] || ''; + const errorCode = firstMessage ? classifyModelDiscoveryError(firstMessage) : 'empty_models'; + const errorMessage = buildModelFailureMessage(errorCode, firstMessage); + await setAccountRuntimeHealth(account.id, { + state: 'unhealthy', + reason: errorMessage, + source: 'model-discovery', + checkedAt: new Date().toISOString(), + }); + return { + accountId, + refreshed: true, + status: 'failed', + errorCode, + errorMessage, + modelCount: 0, + modelsPreview: [], + tokenScanned: scannedTokenCount, + discoveredByCredential, + discoveredApiToken: !!discoveredApiToken, + }; } - - return { - accountId, - refreshed: true, - modelCount: accountModels.size, - tokenScanned: scannedTokenCount, - discoveredByCredential, - discoveredApiToken: !!discoveredApiToken, - }; -} + + const checkedAt = new Date().toISOString(); + await db.insert(schema.modelAvailability).values( + Array.from(accountModels).map((modelName) => ({ + accountId: account.id, + modelName, + available: true, + latencyMs: modelLatency.get(modelName) ?? null, + checkedAt, + })), + ).run(); + + if (isApiKeyConnection(account)) { + await setAccountRuntimeHealth(account.id, { + state: 'healthy', + reason: '模型探测成功', + source: 'model-discovery', + checkedAt, + }); + } + + const modelsPreview = Array.from(accountModels).slice(0, 10); + return { + accountId, + refreshed: true, + status: 'success', + modelCount: accountModels.size, + modelsPreview, + tokenScanned: scannedTokenCount, + discoveredByCredential, + discoveredApiToken: !!discoveredApiToken, + }; +} async function refreshModelsForAllActiveAccounts() { const accounts = await db.select({ id: schema.accounts.id }).from(schema.accounts)