From 3ebe851002c12192a54f797687db8f91c7f2339b Mon Sep 17 00:00:00 2001 From: Cita Date: Wed, 1 Apr 2026 12:56:42 +0800 Subject: [PATCH] Fix update helper health for ready failed release (#361) --- src/server/update-helper/service.test.ts | 74 ++++++++++++++++++++++++ src/server/update-helper/service.ts | 50 +++++++++++++--- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/server/update-helper/service.test.ts b/src/server/update-helper/service.test.ts index 237628a..2beb1b9 100644 --- a/src/server/update-helper/service.test.ts +++ b/src/server/update-helper/service.test.ts @@ -351,4 +351,78 @@ describe('update helper service', () => { ], }); }); + + it('treats ready workloads as healthy even when the latest helm revision is marked failed', async () => { + const status = await getUpdateHelperStatus( + { + namespace: 'ai', + releaseName: 'metapi', + }, + { + runCommand: async ({ command, args }) => { + if (command === 'helm' && args[0] === 'status') { + return { + stdout: JSON.stringify({ + version: 22, + info: { + status: 'failed', + description: 'Upgrade "metapi" failed: context deadline exceeded', + }, + }), + }; + } + if (command === 'helm' && args[0] === 'get') { + return { + stdout: JSON.stringify({ + image: { + repository: '1467078763/metapi', + tag: 'latest', + digest: 'sha256:1523dfcbaca146b8a64a3a022878165027900ee4cb396a45e474aa7a969dacb1', + }, + }), + }; + } + if (command === 'helm' && args[0] === 'history') { + return { + stdout: JSON.stringify([ + { revision: '21', updated: '2026-03-31T14:07:40.115551823Z', status: 'deployed', description: 'Rollback to 17' }, + { revision: '22', updated: '2026-03-31T14:13:24.726024175Z', status: 'failed', description: 'Upgrade "metapi" failed: context deadline exceeded' }, + ]), + }; + } + if (command === 'kubectl' && args[0] === 'get') { + return { + stdout: JSON.stringify({ + items: [ + { + status: { + phase: 'Running', + containerStatuses: [ + { + ready: true, + imageID: 'docker-pullable://1467078763/metapi@sha256:1523dfcbaca146b8a64a3a022878165027900ee4cb396a45e474aa7a969dacb1', + }, + ], + }, + }, + ], + }), + }; + } + throw new Error('unexpected command'); + }, + }, + ); + + expect(status).toMatchObject({ + ok: true, + releaseName: 'metapi', + namespace: 'ai', + revision: '22', + imageRepository: '1467078763/metapi', + imageTag: 'latest', + imageDigest: 'sha256:1523dfcbaca146b8a64a3a022878165027900ee4cb396a45e474aa7a969dacb1', + healthy: true, + }); + }); }); diff --git a/src/server/update-helper/service.ts b/src/server/update-helper/service.ts index 2f3787e..5b604ed 100644 --- a/src/server/update-helper/service.ts +++ b/src/server/update-helper/service.ts @@ -82,6 +82,11 @@ type ParsedReleaseValues = { imageDigest: string | null; }; +type ParsedRuntimePodState = { + imageDigest: string | null; + allPodsReady: boolean; +}; + async function runCommand(input: RunCommandInput): Promise { return await new Promise((resolve, reject) => { const child = spawn(input.command, input.args, { @@ -172,12 +177,14 @@ function parseReleaseValues(stdout: string): ParsedReleaseValues { } } -function parseRuntimeImageDigest(stdout: string): string | null { +function parseRuntimePodState(stdout: string): ParsedRuntimePodState { try { const payload = JSON.parse(stdout || '{}') as { items?: Array<{ status?: { + phase?: string | null; containerStatuses?: Array<{ + ready?: boolean | null; imageID?: string | null; image?: string | null; }>; @@ -185,18 +192,42 @@ function parseRuntimeImageDigest(stdout: string): string | null { }>; }; + let imageDigest: string | null = null; + const items = payload.items || []; + if (items.length <= 0) { + return { + imageDigest: null, + allPodsReady: false, + }; + } + + let allPodsReady = true; + for (const item of payload.items || []) { - for (const status of item.status?.containerStatuses || []) { + const phase = String(item.status?.phase || '').trim().toLowerCase(); + const containerStatuses = item.status?.containerStatuses || []; + if (phase !== 'running' || containerStatuses.length <= 0 || containerStatuses.some((status) => status.ready !== true)) { + allPodsReady = false; + } + + for (const status of containerStatuses) { const candidate = `${status.imageID || ''} ${status.image || ''}`; const match = candidate.match(/(sha256:[a-f0-9]{64})/i); - if (match?.[1]) { - return match[1].toLowerCase(); + if (!imageDigest && match?.[1]) { + imageDigest = match[1].toLowerCase(); } } } - return null; + + return { + imageDigest, + allPodsReady, + }; } catch { - return null; + return { + imageDigest: null, + allPodsReady: false, + }; } } @@ -388,8 +419,9 @@ export async function getUpdateHelperStatus( }; }; const currentValues = parseReleaseValues(values.stdout); - const runtimeDigest = parseRuntimeImageDigest(pods.stdout); + const runtimeState = parseRuntimePodState(pods.stdout); const historyRows = parseHistoryRows(history.stdout).slice(0, 5); + const releaseStatus = String(statusPayload.info?.status || '').trim().toLowerCase(); const historyEntries: UpdateHelperHistoryEntry[] = []; for (const row of historyRows) { @@ -426,8 +458,8 @@ export async function getUpdateHelperStatus( revision: statusPayload.version === undefined || statusPayload.version === null ? null : String(statusPayload.version), imageRepository: currentValues.imageRepository, imageTag: currentValues.imageTag, - imageDigest: runtimeDigest || currentValues.imageDigest, - healthy: String(statusPayload.info?.status || '').toLowerCase() === 'deployed', + imageDigest: runtimeState.imageDigest || currentValues.imageDigest, + healthy: releaseStatus === 'deployed' || (releaseStatus === 'failed' && runtimeState.allPodsReady), history: historyEntries, }; }