fix(applaunchpad): preserve network resources on storage update (#7165)

* fix(applaunchpad): preserve network resources on storage update

* fix(applaunchpad): rebind recreated statefulset dependents

* fix(applaunchpad): remove unsafe statefulset recreation fallback
This commit is contained in:
Alex Lee
2026-08-10 14:44:18 +08:00
committed by GitHub
parent aa6ca08e05
commit 5c918522b6
2 changed files with 693 additions and 54 deletions
@@ -0,0 +1,340 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import handler from '@/pages/api/updateApp';
const initK8sMock = vi.hoisted(() => vi.fn());
vi.mock('sealos-desktop-sdk/service', () => ({
initK8s: initK8sMock
}));
vi.mock('sealos-desktop-sdk', () => ({
errLog: vi.fn(),
infoLog: vi.fn(),
warnLog: vi.fn()
}));
function notFound() {
return Promise.reject({
body: {
code: 404
}
});
}
const statefulSetYaml = `
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: demo
spec:
selector:
matchLabels:
app: demo
serviceName: demo-service
template:
metadata:
labels:
app: demo
spec:
containers:
- name: demo
image: nginx
volumeMounts:
- name: data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: data
annotations:
path: /data
value: "1"
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
`;
function createRequest() {
return {
body: {
appName: 'demo',
stateFulSetYaml: statefulSetYaml,
patch: [
{
type: 'create',
kind: 'StatefulSet',
value: statefulSetYaml
},
{
type: 'delete',
kind: 'Deployment',
name: 'demo'
}
]
}
} as any;
}
function createResponse() {
return {
json: vi.fn((payload) => payload)
} as any;
}
function createK8sContext() {
return {
namespace: 'ns-demo',
applyYamlList: vi.fn(() => Promise.resolve([{ kind: 'StatefulSet' }])),
k8sApp: {
readNamespacedDeployment: vi.fn(() =>
Promise.resolve({
body: {
metadata: {
uid: 'old-deployment-uid'
}
}
})
),
readNamespacedStatefulSet: vi.fn(() =>
Promise.resolve({
body: {
metadata: {
uid: 'new-statefulset-uid'
}
}
})
),
patchNamespacedDeployment: vi.fn(() => Promise.resolve({})),
patchNamespacedStatefulSet: vi.fn(() => Promise.resolve({})),
replaceNamespacedStatefulSet: vi.fn(() => Promise.resolve({})),
createNamespacedStatefulSet: vi.fn(() => Promise.resolve({})),
deleteNamespacedDeployment: vi.fn(() => Promise.resolve({})),
deleteNamespacedStatefulSet: vi.fn(() => Promise.resolve({}))
},
k8sCore: {
listNamespacedPersistentVolumeClaim: vi.fn(() =>
Promise.resolve({
body: {
items: [
{
metadata: {
name: 'demo-data-0',
annotations: {
path: '/data',
value: '1'
}
},
spec: {
resources: {
requests: {
storage: '1Gi'
}
}
}
}
]
}
})
),
listNamespacedService: vi.fn(() =>
Promise.resolve({
body: {
items: [
{
metadata: {
name: 'demo-service'
}
}
]
}
})
),
patchNamespacedService: vi.fn(() => Promise.resolve({})),
patchNamespacedPersistentVolumeClaim: vi.fn(() => Promise.resolve({})),
patchNamespacedConfigMap: vi.fn(() => notFound()),
patchNamespacedSecret: vi.fn(() => notFound()),
replaceNamespacedService: vi.fn(() => Promise.resolve({})),
deleteNamespacedService: vi.fn(() => Promise.resolve({})),
replaceNamespacedConfigMap: vi.fn(() => Promise.resolve({})),
deleteNamespacedConfigMap: vi.fn(() => Promise.resolve({})),
deleteNamespacedSecret: vi.fn(() => Promise.resolve({})),
deleteNamespacedPersistentVolumeClaim: vi.fn(() => Promise.resolve({}))
},
k8sNetworkingApp: {
listNamespacedIngress: vi.fn(() =>
Promise.resolve({
body: {
items: [
{
metadata: {
name: 'demo-ingress'
}
}
]
}
})
),
patchNamespacedIngress: vi.fn(() => Promise.resolve({})),
deleteNamespacedIngress: vi.fn(() => Promise.resolve({}))
},
k8sAutoscaling: {
patchNamespacedHorizontalPodAutoscaler: vi.fn(() => notFound()),
deleteNamespacedHorizontalPodAutoscaler: vi.fn(() => Promise.resolve({}))
},
k8sCustomObjects: {
listNamespacedCustomObject: vi.fn(() =>
Promise.resolve({
body: {
items: []
}
})
),
patchNamespacedCustomObject: vi.fn(() => Promise.resolve({})),
getNamespacedCustomObject: vi.fn(() =>
Promise.reject({
body: {
code: 404,
message: 'not found'
}
})
),
deleteNamespacedCustomObject: vi.fn(() => Promise.resolve({}))
}
};
}
describe('/api/updateApp', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('moves existing network resources to the new StatefulSet before deleting the old Deployment', async () => {
const k8s = createK8sContext();
initK8sMock.mockResolvedValue(k8s);
const res = createResponse();
await handler(createRequest(), res);
const ownerReferences = [
{
apiVersion: 'apps/v1',
kind: 'StatefulSet',
name: 'demo',
uid: 'new-statefulset-uid',
controller: true,
blockOwnerDeletion: true
}
];
const ownerReferencePatch = {
metadata: {
ownerReferences
}
};
expect(k8s.k8sApp.readNamespacedStatefulSet).toHaveBeenCalledWith('demo', 'ns-demo');
expect(k8s.k8sCore.patchNamespacedService).toHaveBeenCalledWith(
'demo-service',
'ns-demo',
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
expect.objectContaining({
headers: {
'Content-type': 'application/merge-patch+json'
}
})
);
expect(k8s.k8sCore.patchNamespacedPersistentVolumeClaim).toHaveBeenCalledWith(
'demo-data-0',
'ns-demo',
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
expect.objectContaining({
headers: {
'Content-type': 'application/merge-patch+json'
}
})
);
expect(k8s.k8sNetworkingApp.patchNamespacedIngress).toHaveBeenCalledWith(
'demo-ingress',
'ns-demo',
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
expect.objectContaining({
headers: {
'Content-type': 'application/merge-patch+json'
}
})
);
expect(k8s.k8sCore.patchNamespacedService.mock.invocationCallOrder[0]).toBeLessThan(
k8s.k8sApp.deleteNamespacedDeployment.mock.invocationCallOrder[0]
);
expect(k8s.k8sNetworkingApp.patchNamespacedIngress.mock.invocationCallOrder[0]).toBeLessThan(
k8s.k8sApp.deleteNamespacedDeployment.mock.invocationCallOrder[0]
);
expect(k8s.k8sCore.patchNamespacedPersistentVolumeClaim.mock.invocationCallOrder[0]).toBeLessThan(
k8s.k8sApp.deleteNamespacedDeployment.mock.invocationCallOrder[0]
);
expect(res.json).toHaveBeenCalledWith({
code: 200,
message: 'Success',
data: undefined,
error: undefined
});
});
it('returns an error instead of deleting and recreating a StatefulSet when patching fails', async () => {
const k8s = createK8sContext();
k8s.k8sApp.patchNamespacedStatefulSet.mockRejectedValueOnce(new Error('patch failed'));
k8s.k8sApp.replaceNamespacedStatefulSet.mockRejectedValueOnce(new Error('replace failed'));
initK8sMock.mockResolvedValue(k8s);
const res = createResponse();
await handler(
{
body: {
appName: 'demo',
stateFulSetYaml: statefulSetYaml,
patch: [
{
type: 'patch',
kind: 'StatefulSet',
value: {
kind: 'StatefulSet',
metadata: {
name: 'demo'
},
spec: {}
}
}
]
}
} as any,
res
);
expect(k8s.k8sApp.deleteNamespacedStatefulSet).not.toHaveBeenCalled();
expect(k8s.k8sApp.createNamespacedStatefulSet).not.toHaveBeenCalled();
expect(k8s.k8sCore.patchNamespacedPersistentVolumeClaim).not.toHaveBeenCalled();
expect(res.json).toHaveBeenCalledWith({
code: 500,
message: 'replace failed',
data: undefined,
error: undefined
});
});
});
@@ -10,6 +10,7 @@ import { initK8s } from 'sealos-desktop-sdk/service';
import { errLog, infoLog, warnLog } from 'sealos-desktop-sdk';
import type { V1Service } from '@kubernetes/client-node';
import { generateOwnerReference, shouldHaveOwnerReference } from '@/utils/deployYaml2Json';
import { appDeployKey } from '@/constants/app';
import { buildExternalUrl } from '@/utils/network-url';
import { ResponseCode } from '@/types/response';
@@ -62,6 +63,270 @@ const normalizeNetworkResource = <T extends Record<string, any>>(resource: T): T
const isWorkloadKind = (kind?: string) =>
kind === YamlKindEnum.Deployment || kind === YamlKindEnum.StatefulSet;
type CreatePatchItem = Extract<AppPatchPropsType[number], { type: 'create' }>;
type CreateResourceItem = {
item: CreatePatchItem;
resource: Record<string, any>;
};
const getK8sErrorCode = (error: any) =>
error?.body?.code || error?.response?.body?.code || error?.response?.statusCode;
const ignoreNotFound = async <T>(promise: Promise<T>) => {
try {
return await promise;
} catch (error: any) {
if (Number(getK8sErrorCode(error)) === 404) {
return undefined;
}
throw error;
}
};
async function getWorkloadOwnerReferences({
k8sApp,
namespace,
appName,
kind
}: {
k8sApp: {
readNamespacedDeployment: (name: string, namespace: string) => Promise<any>;
readNamespacedStatefulSet: (name: string, namespace: string) => Promise<any>;
};
namespace: string;
appName: string;
kind?: 'Deployment' | 'StatefulSet';
}) {
const readWorkload = async (targetKind: 'Deployment' | 'StatefulSet') => {
const workload =
targetKind === 'Deployment'
? await k8sApp.readNamespacedDeployment(appName, namespace)
: await k8sApp.readNamespacedStatefulSet(appName, namespace);
const uid = workload.body.metadata?.uid;
if (!uid) {
throw new Error(`${targetKind} UID is empty`);
}
return generateOwnerReference(appName, targetKind, uid);
};
if (kind) {
return readWorkload(kind);
}
try {
return await readWorkload('Deployment');
} catch (error: any) {
if (Number(getK8sErrorCode(error)) !== 404) {
throw error;
}
return readWorkload('StatefulSet');
}
}
const withOwnerReferences = (
yamlStr: string,
ownerReferences: ReturnType<typeof generateOwnerReference>
) => {
const resource = normalizeNetworkResource(yaml.load(yamlStr) as any);
if (resource?.kind && shouldHaveOwnerReference(resource.kind)) {
resource.metadata = resource.metadata || {};
resource.metadata.ownerReferences = ownerReferences;
infoLog('Added ownerReferences to new resource', {
kind: resource.kind,
name: resource.metadata.name
});
}
return yaml.dump(resource);
};
async function patchExistingOwnerReferences({
k8sCore,
k8sNetworkingApp,
k8sAutoscaling,
k8sCustomObjects,
namespace,
appName,
ownerReferences
}: {
k8sCore: any;
k8sNetworkingApp: any;
k8sAutoscaling: any;
k8sCustomObjects: CustomObjectsApi;
namespace: string;
appName: string;
ownerReferences: ReturnType<typeof generateOwnerReference>;
}) {
const mergePatchOptions = {
headers: { 'Content-type': PatchUtils.PATCH_FORMAT_JSON_MERGE_PATCH }
};
const ownerReferencePatch = {
metadata: {
ownerReferences
}
};
const labelSelector = `${appDeployKey}=${appName}`;
const patchServices = k8sCore
.listNamespacedService(namespace, undefined, undefined, undefined, undefined, labelSelector)
.then((res: { body: { items: V1Service[] } }) =>
Promise.all(
res.body.items.map((service) =>
service.metadata?.name
? k8sCore.patchNamespacedService(
service.metadata.name,
namespace,
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
mergePatchOptions
)
: undefined
)
)
);
const patchIngresses = k8sNetworkingApp
.listNamespacedIngress(namespace, undefined, undefined, undefined, undefined, labelSelector)
.then((res: { body: { items: Array<{ metadata?: { name?: string } }> } }) =>
Promise.all(
res.body.items.map((ingress) =>
ingress.metadata?.name
? k8sNetworkingApp.patchNamespacedIngress(
ingress.metadata.name,
namespace,
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
mergePatchOptions
)
: undefined
)
)
);
const patchPvcs = k8sCore
.listNamespacedPersistentVolumeClaim(
namespace,
undefined,
undefined,
undefined,
undefined,
`app=${appName}`
)
.then((res: { body: { items: Array<{ metadata?: { name?: string } }> } }) =>
Promise.all(
res.body.items.map((pvc) =>
pvc.metadata?.name
? k8sCore.patchNamespacedPersistentVolumeClaim(
pvc.metadata.name,
namespace,
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
mergePatchOptions
)
: undefined
)
)
);
const patchAppNamedResources = Promise.all([
ignoreNotFound(
k8sCore.patchNamespacedConfigMap(
appName,
namespace,
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
mergePatchOptions
)
),
ignoreNotFound(
k8sCore.patchNamespacedSecret(
appName,
namespace,
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
mergePatchOptions
)
),
ignoreNotFound(
k8sAutoscaling.patchNamespacedHorizontalPodAutoscaler(
appName,
namespace,
ownerReferencePatch,
undefined,
undefined,
undefined,
undefined,
undefined,
mergePatchOptions
)
)
]);
const patchCustomObjects = async (plural: 'issuers' | 'certificates') => {
const response = await ignoreNotFound(
k8sCustomObjects.listNamespacedCustomObject(
'cert-manager.io',
'v1',
namespace,
plural,
undefined,
undefined,
undefined,
undefined,
labelSelector
)
);
const items = ((response as any)?.body?.items || []) as Array<{ metadata?: { name?: string } }>;
await Promise.all(
items.map((item) =>
item.metadata?.name
? k8sCustomObjects.patchNamespacedCustomObject(
'cert-manager.io',
'v1',
namespace,
plural,
item.metadata.name,
ownerReferencePatch,
undefined,
undefined,
undefined,
mergePatchOptions
)
: undefined
)
);
};
await Promise.all([
patchServices,
patchIngresses,
patchPvcs,
patchAppNamedResources,
patchCustomObjects('issuers'),
patchCustomObjects('certificates')
]);
}
async function updateAppCRUrl(
k8sCustomObjects: CustomObjectsApi,
namespace: string,
@@ -186,7 +451,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
},
[YamlKindEnum.StatefulSet]: {
patch: async (jsonPatch: Object) => {
// patch -> replace -> delete and create
// patch -> replace; fail closed if both fail
try {
await k8sApp.patchNamespacedStatefulSet(
appName,
@@ -199,13 +464,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
undefined,
{ headers: { 'Content-type': PatchUtils.PATCH_FORMAT_JSON_MERGE_PATCH } }
);
} catch (error) {
return { recreated: false, kind: YamlKindEnum.StatefulSet };
} catch (patchError) {
try {
await k8sApp.replaceNamespacedStatefulSet(appName, namespace, jsonPatch);
} catch (error) {
warnLog('delete and create statefulSet', { yaml: yaml.dump(jsonPatch) });
await k8sApp.deleteNamespacedStatefulSet(appName, namespace);
await k8sApp.createNamespacedStatefulSet(namespace, jsonPatch);
return { recreated: false, kind: YamlKindEnum.StatefulSet };
} catch (replaceError) {
warnLog('statefulSet patch/replace failed; not falling back to delete/create', {
yaml: yaml.dump(jsonPatch)
});
throw replaceError || patchError;
}
}
},
@@ -420,70 +688,101 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await Promise.all(regularPatches.map(applyPatchItem));
// create
const createYamlList = patch
const createItems = patch
.map((item) => {
const cr = crMap[item.kind];
if (!cr || item.type !== 'create') {
return;
}
return item.value;
const resource = normalizeNetworkResource(yaml.load(item.value as string) as any);
return {
item,
resource
};
})
.filter((item) => item);
.filter((item): item is CreateResourceItem => !!item?.resource);
// Add ownerReferences to newly created resources
if (createYamlList.length > 0) {
// Get workload UID
let workloadUid: string | undefined;
let workloadKind: 'Deployment' | 'StatefulSet' | undefined;
const workloadCreateItems = createItems.filter(
({ resource }) => isWorkloadKind(resource.kind) && resource.metadata?.name === appName
);
const dependentCreateItems = createItems.filter(
({ resource }) => !isWorkloadKind(resource.kind) || resource.metadata?.name !== appName
);
const replacingWorkload =
workloadCreateItems.length > 0 &&
patch.some(
(item) => item.type === 'delete' && isWorkloadKind(item.kind) && item.name === appName
);
let createdWorkloadOwnerReferences: ReturnType<typeof generateOwnerReference> | undefined;
try {
// Try to read Deployment first
const deployment = await k8sApp.readNamespacedDeployment(appName, namespace);
workloadUid = deployment.body.metadata?.uid;
workloadKind = 'Deployment';
} catch (err: any) {
if (err?.body?.code === 404) {
// Try StatefulSet
try {
const statefulSet = await k8sApp.readNamespacedStatefulSet(appName, namespace);
workloadUid = statefulSet.body.metadata?.uid;
workloadKind = 'StatefulSet';
} catch (err2) {
warnLog('Could not find workload for ownerReferences', { appName });
}
}
}
if (workloadCreateItems.length > 0) {
await applyYamlList(
workloadCreateItems.map(({ resource }) => yaml.dump(resource)),
'create'
);
// Add ownerReferences to new resources
if (workloadUid && workloadKind) {
const ownerReferences = generateOwnerReference(appName, workloadKind, workloadUid);
const updatedCreateYamlList = createYamlList.map((yamlStr) => {
const resource = yaml.load(yamlStr as string) as any;
normalizeNetworkResource(resource);
if (shouldHaveOwnerReference(resource.kind)) {
if (!resource.metadata) {
resource.metadata = {};
}
resource.metadata.ownerReferences = ownerReferences;
infoLog('Added ownerReferences to new resource', {
kind: resource.kind,
name: resource.metadata.name
});
}
return yaml.dump(resource);
});
await applyYamlList(updatedCreateYamlList, 'create');
} else {
const createdWorkloadKind = workloadCreateItems[0].resource.kind as
'Deployment' | 'StatefulSet';
createdWorkloadOwnerReferences = await getWorkloadOwnerReferences({
k8sApp,
namespace,
appName,
kind: createdWorkloadKind
});
if (dependentCreateItems.length > 0) {
await applyYamlList(
createYamlList.map((yamlStr) =>
yaml.dump(normalizeNetworkResource(yaml.load(yamlStr as string) as any))
dependentCreateItems.map(({ resource }) =>
withOwnerReferences(yaml.dump(resource), createdWorkloadOwnerReferences!)
),
'create'
);
}
} else if (dependentCreateItems.length > 0) {
try {
const ownerReferences = await getWorkloadOwnerReferences({
k8sApp,
namespace,
appName
});
await applyYamlList(
dependentCreateItems.map(({ resource }) =>
withOwnerReferences(yaml.dump(resource), ownerReferences)
),
'create'
);
} catch (error) {
warnLog('Could not find workload for ownerReferences', { appName });
await applyYamlList(
dependentCreateItems.map(({ resource }) => yaml.dump(resource)),
'create'
);
}
}
await Promise.all(workloadPatches.map(applyPatchItem));
const workloadPatchResults = await Promise.all(workloadPatches.map(applyPatchItem));
const recreatedWorkloadPatch = workloadPatchResults.find((result) => result?.recreated);
if (recreatedWorkloadPatch) {
createdWorkloadOwnerReferences = await getWorkloadOwnerReferences({
k8sApp,
namespace,
appName,
kind: recreatedWorkloadPatch.kind
});
}
if (createdWorkloadOwnerReferences && (replacingWorkload || recreatedWorkloadPatch)) {
await patchExistingOwnerReferences({
k8sCore,
k8sNetworkingApp,
k8sAutoscaling,
k8sCustomObjects,
namespace,
appName,
ownerReferences: createdWorkloadOwnerReferences
});
}
// delete
await Promise.all(