fix(upgrade): resolve application upgrade failures (#13888)

agent/app/service/app_upgrade.go

@@ -40,14 +40,13 @@ const (
 	appUpgradeDown
 	appUpgradeMutated
 	appUpgradeStarted
-	appUpgradeReady
 	appUpgradeCommitted
 )
 
-const composeServiceLabel = "com.docker.compose.service"
-
 var appUpgradeLocks sync.Map
 
+const composeServiceLabel = "com.docker.compose.service"
+
 type appUpgradeSnapshot interface {
 	Restore() error
 	Cleanup()
@@ -439,15 +438,14 @@ func (u *appUpgradeContext) cutover(t *task.Task) error {
 	t.LogSuccess(logStr)
 	u.phase = appUpgradeStarted
 
-	t.LogStart(i18n.GetMsgByKey("UpgradeWaitReady"))
-	containerNames, err := waitAppContainersReady(context.Background(), u.candidate)
-	if err != nil {
-		t.LogFailedWithErr(i18n.GetMsgByKey("UpgradeWaitReady"), err)
-		return err
+	containerNames, discoverErr := discoverUpgradeContainerNames(u.candidate, u.envContent)
+	if discoverErr != nil {
+		t.Logf("WARNING: discover upgraded application containers failed: %v", discoverErr)
+	} else if len(containerNames) > 0 {
+		u.candidate.ContainerName = strings.Join(containerNames, ",")
+	} else {
+		t.Log("WARNING: no containers found for the upgraded application")
 	}
-	t.LogSuccess(i18n.GetMsgByKey("UpgradeWaitReady"))
-	u.phase = appUpgradeReady
-	u.candidate.ContainerName = strings.Join(containerNames, ",")
 	u.candidate.Status = constant.StatusRunning
 	u.candidate.Message = ""
 
@@ -472,6 +470,11 @@ func (u *appUpgradeContext) cutover(t *task.Task) error {
 	} else if err = appInstallRepo.Save(context.Background(), &u.candidate); err != nil {
 		return err
 	}
+	if discoverErr == nil && len(containerNames) > 0 {
+		if syncErr := syncAppInstallStatus(&u.candidate, true); syncErr != nil {
+			t.Logf("WARNING: sync upgraded application status failed: %v", syncErr)
+		}
+	}
 	u.phase = appUpgradeCommitted
 	u.deleteOldImages(t)
 	return nil
@@ -591,9 +594,6 @@ func (u *appUpgradeContext) rollback(t *task.Task) (rollbackErr error) {
 }
 
 func (u *appUpgradeContext) finishRollback() error {
-	if _, err := waitAppContainersReady(context.Background(), u.original); err != nil {
-		return err
-	}
 	restored := u.original
 	if err := appInstallRepo.Save(context.Background(), &restored); err != nil {
 		return err
@@ -881,28 +881,7 @@ func (s *upgradeFileSnapshot) Cleanup() {
 	}
 }
 
-type appContainerReadinessClient interface {
-	ContainerList(context.Context, container.ListOptions) ([]container.Summary, error)
-	ContainerInspect(context.Context, string) (container.InspectResponse, error)
-}
-
-func waitAppContainersReady(ctx context.Context, install model.AppInstall) ([]string, error) {
-	client, err := docker.NewDockerClient()
-	if err != nil {
-		return nil, err
-	}
-	defer client.Close()
-	return waitAppContainersReadyWithClient(ctx, client, install)
-}
-
-func waitAppContainersReadyWithClient(ctx context.Context, client appContainerReadinessClient, install model.AppInstall) ([]string, error) {
-	envContent, err := os.ReadFile(install.GetEnvPath())
-	if err != nil {
-		envContent, err = renderUpgradeEnv(&install, nil)
-		if err != nil {
-			return nil, err
-		}
-	}
+func discoverUpgradeContainerNames(install model.AppInstall, envContent []byte) ([]string, error) {
 	project, err := docker.GetComposeProject(install.Name, install.GetPath(), []byte(install.DockerCompose), envContent, false)
 	if err != nil {
 		return nil, err
@@ -916,36 +895,24 @@ func waitAppContainersReadyWithClient(ctx context.Context, client appContainerRe
 	if len(expectedServices) == 0 {
 		return strings.Split(install.ContainerName, ","), nil
 	}
-	options := container.ListOptions{
-		All: true,
-		Filters: filters.NewArgs(
-			filters.Arg("label", composeWorkdirLabel+"="+install.GetPath()),
-		),
+	client, err := docker.NewDockerClient()
+	if err != nil {
+		return nil, err
 	}
-	containers, err := client.ContainerList(ctx, options)
+	defer client.Close()
+	containers, err := client.ContainerList(context.Background(), container.ListOptions{
+		All:     true,
+		Filters: filters.NewArgs(filters.Arg("label", composeWorkdirLabel+"="+install.GetPath())),
+	})
 	if err != nil {
 		return nil, err
 	}
-	foundServices := make(map[string]bool, len(expectedServices))
 	containerNames := make([]string, 0, len(containers))
 	for _, item := range containers {
-		serviceName := item.Labels[composeServiceLabel]
-		if _, ok := expectedServices[serviceName]; !ok {
-			continue
-		}
-		if err = waitContainerReady(ctx, client, item.ID); err != nil {
-			return nil, fmt.Errorf("container %s is not ready: %w", serviceName, err)
-		}
-		foundServices[serviceName] = true
-		if len(item.Names) > 0 {
+		if _, ok := expectedServices[item.Labels[composeServiceLabel]]; ok && len(item.Names) > 0 {
 			containerNames = append(containerNames, strings.TrimPrefix(item.Names[0], "/"))
 		}
 	}
-	for serviceName := range expectedServices {
-		if !foundServices[serviceName] {
-			return nil, fmt.Errorf("container for service %s was not created", serviceName)
-		}
-	}
 	sort.Strings(containerNames)
 	return containerNames, nil
 }

agent/app/service/app_utils.go

@@ -352,22 +352,34 @@ func deleteAppInstall(deleteReq request.AppInstallDelete) error {
 		if dir != nil {
 			logStr := i18n.GetMsgByKey("Stop") + i18n.GetMsgByKey("App")
 			t.Log(logStr)
+			cleanupFailed := false
 
 			if deleteReq.UseLifecycleScripts {
-				if err = runScript(t, &install, "uninstall"); err != nil {
-					return err
+				if scriptErr := runScript(t, &install, "uninstall"); scriptErr != nil {
+					cleanupFailed = true
+					if !deleteReq.ForceDelete {
+						return scriptErr
+					}
 				}
 			} else {
-				out, err := compose.Down(install.GetComposePath())
-				if err != nil && !deleteReq.ForceDelete {
-					return handleErr(install, err, out)
+				out, downErr := compose.Down(install.GetComposePath())
+				if downErr != nil {
+					cleanupFailed = true
+					if !deleteReq.ForceDelete {
+						return handleErr(install, downErr, out)
+					}
 				}
-				if err = runScript(t, &install, "uninstall"); err != nil {
-					_, _ = compose.Up(install.GetComposePath())
-					return err
+				if scriptErr := runScript(t, &install, "uninstall"); scriptErr != nil {
+					cleanupFailed = true
+					if !deleteReq.ForceDelete {
+						_, _ = compose.Up(install.GetComposePath())
+						return scriptErr
+					}
 				}
 			}
-			t.LogSuccess(logStr)
+			if !cleanupFailed {
+				t.LogSuccess(logStr)
+			}
 			if deleteReq.DeleteImage {
 				content, err := op.GetContent(install.GetEnvPath())
 				if err != nil {
@@ -466,8 +478,9 @@ func deleteAppInstall(deleteReq request.AppInstallDelete) error {
 	}
 	uninstallTask.AddSubTask(task.GetTaskName(install.Name, task.TaskUninstall, task.TaskScopeApp), uninstall, nil)
 	go func() {
-		if err := uninstallTask.Execute(); err != nil && !deleteReq.ForceDelete {
+		if err := uninstallTask.Execute(); err != nil {
 			install.Status = constant.StatusError
+			install.Message = err.Error()
 			_ = appInstallRepo.Save(context.Background(), &install)
 		}
 	}()