From 56efaed760e134060bf34ec3f91cbe94ceaba043 Mon Sep 17 00:00:00 2001
From: Zexi Li <zexi.li@icloud.com>
Date: Thu, 26 Dec 2024 14:14:57 +0800
Subject: [PATCH] fix(host): SetDirtyContainer when updating container status
 error (#21901)

---
 .../container/prober/prober_manager.go        | 13 +++++++++----
 .../container/status/status_manager.go        | 19 ++++++++++++++++---
 pkg/hostman/guestman/pod.go                   |  4 ++--
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/pkg/hostman/container/prober/prober_manager.go b/pkg/hostman/container/prober/prober_manager.go
index 344c4dce20..556bc32f39 100644
--- a/pkg/hostman/container/prober/prober_manager.go
+++ b/pkg/hostman/container/prober/prober_manager.go
@@ -31,6 +31,7 @@ limitations under the License.
 package prober
 
 import (
+	"fmt"
 	"sync"
 
 	"yunion.io/x/log"
@@ -85,7 +86,7 @@ type Manager interface {
 	// Start starts the Manager sync loops.
 	Start()
 
-	SetDirtyContainer(ctrId string)
+	SetDirtyContainer(ctrId string, reason string)
 }
 
 type manager struct {
@@ -128,7 +129,8 @@ func NewManager(
 	}
 }
 
-func (m *manager) SetDirtyContainer(ctrId string) {
+func (m *manager) SetDirtyContainer(ctrId string, reason string) {
+	log.Infof("[set dirty container] %s: %s", ctrId, reason)
 	m.dirtyContainers.Store(ctrId, true)
 }
 
@@ -235,11 +237,14 @@ func (m *manager) updateStartup() {
 	update := <-m.startupManager.Updates()
 
 	started := update.Result.Result == results.Success
-	m.statusManager.SetContainerStartup(
+	if err := m.statusManager.SetContainerStartup(
 		update.PodUID,
 		update.ContainerID,
 		started,
 		update.Result,
 		update.Pod,
-	)
+	); err != nil {
+		reason := fmt.Sprintf("set container %s/%s startup error: %v", update.PodUID, update.ContainerID, err)
+		m.SetDirtyContainer(update.ContainerID, reason)
+	}
 }
diff --git a/pkg/hostman/container/status/status_manager.go b/pkg/hostman/container/status/status_manager.go
index 3c9e6b2149..886bb7dd9b 100644
--- a/pkg/hostman/container/status/status_manager.go
+++ b/pkg/hostman/container/status/status_manager.go
@@ -16,9 +16,11 @@ package status
 
 import (
 	"context"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
@@ -29,7 +31,7 @@ import (
 type Manager interface {
 	// SetContainerStartup updates the container status with the given startup
 	// and triggers a status update.
-	SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult, pod results.IPod)
+	SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult, pod results.IPod) error
 }
 
 type manager struct{}
@@ -38,7 +40,7 @@ func NewManager() Manager {
 	return &manager{}
 }
 
-func (m *manager) SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult, pod results.IPod) {
+func (m *manager) SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult, pod results.IPod) error {
 	status := computeapi.CONTAINER_STATUS_PROBE_FAILED
 	if started {
 		status = computeapi.CONTAINER_STATUS_RUNNING
@@ -54,8 +56,19 @@ func (m *manager) SetContainerStartup(podId string, containerId string, started
 		},
 	}
 	if _, err := hostutils.UpdateContainerStatus(context.Background(), containerId, input); err != nil {
-		log.Errorf("set container(%s/%s) status failed: %s", podId, containerId, err)
+		err = errors.Wrapf(err, "set container(%s/%s) status failed, input: %s", podId, containerId, jsonutils.Marshal(input))
+		log.Warningf(err.Error())
+		errMsg := []string{
+			"service is abnormal",
+			"connection refused",
+		}
+		for _, msg := range errMsg {
+			if strings.Contains(err.Error(), msg) {
+				return errors.Wrap(err, "update container status")
+			}
+		}
 	} else {
 		log.Infof("set container(%s/%s) status to %s", podId, containerId, jsonutils.Marshal(input).String())
 	}
+	return nil
 }
diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go
index 9492e435fa..ebd8dce472 100644
--- a/pkg/hostman/guestman/pod.go
+++ b/pkg/hostman/guestman/pod.go
@@ -1930,8 +1930,8 @@ func (s *sPodGuestInstance) getContainerStatus(ctx context.Context, ctrId string
 
 func (s *sPodGuestInstance) markContainerProbeDirty(status, ctrId string, reason string) {
 	if status == computeapi.CONTAINER_STATUS_PROBING {
-		log.Infof("mark container %s to dirty: %s", ctrId, reason)
-		s.getProbeManager().SetDirtyContainer(ctrId)
+		reason = fmt.Sprintf("status is probing: %s", reason)
+		s.getProbeManager().SetDirtyContainer(ctrId, reason)
 	}
 }