From cc0b8ce289e27e1e630797540235ee14600b2d42 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 29 Oct 2024 20:56:43 +0800 Subject: [PATCH] =?UTF-8?q?fix(host):=20=E4=BF=AE=E5=A4=8D=20dirty=20shutd?= =?UTF-8?q?own=20=E6=9C=BA=E5=88=B6=E5=92=8C=20reconcile=20=E7=9A=84?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98=20(#21492)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/hostman/guestman/guestman.go | 2 +- pkg/hostman/guestman/pod.go | 32 +++++++++++++-------------- pkg/hostman/guestman/pod_sync_loop.go | 10 +++++++++ 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 577eafd2d0..5c68e2b755 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -152,7 +152,6 @@ func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (* manager.containerRuntimeManager = runtimeMan manager.pleg = pleg.NewGenericPLEG(runtimeMan, pleg.ChannelCapacity, pleg.RelistPeriod, manager.podCache, clock.RealClock{}) manager.pleg.Start() - manager.startContainerSyncLoop() } return manager, nil } @@ -441,6 +440,7 @@ func (m *SGuestManager) OnLoadExistingGuestsComplete() { if !options.HostOptions.EnableCpuBinding { m.ClenaupCpuset() } + m.startContainerSyncLoop() } func (m *SGuestManager) verifyDirtyServers() { diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 5ce298f823..629fbd07f0 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -273,13 +273,11 @@ func (s *sPodGuestInstance) ImportServer(pendingDelete bool) { s.manager.SaveServer(s.Id, s) s.manager.RemoveCandidateServer(s) if s.IsDaemon() || s.IsDirtyShutdown() { - /*ctx := context.Background() + ctx := context.Background() cred := hostutils.GetComputeSession(ctx).GetToken() - if err := s.StartLocalPod(ctx, cred); err != nil { + if err := s.StartLocalDirtyPod(ctx, cred); err != nil { log.Errorf("start local pod err %s", err.Error()) - }*/ - log.Warningf("pod %s need started, waiting sync loop to manage it", s.GetName()) - s.SyncStatus(fmt.Sprintf("sync status is_dirty_shutdown: %v, is_daemon: %v", s.IsDirtyShutdown(), s.IsDaemon())) + } } else { s.SyncStatus("sync status after host started") s.getProbeManager().AddPod(s.Desc) @@ -688,44 +686,44 @@ func (s *sPodGuestInstance) getCgroupParent() string { return "/cloudpods" } -type localPodStartTask struct { +type localDirtyPodStartTask struct { ctx context.Context userCred mcclient.TokenCredential pod *sPodGuestInstance } -func newLocalPodStartTask(ctx context.Context, userCred mcclient.TokenCredential, pod *sPodGuestInstance) *localPodStartTask { - return &localPodStartTask{ +func newLocalDirtyPodStartTask(ctx context.Context, userCred mcclient.TokenCredential, pod *sPodGuestInstance) *localDirtyPodStartTask { + return &localDirtyPodStartTask{ ctx: ctx, userCred: userCred, pod: pod, } } -func (t *localPodStartTask) Run() { +func (t *localDirtyPodStartTask) Run() { if t.pod.isPodDirtyShutdown() { - log.Infof("start pod locally (%s/%s)", t.pod.Id, t.pod.GetName()) + log.Infof("start dirty pod locally (%s/%s)", t.pod.Id, t.pod.GetName()) if _, err := t.pod.startPod(t.ctx, t.userCred); err != nil { - log.Errorf("start pod(%s/%s) err: %s", t.pod.GetId(), t.pod.GetName(), err.Error()) + log.Errorf("start dirty pod(%s/%s) err: %s", t.pod.GetId(), t.pod.GetName(), err.Error()) } } for _, ctr := range t.pod.GetContainers() { if t.pod.isContainerDirtyShutdown(ctr.Id) { - log.Infof("start container locally (%s/%s/%s/%s)", t.pod.Id, t.pod.GetName(), ctr.Id, ctr.Name) + log.Infof("start dirty container locally (%s/%s/%s/%s)", t.pod.Id, t.pod.GetName(), ctr.Id, ctr.Name) if _, err := t.pod.StartLocalContainer(t.ctx, t.userCred, ctr.Id); err != nil { - log.Errorf("start container %s err: %s", ctr.Id, err.Error()) + log.Errorf("start dirty container %s err: %s", ctr.Id, err.Error()) } } } - t.pod.SyncStatus("sync status after pod start locally") + t.pod.SyncStatus("sync status after dirty pod start locally") } -func (t *localPodStartTask) Dump() string { +func (t *localDirtyPodStartTask) Dump() string { return fmt.Sprintf("pod start task %s/%s", t.pod.GetId(), t.pod.GetName()) } -func (s *sPodGuestInstance) StartLocalPod(ctx context.Context, userCred mcclient.TokenCredential) error { - s.manager.GuestStartWorker.Run(newLocalPodStartTask(ctx, userCred, s), nil, nil) +func (s *sPodGuestInstance) StartLocalDirtyPod(ctx context.Context, userCred mcclient.TokenCredential) error { + s.manager.GuestStartWorker.Run(newLocalDirtyPodStartTask(ctx, userCred, s), nil, nil) return nil } diff --git a/pkg/hostman/guestman/pod_sync_loop.go b/pkg/hostman/guestman/pod_sync_loop.go index 3ba77a6f6f..11528cc47c 100644 --- a/pkg/hostman/guestman/pod_sync_loop.go +++ b/pkg/hostman/guestman/pod_sync_loop.go @@ -20,12 +20,17 @@ import ( ) func (m *SGuestManager) reconcileContainerLoop(cache runtime.Cache) { + log.Infof("start reconcile container loop") for { m.Servers.Range(func(id, obj interface{}) bool { podObj, ok := obj.(*sPodGuestInstance) if !ok { return true } + if podObj.isPodDirtyShutdown() { + log.Infof("pod %s is dirty shutdown, using dirty shutdown manager to start it", podObj.GetName()) + return true + } if err := m.reconcileContainer(podObj, cache); err != nil { log.Warningf("reconcile pod %s: %v", podObj.GetId(), err) } @@ -108,6 +113,7 @@ func (m *SGuestManager) startContainer(obj *sPodGuestInstance, ctr *hostapi.Cont } func (m *SGuestManager) syncContainerLoop(plegCh chan *pleg.PodLifecycleEvent) { + log.Infof("start sync container loop") for { m.syncContainerLoopIteration(plegCh) } @@ -121,6 +127,10 @@ func (m *SGuestManager) syncContainerLoopIteration(plegCh chan *pleg.PodLifecycl log.Warningf("can not find pod manager by %s", jsonutils.Marshal(e)) return } + if podMan.(*sPodGuestInstance).isPodDirtyShutdown() { + log.Infof("pod %s is dirty shutdown, waiting it to started", podMan.GetName()) + return + } if e.Type == pleg.ContainerStarted { log.Infof("pod container started: %s", jsonutils.Marshal(e)) ctrId := e.Data.(string)