From e41a91ff601622a39ce3a4d1b9a7fa03762864c6 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 16 Jan 2025 10:42:00 +0800 Subject: [PATCH] fix(host): container exec probing (#21979) --- pkg/compute/models/containers.go | 7 +++++ pkg/hostman/container/lifecycle/exec.go | 11 +------ pkg/hostman/container/prober/prober.go | 9 +++++- .../container/volume_mount/disk/disk.go | 2 +- pkg/hostman/guestman/pod.go | 7 +---- pkg/util/pod/pod.go | 31 +++++++++++++++++++ pkg/util/probe/exec/exec.go | 12 ++++--- pkg/util/probe/exec/exec_test.go | 6 ++-- 8 files changed, 59 insertions(+), 26 deletions(-) diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index 6382c6f3bb..207dc8a9e9 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -437,6 +437,13 @@ func (c *SContainer) ValidateUpdateData(ctx context.Context, userCred mcclient.T return input, nil } +func (c *SContainer) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + c.SVirtualResourceBase.PostUpdate(ctx, userCred, query, data) + if err := c.GetPod().StartSyncTaskWithoutSyncstatus(ctx, userCred, false, ""); err != nil { + log.Errorf("container %s StartSyncTaskWithoutSyncstatus error: %v", c.GetName(), err) + } +} + func (c *SContainer) GetPod() *SGuest { return GuestManager.FetchGuestById(c.GuestId) } diff --git a/pkg/hostman/container/lifecycle/exec.go b/pkg/hostman/container/lifecycle/exec.go index 813efcd301..591d5bc1ae 100644 --- a/pkg/hostman/container/lifecycle/exec.go +++ b/pkg/hostman/container/lifecycle/exec.go @@ -17,8 +17,6 @@ package lifecycle import ( "context" - runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -42,17 +40,10 @@ func (e execDriver) GetType() apis.ContainerLifecyleHandlerType { func (e execDriver) Run(ctx context.Context, input *apis.ContainerLifecyleHandler, cri pod.CRI, id string) error { cfg := input.Exec - cli := cri.GetRuntimeClient() - resp, err := cli.ExecSync(ctx, &runtimeapi.ExecSyncRequest{ - ContainerId: id, - Cmd: cfg.Command, - }) + resp, err := cri.ExecSync(ctx, id, cfg.Command, 0) if err != nil { return errors.Wrapf(err, "exec sync") } - if resp.GetExitCode() != 0 { - return errors.Wrapf(err, "stdout: %s, stderr: %s, exited: %d", resp.GetStdout(), resp.GetStderr(), resp.GetExitCode()) - } log.Infof("run command %v: stdout: %s, stderr: %s", cfg.Command, resp.Stdout, resp.Stderr) return nil } diff --git a/pkg/hostman/container/prober/prober.go b/pkg/hostman/container/prober/prober.go index 6f2e3691f5..7d965677f9 100644 --- a/pkg/hostman/container/prober/prober.go +++ b/pkg/hostman/container/prober/prober.go @@ -33,6 +33,7 @@ package prober import ( "fmt" "io" + "strings" "time" "yunion.io/x/log" @@ -127,7 +128,7 @@ func (pb *prober) runProbe(probeType apis.ContainerProbeType, p *apis.ContainerP timeout := time.Duration(p.TimeoutSeconds) * time.Second if p.Exec != nil { log.Debugf("Exec-Probe Pod: %v, Container: %v, Command: %v", pod.GetDesc().Name, container.Name, p.Exec.Command) - return pb.exec.Probe(pb.newExecInContainer(pod, container, p.Exec.Command, timeout)) + return pb.exec.Probe(pb.newExecInContainer(pod, container, p.Exec.Command, timeout), strings.Join(p.Exec.Command, " ")) } if p.TCPSocket != nil { port := p.TCPSocket.Port @@ -156,16 +157,22 @@ type execInContainer struct { // error is returned if one occurred. run func() ([]byte, error) writer io.Writer + cmd []string } func (pb *prober) newExecInContainer(pod IPod, container *hostapi.ContainerDesc, cmd []string, timeout time.Duration) exec.Cmd { return &execInContainer{ + cmd: cmd, run: func() ([]byte, error) { return pb.runner.RunInContainer(pod.GetId(), container.Id, cmd, timeout) }, } } +func (eic *execInContainer) Command() []string { + return eic.cmd +} + func (eic *execInContainer) Run() error { return nil } diff --git a/pkg/hostman/container/volume_mount/disk/disk.go b/pkg/hostman/container/volume_mount/disk/disk.go index cee20a4289..100e42d921 100644 --- a/pkg/hostman/container/volume_mount/disk/disk.go +++ b/pkg/hostman/container/volume_mount/disk/disk.go @@ -287,7 +287,7 @@ func (d disk) Unmount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.Conta } if len(vm.Disk.PostOverlay) != 0 { if err := d.UnmountPostOverlays(pod, ctrId, vm, vm.Disk.PostOverlay, false, false); err != nil { - return errors.Wrap(err, "mount post overlay dirs") + return errors.Wrap(err, "umount post overlay dirs") } } if vm.Disk.Overlay != nil { diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 43f5556a83..4ca88201cf 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -99,12 +99,7 @@ func (cr *containerRunner) RunInContainer(podId string, containerId string, cmd if err != nil { return nil, errors.Wrap(err, "get container cri id") } - cli := s.getCRI().GetRuntimeClient() - resp, err := cli.ExecSync(context.Background(), &runtimeapi.ExecSyncRequest{ - ContainerId: ctrCriId, - Cmd: cmd, - Timeout: int64(timeout), - }) + resp, err := s.getCRI().ExecSync(context.Background(), ctrCriId, cmd, int64(timeout.Seconds())) if err != nil { return nil, errors.Wrapf(err, "exec sync %#v to %s", cmd, ctrCriId) } diff --git a/pkg/util/pod/pod.go b/pkg/util/pod/pod.go index de9244d581..e56d4b334d 100644 --- a/pkg/util/pod/pod.go +++ b/pkg/util/pod/pod.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/util/exec" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -49,6 +50,7 @@ type CRI interface { ListImages(ctx context.Context, filter *runtimeapi.ImageFilter) ([]*runtimeapi.Image, error) PullImage(ctx context.Context, req *runtimeapi.PullImageRequest) (*runtimeapi.PullImageResponse, error) ImageStatus(ctx context.Context, req *runtimeapi.ImageStatusRequest) (*runtimeapi.ImageStatusResponse, error) + ExecSync(ctx context.Context, ctrId string, command []string, timeout int64) (*ExecSyncResponse, error) // lower layer client GetImageClient() runtimeapi.ImageServiceClient @@ -509,3 +511,32 @@ func (c crictl) PullImage(ctx context.Context, req *runtimeapi.PullImageRequest) func (c crictl) ImageStatus(ctx context.Context, req *runtimeapi.ImageStatusRequest) (*runtimeapi.ImageStatusResponse, error) { return c.GetImageClient().ImageStatus(ctx, req) } + +type ExecSyncResponse struct { + Stdout []byte + Stderr []byte + ExitCode int32 +} + +func (c crictl) ExecSync(ctx context.Context, ctrId string, command []string, timeout int64) (*ExecSyncResponse, error) { + cli := c.GetRuntimeClient() + resp, err := cli.ExecSync(ctx, &runtimeapi.ExecSyncRequest{ + ContainerId: ctrId, + Cmd: command, + Timeout: timeout, + }) + if err != nil { + return nil, errors.Wrapf(err, "exec sync container %s with command: %v", ctrId, command) + } + if resp.GetExitCode() != 0 { + return nil, exec.CodeExitError{ + Err: errors.Errorf("stdout: %s, stderr: %s, exited: %d", resp.GetStdout(), resp.GetStderr(), resp.GetExitCode()), + Code: int(resp.ExitCode), + } + } + return &ExecSyncResponse{ + Stdout: resp.Stdout, + Stderr: resp.Stderr, + ExitCode: resp.ExitCode, + }, nil +} diff --git a/pkg/util/probe/exec/exec.go b/pkg/util/probe/exec/exec.go index 739e53cbba..60361b152f 100644 --- a/pkg/util/probe/exec/exec.go +++ b/pkg/util/probe/exec/exec.go @@ -32,8 +32,10 @@ package exec import ( "bytes" + "fmt" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/util/exec" "yunion.io/x/onecloud/pkg/util/ioutils" @@ -46,7 +48,7 @@ const ( // Prober is an interface defining the Probe object for container readiness/liveness checks. type Prober interface { - Probe(e exec.Cmd) (probe.Result, string, error) + Probe(e exec.Cmd, info string) (probe.Result, string, error) } // New creates a Prober. @@ -59,7 +61,7 @@ type execProber struct{} // Probe executes a command to check the liveness/readiness of container // from executing a command. Returns the Result status, command output, and // errors if any. -func (pr execProber) Probe(e exec.Cmd) (probe.Result, string, error) { +func (pr execProber) Probe(e exec.Cmd, info string) (probe.Result, string, error) { var dataBuffer bytes.Buffer writer := ioutils.LimitWriter(&dataBuffer, maxReadLength) @@ -71,14 +73,14 @@ func (pr execProber) Probe(e exec.Cmd) (probe.Result, string, error) { } data := dataBuffer.Bytes() - log.Infof("Exec probe response: %q", string(data)) + log.Debugf("Exec probe response: %q, error: %v", string(data), err) if err != nil { - exit, ok := err.(exec.ExitError) + exit, ok := errors.Cause(err).(exec.ExitError) if ok { if exit.ExitStatus() == 0 { return probe.Success, string(data), nil } - return probe.Failure, string(data), nil + return probe.Failure, fmt.Sprintf("%s, %s: %s", info, exit, data), nil } timeoutErr, ok := err.(*TimeoutError) diff --git a/pkg/util/probe/exec/exec_test.go b/pkg/util/probe/exec/exec_test.go index 640aaa85b0..0e3351fe13 100644 --- a/pkg/util/probe/exec/exec_test.go +++ b/pkg/util/probe/exec/exec_test.go @@ -142,7 +142,7 @@ func TestExec(t *testing.T) { out: []byte(test.output), err: test.err, } - status, output, err := prober.Probe(&fake) + status, output, err := prober.Probe(&fake, "") if status != test.expectedStatus { t.Errorf("[%d] expected %v, got %v", i, test.expectedStatus, status) } @@ -152,8 +152,8 @@ func TestExec(t *testing.T) { if err == nil && test.expectError == true { t.Errorf("[%d] unexpected non-error", i) } - if test.output != output { - t.Errorf("[%d] expected %s, got %s", i, test.output, output) + if status == probe.Success && test.output != output { + t.Errorf("[%d] expected %q, got %q", i, test.output, output) } } }