From 13034b73877b403d211cd3f200daab30de8121e9 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 28 Sep 2026 20:51:51 +0800 Subject: [PATCH] feat(devtool,host,ansible): offline monitor agent and ixsmi path resolve (#25743) - Add NPU/GPU accelerator inputs (npu_smi, ixsmi, ppusmi, hysmi, xpusmi) to baremetal telegraf config. - Resolve Iluvatar ixsmi symlink to detect corex home dynamically. - Treat baremetal like KVM for direct service URL probing. - Improve ansible error logging and fix playbook state/key-file handling. --- build/docker/Dockerfile.ansibleserver | 2 +- pkg/devtool/utils/gen_telegraf_conf.go | 10 ++++ pkg/devtool/utils/gen_telegraf_conf_test.go | 21 ++++++++ pkg/devtool/utils/service_url.go | 11 ++-- pkg/hostman/hostinfo/hostinfo.go | 30 ++++++++--- .../hostinfo/hostinfo_telegraf_test.go | 49 +++++++++++++++++- .../container_device/iluvatar_gpu.go | 29 ++++++++--- .../container_device/iluvatar_ixsmi_test.go | 51 +++++++++++++++++++ pkg/hostman/options/options.go | 3 +- pkg/util/ansible/playbook.go | 7 ++- 10 files changed, 189 insertions(+), 24 deletions(-) diff --git a/build/docker/Dockerfile.ansibleserver b/build/docker/Dockerfile.ansibleserver index 5dd396950c..d99c7dd2d9 100644 --- a/build/docker/Dockerfile.ansibleserver +++ b/build/docker/Dockerfile.ansibleserver @@ -1,4 +1,4 @@ -ARG FILE_REPO_TAG=v0.5.9 +ARG FILE_REPO_TAG=v0.5.10 # https://github.com/yunionio/telegraf/blob/yn-release-1.36/Makefile.yn#L28 # install playbook and telegraf install pkg diff --git a/pkg/devtool/utils/gen_telegraf_conf.go b/pkg/devtool/utils/gen_telegraf_conf.go index 6f18131661..45c91e166f 100644 --- a/pkg/devtool/utils/gen_telegraf_conf.go +++ b/pkg/devtool/utils/gen_telegraf_conf.go @@ -171,6 +171,16 @@ const TELEGRAF_INPUT_BAREMETAL = ` collect_memstats = false [[inputs.nvidia_smi]] name_prefix = "agent_" +[[inputs.npu_smi]] + name_prefix = "agent_" +[[inputs.ixsmi]] + name_prefix = "agent_" +[[inputs.ppusmi]] + name_prefix = "agent_" +[[inputs.hysmi]] + name_prefix = "agent_" +[[inputs.xpusmi]] + name_prefix = "agent_" ` var temp *template.Template diff --git a/pkg/devtool/utils/gen_telegraf_conf_test.go b/pkg/devtool/utils/gen_telegraf_conf_test.go index bcbdc5fcc4..b5a2d6b9e7 100644 --- a/pkg/devtool/utils/gen_telegraf_conf_test.go +++ b/pkg/devtool/utils/gen_telegraf_conf_test.go @@ -15,6 +15,8 @@ package utils import ( + "fmt" + "strings" "testing" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -39,6 +41,12 @@ func TestGenerateTelegrafConf(t *testing.T) { want: "", wantErr: false, }, + { + name: "test_baremetal_accelerators", + args: args{new(api.ServerDetails), "http://127.0.0.1:8086", "Linux", api.HYPERVISOR_BAREMETAL}, + want: "", + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -47,6 +55,19 @@ func TestGenerateTelegrafConf(t *testing.T) { t.Errorf("GenerateTelegrafConf() error = %v, wantErr %v", err, tt.wantErr) return } + if tt.args.serverType == api.HYPERVISOR_BAREMETAL { + for _, plugin := range []string{ + "nvidia_smi", "npu_smi", "ixsmi", + "ppusmi", "hysmi", "xpusmi", + } { + if !strings.Contains(got, fmt.Sprintf("[[inputs.%s]]", plugin)) { + t.Errorf("GenerateTelegrafConf() missing input %s", plugin) + } + } + if !strings.Contains(got, `name_prefix = "agent_"`) { + t.Errorf("GenerateTelegrafConf() missing name_prefix = \"agent_\"") + } + } t.Logf("got %s", got) }) } diff --git a/pkg/devtool/utils/service_url.go b/pkg/devtool/utils/service_url.go index 3d76ed90d4..b20c560a5f 100644 --- a/pkg/devtool/utils/service_url.go +++ b/pkg/devtool/utils/service_url.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/httputils" "yunion.io/x/pkg/util/sets" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" ansible_api "yunion.io/x/onecloud/pkg/apis/ansibleserver" @@ -121,7 +122,7 @@ func serviceUrlDirect(ctx context.Context, service Service, proxyEndpointId stri url, code := serviceComplete2(service) ok, err := checkUrl(ctx, url, code, host) if err != nil { - return "", err + return "", errors.Wrapf(err, "check url %s with returned code %d: %s", url, code, jsonutils.Marshal(host)) } if ok { return service.Url, nil @@ -192,13 +193,11 @@ func serviceUrlViaProxyEndpoint(ctx context.Context, service Service, proxyEndpo func FindValidServiceUrl(ctx context.Context, service Service, proxyEndpointId string, info sServerInfo, host *ansible_api.AnsibleHost) (string, error) { findFuncs := []func(ctx context.Context, service Service, proxyEndpointId string, info sServerInfo, host *ansible_api.AnsibleHost) (string, error){} - if info.serverDetails.Hypervisor == comapi.HYPERVISOR_KVM { + if utils.IsInStringArray(info.serverDetails.Hypervisor, []string{comapi.HYPERVISOR_KVM, comapi.HYPERVISOR_BAREMETAL}) { // KVM guests report metrics through the metadata service on the host or // the public TSDB endpoint, so never create a proxy endpoint forward for // them. findFuncs = append(findFuncs, serviceUrlDirect) - } else if info.serverDetails.Hypervisor == comapi.HYPERVISOR_BAREMETAL { - findFuncs = append(findFuncs, serviceUrlDirect, serviceUrlViaProxyEndpoint) } else { findFuncs = append(findFuncs, serviceUrlViaProxyEndpoint, serviceUrlDirect) } @@ -399,6 +398,10 @@ func checkUrl(ctx context.Context, completeUrl string, expectedCode int, host *a case ansible_api.AnsiblePlaybookStatusInit, ansible_api.AnsiblePlaybookStatusRunning: continue case ansible_api.AnsiblePlaybookStatusFailed, ansible_api.AnsiblePlaybookStatusCanceled, ansible_api.AnsiblePlaybookStatusUnknown: + if status == ansible_api.AnsiblePlaybookStatusFailed { + obj, err := ansible_modules.AnsiblePlaybooks.Get(session, id, nil) + log.Errorf("run ansible playbook %s failed: %s, err: %v", id, jsonutils.Marshal(obj).PrettyString(), err) + } return false, nil case ansible_api.AnsiblePlaybookStatusSucceeded: return true, nil diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index bb716381d3..f7fb820f9a 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2689,6 +2689,22 @@ func (h *SHostInfo) OnCatalogChanged(catalog mcclient.KeystoneServiceCatalogV3) }*/ } +func resolveSmiBinPath(binPath string) string { + return resolveSmiBinPathWithReadlink(binPath, procutils.RemoteReadlink) +} + +func resolveSmiBinPathWithReadlink(binPath string, readlink func(string) (string, error)) string { + if binPath == "" { + return binPath + } + resolved, err := readlink(binPath) + if err != nil || resolved == "" { + log.Warningf("failed to resolve smi binary path %q: %v", binPath, err) + return binPath + } + return resolved +} + func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { devs := h.GetIsolatedDeviceManager().GetDevices() if len(devs) == 0 { @@ -2764,16 +2780,18 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { } } if hasIluvatar { - corexHome := options.HostOptions.IluvatarCorexHome - if corexHome == "" { - corexHome = "/usr/local/corex-4.4.0" - } + defaultCorexHome := "/usr/local/corex-4.4.0" ixsmiPath := options.HostOptions.IluvatarIxsmiPath if ixsmiPath == "" { - ixsmiPath = path.Join(corexHome, "bin", "ixsmi") + ixsmiPath = "/usr/local/bin/ixsmi" + } + resolvedPath := resolveSmiBinPath(ixsmiPath) + corexHome := defaultCorexHome + if resolvedPath != ixsmiPath { + corexHome = path.Dir(path.Dir(resolvedPath)) } conf[system_service.TELEGRAF_INPUT_IXSMI] = map[string]interface{}{ - system_service.TELEGRAF_INPUT_CONF_BIN_PATH: ixsmiPath, + system_service.TELEGRAF_INPUT_CONF_BIN_PATH: resolvedPath, system_service.TELEGRAF_INPUT_CONF_LIB_PATH: path.Join(corexHome, "lib64"), } } diff --git a/pkg/hostman/hostinfo/hostinfo_telegraf_test.go b/pkg/hostman/hostinfo/hostinfo_telegraf_test.go index d2ec0fcdf1..64e324b95e 100644 --- a/pkg/hostman/hostinfo/hostinfo_telegraf_test.go +++ b/pkg/hostman/hostinfo/hostinfo_telegraf_test.go @@ -14,7 +14,10 @@ package hostinfo -import "testing" +import ( + "fmt" + "testing" +) func TestResolveKunlunxinXpuLibDir(t *testing.T) { cases := []struct { @@ -72,3 +75,47 @@ func TestResolveKunlunxinXpuLibDir(t *testing.T) { t.Errorf("resolveKunlunxinXpuLibDir with nil exists = %q, expected %q", got, "/usr/local/xpu/so") } } + +func TestResolveSmiBinPathWithReadlink(t *testing.T) { + cases := []struct { + name string + binPath string + resolved string + readErr error + want string + }{ + { + name: "resolve symlink", + binPath: "/usr/local/bin/ixsmi", + resolved: "/usr/local/corex-4.3.0/bin/ixsmi", + want: "/usr/local/corex-4.3.0/bin/ixsmi", + }, + { + name: "fallback on error", + binPath: "/usr/local/bin/ixsmi", + readErr: fmt.Errorf("no such file"), + want: "/usr/local/bin/ixsmi", + }, + { + name: "fallback on empty result", + binPath: "/usr/local/bin/ixsmi", + resolved: "", + want: "/usr/local/bin/ixsmi", + }, + { + name: "empty input", + binPath: "", + want: "", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + readlink := func(string) (string, error) { + return c.resolved, c.readErr + } + if got := resolveSmiBinPathWithReadlink(c.binPath, readlink); got != c.want { + t.Errorf("resolveSmiBinPathWithReadlink(%q) = %q, expected %q", c.binPath, got, c.want) + } + }) + } +} diff --git a/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go b/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go index b1783c6820..a54bac0ad6 100644 --- a/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go +++ b/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go @@ -120,20 +120,33 @@ func iluvatarCommonDevices() []*runtimeapi.Device { return devs } -func iluvatarCorexHome() string { - home := options.HostOptions.IluvatarCorexHome - if home == "" { - return defaultIluvatarCorexHome - } - return home -} +const defaultIluvatarIxsmiPath = "/usr/local/bin/ixsmi" func iluvatarIxsmiPath() string { p := options.HostOptions.IluvatarIxsmiPath if p != "" { return p } - return path.Join(iluvatarCorexHome(), "bin", "ixsmi") + return defaultIluvatarIxsmiPath +} + +func iluvatarCorexHome() string { + return iluvatarCorexHomeWithReadlink(iluvatarIxsmiPath(), procutils.RemoteReadlink) +} + +func iluvatarCorexHomeWithReadlink(ixsmiPath string, readlink func(string) (string, error)) string { + if ixsmiPath == "" { + ixsmiPath = defaultIluvatarIxsmiPath + } + resolved, err := readlink(ixsmiPath) + if err != nil || resolved == "" || resolved == ixsmiPath { + return defaultIluvatarCorexHome + } + home := path.Dir(path.Dir(resolved)) + if home == "" || home == "/" { + return defaultIluvatarCorexHome + } + return home } func iluvatarLDLibraryPath() string { diff --git a/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go b/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go index 8852d5acb4..7ff4acb5cb 100644 --- a/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go +++ b/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go @@ -15,6 +15,7 @@ package container_device import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -104,3 +105,53 @@ func TestIluvatarPCIAddrCandidates(t *testing.T) { cands := iluvatarPCIAddrCandidates("00000000:26:00.0") assert.Equal(t, []string{"00000000:26:00.0", "0000:26:00.0", "26:00.0"}, cands) } + +func TestIluvatarCorexHomeWithReadlink(t *testing.T) { + cases := []struct { + name string + ixsmi string + resolved string + readErr error + want string + }{ + { + name: "resolve symlink", + ixsmi: "/usr/local/bin/ixsmi", + resolved: "/usr/local/corex-4.3.0/bin/ixsmi", + want: "/usr/local/corex-4.3.0", + }, + { + name: "fallback on error", + ixsmi: "/usr/local/bin/ixsmi", + readErr: fmt.Errorf("no such file"), + want: defaultIluvatarCorexHome, + }, + { + name: "fallback when not a symlink", + ixsmi: "/usr/local/bin/ixsmi", + resolved: "/usr/local/bin/ixsmi", + want: defaultIluvatarCorexHome, + }, + { + name: "fallback on empty result", + ixsmi: "/usr/local/bin/ixsmi", + resolved: "", + want: defaultIluvatarCorexHome, + }, + { + name: "fallback on empty input", + ixsmi: "", + want: defaultIluvatarCorexHome, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + readlink := func(string) (string, error) { + return c.resolved, c.readErr + } + if got := iluvatarCorexHomeWithReadlink(c.ixsmi, readlink); got != c.want { + t.Errorf("iluvatarCorexHomeWithReadlink(%q) = %q, expected %q", c.ixsmi, got, c.want) + } + }) + } +} diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 541e28027b..01c56fd889 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -290,8 +290,7 @@ type SHostOptions struct { HygonVgpuCacheDir string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"` EnableContainerIluvatarGPU bool `help:"enable container iluvatar gpu" default:"true"` - IluvatarCorexHome string `help:"iluvatar corex home" default:"/usr/local/corex-4.4.0"` - IluvatarIxsmiPath string `help:"iluvatar ixsmi path" default:"/usr/local/corex-4.4.0/bin/ixsmi"` + IluvatarIxsmiPath string `help:"iluvatar ixsmi path" default:"/usr/local/bin/ixsmi"` EnableContainerTHeadPPU bool `help:"enable container t-head ppu" default:"true"` THeadPpuSdkHome string `help:"t-head ppu sdk home" default:"/usr/local/PPU_SDK"` diff --git a/pkg/util/ansible/playbook.go b/pkg/util/ansible/playbook.go index 6cd0b4f47d..0ae137a0fe 100644 --- a/pkg/util/ansible/playbook.go +++ b/pkg/util/ansible/playbook.go @@ -103,10 +103,13 @@ func (pb *Playbook) Running() bool { func (pb *Playbook) Run(ctx context.Context) error { errs := make([]error, 0, len(pb.PrivateKeys)) - for _, privateKey := range pb.PrivateKeys { + for i, privateKey := range pb.PrivateKeys { err := pb.runOnce(ctx, privateKey) if err != nil { errs = append(errs, err) + if i != len(pb.PrivateKeys)-1 { + pb.state = pbStateInit + } } else { return nil } @@ -203,7 +206,7 @@ func (pb *Playbook) runOnce(ctx context.Context, privateKey string) error { "all", } if privateKey != "" { - args = append(args, "--private-key", privateKey) + args = append(args, "--private-key", privateKeyFile) } cmd := exec.CommandContext(ctx, "ansible", args...) cmd.Dir = pb.tmpdir