opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-10-16 15:07:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(devtool,host,ansible): offline monitor agent and ixsmi path resolve (#25743)

Browse Source 
- Add NPU/GPU accelerator inputs (npu_smi, ixsmi, ppusmi, hysmi, xpusmi)
  to baremetal telegraf config.
- Resolve Iluvatar ixsmi symlink to detect corex home dynamically.
- Treat baremetal like KVM for direct service URL probing.
- Improve ansible error logging and fix playbook state/key-file handling.
This commit is contained in:
Zexi Li
2026-09-28 20:51:51 +08:00
committed by GitHub
parent a6dc2f471f
commit 13034b7387
10 changed files with 189 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/docker/Dockerfile.ansibleserver
+1 -1
View File
@@ -1,4 +1,4 @@
ARG FILE_REPO_TAG=v0.5.9
ARG FILE_REPO_TAG=v0.5.10
# https://github.com/yunionio/telegraf/blob/yn-release-1.36/Makefile.yn#L28
# install playbook and telegraf install pkg
pkg/devtool/utils/gen_telegraf_conf.go
+10
View File
@@ -171,6 +171,16 @@ const TELEGRAF_INPUT_BAREMETAL = `
collect_memstats = false
[[inputs.nvidia_smi]]
name_prefix = "agent_"
[[inputs.npu_smi]]
name_prefix = "agent_"
[[inputs.ixsmi]]
name_prefix = "agent_"
[[inputs.ppusmi]]
name_prefix = "agent_"
[[inputs.hysmi]]
name_prefix = "agent_"
[[inputs.xpusmi]]
name_prefix = "agent_"
`
var temp *template.Template
pkg/devtool/utils/gen_telegraf_conf_test.go
+21
View File
@@ -15,6 +15,8 @@
package utils
import (
"fmt"
"strings"
"testing"
api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -39,6 +41,12 @@ func TestGenerateTelegrafConf(t *testing.T) {
want: "",
wantErr: false,
},
{
name: "test_baremetal_accelerators",
args: args{new(api.ServerDetails), "http://127.0.0.1:8086", "Linux", api.HYPERVISOR_BAREMETAL},
want: "",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -47,6 +55,19 @@ func TestGenerateTelegrafConf(t *testing.T) {
t.Errorf("GenerateTelegrafConf() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.args.serverType == api.HYPERVISOR_BAREMETAL {
for _, plugin := range []string{
"nvidia_smi", "npu_smi", "ixsmi",
"ppusmi", "hysmi", "xpusmi",
} {
if !strings.Contains(got, fmt.Sprintf("[[inputs.%s]]", plugin)) {
t.Errorf("GenerateTelegrafConf() missing input %s", plugin)
}
}
if !strings.Contains(got, `name_prefix = "agent_"`) {
t.Errorf("GenerateTelegrafConf() missing name_prefix = \"agent_\"")
}
}
t.Logf("got %s", got)
})
}
pkg/devtool/utils/service_url.go
+7 -4
View File
@@ -25,6 +25,7 @@ import (
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/httputils"
"yunion.io/x/pkg/util/sets"
"yunion.io/x/pkg/utils"
"yunion.io/x/onecloud/pkg/apis"
ansible_api "yunion.io/x/onecloud/pkg/apis/ansibleserver"
@@ -121,7 +122,7 @@ func serviceUrlDirect(ctx context.Context, service Service, proxyEndpointId stri
url, code := serviceComplete2(service)
ok, err := checkUrl(ctx, url, code, host)
if err != nil {
return "", err
return "", errors.Wrapf(err, "check url %s with returned code %d: %s", url, code, jsonutils.Marshal(host))
}
if ok {
return service.Url, nil
@@ -192,13 +193,11 @@ func serviceUrlViaProxyEndpoint(ctx context.Context, service Service, proxyEndpo
func FindValidServiceUrl(ctx context.Context, service Service, proxyEndpointId string, info sServerInfo, host *ansible_api.AnsibleHost) (string, error) {
findFuncs := []func(ctx context.Context, service Service, proxyEndpointId string, info sServerInfo, host *ansible_api.AnsibleHost) (string, error){}
if info.serverDetails.Hypervisor == comapi.HYPERVISOR_KVM {
if utils.IsInStringArray(info.serverDetails.Hypervisor, []string{comapi.HYPERVISOR_KVM, comapi.HYPERVISOR_BAREMETAL}) {
// KVM guests report metrics through the metadata service on the host or
// the public TSDB endpoint, so never create a proxy endpoint forward for
// them.
findFuncs = append(findFuncs, serviceUrlDirect)
} else if info.serverDetails.Hypervisor == comapi.HYPERVISOR_BAREMETAL {
findFuncs = append(findFuncs, serviceUrlDirect, serviceUrlViaProxyEndpoint)
} else {
findFuncs = append(findFuncs, serviceUrlViaProxyEndpoint, serviceUrlDirect)
}
@@ -399,6 +398,10 @@ func checkUrl(ctx context.Context, completeUrl string, expectedCode int, host *a
case ansible_api.AnsiblePlaybookStatusInit, ansible_api.AnsiblePlaybookStatusRunning:
continue
case ansible_api.AnsiblePlaybookStatusFailed, ansible_api.AnsiblePlaybookStatusCanceled, ansible_api.AnsiblePlaybookStatusUnknown:
if status == ansible_api.AnsiblePlaybookStatusFailed {
obj, err := ansible_modules.AnsiblePlaybooks.Get(session, id, nil)
log.Errorf("run ansible playbook %s failed: %s, err: %v", id, jsonutils.Marshal(obj).PrettyString(), err)
}
return false, nil
case ansible_api.AnsiblePlaybookStatusSucceeded:
return true, nil
pkg/hostman/hostinfo/hostinfo.go
+24 -6
View File
@@ -2689,6 +2689,22 @@ func (h *SHostInfo) OnCatalogChanged(catalog mcclient.KeystoneServiceCatalogV3)
}*/
}
func resolveSmiBinPath(binPath string) string {
return resolveSmiBinPathWithReadlink(binPath, procutils.RemoteReadlink)
}
func resolveSmiBinPathWithReadlink(binPath string, readlink func(string) (string, error)) string {
if binPath == "" {
return binPath
}
resolved, err := readlink(binPath)
if err != nil || resolved == "" {
log.Warningf("failed to resolve smi binary path %q: %v", binPath, err)
return binPath
}
return resolved
}
func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
devs := h.GetIsolatedDeviceManager().GetDevices()
if len(devs) == 0 {
@@ -2764,16 +2780,18 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
}
}
if hasIluvatar {
corexHome := options.HostOptions.IluvatarCorexHome
if corexHome == "" {
corexHome = "/usr/local/corex-4.4.0"
}
defaultCorexHome := "/usr/local/corex-4.4.0"
ixsmiPath := options.HostOptions.IluvatarIxsmiPath
if ixsmiPath == "" {
ixsmiPath = path.Join(corexHome, "bin", "ixsmi")
ixsmiPath = "/usr/local/bin/ixsmi"
}
resolvedPath := resolveSmiBinPath(ixsmiPath)
corexHome := defaultCorexHome
if resolvedPath != ixsmiPath {
corexHome = path.Dir(path.Dir(resolvedPath))
}
conf[system_service.TELEGRAF_INPUT_IXSMI] = map[string]interface{}{
system_service.TELEGRAF_INPUT_CONF_BIN_PATH: ixsmiPath,
system_service.TELEGRAF_INPUT_CONF_BIN_PATH: resolvedPath,
system_service.TELEGRAF_INPUT_CONF_LIB_PATH: path.Join(corexHome, "lib64"),
}
}
pkg/hostman/hostinfo/hostinfo_telegraf_test.go
+48 -1
View File
@@ -14,7 +14,10 @@
package hostinfo
import "testing"
import (
"fmt"
"testing"
)
func TestResolveKunlunxinXpuLibDir(t *testing.T) {
cases := []struct {
@@ -72,3 +75,47 @@ func TestResolveKunlunxinXpuLibDir(t *testing.T) {
t.Errorf("resolveKunlunxinXpuLibDir with nil exists = %q, expected %q", got, "/usr/local/xpu/so")
}
}
func TestResolveSmiBinPathWithReadlink(t *testing.T) {
cases := []struct {
name string
binPath string
resolved string
readErr error
want string
}{
{
name: "resolve symlink",
binPath: "/usr/local/bin/ixsmi",
resolved: "/usr/local/corex-4.3.0/bin/ixsmi",
want: "/usr/local/corex-4.3.0/bin/ixsmi",
},
{
name: "fallback on error",
binPath: "/usr/local/bin/ixsmi",
readErr: fmt.Errorf("no such file"),
want: "/usr/local/bin/ixsmi",
},
{
name: "fallback on empty result",
binPath: "/usr/local/bin/ixsmi",
resolved: "",
want: "/usr/local/bin/ixsmi",
},
{
name: "empty input",
binPath: "",
want: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
readlink := func(string) (string, error) {
return c.resolved, c.readErr
}
if got := resolveSmiBinPathWithReadlink(c.binPath, readlink); got != c.want {
t.Errorf("resolveSmiBinPathWithReadlink(%q) = %q, expected %q", c.binPath, got, c.want)
}
})
}
}
pkg/hostman/isolated_device/container_device/iluvatar_gpu.go
+21 -8
View File
@@ -120,20 +120,33 @@ func iluvatarCommonDevices() []*runtimeapi.Device {
return devs
}
func iluvatarCorexHome() string {
home := options.HostOptions.IluvatarCorexHome
if home == "" {
return defaultIluvatarCorexHome
}
return home
}
const defaultIluvatarIxsmiPath = "/usr/local/bin/ixsmi"
func iluvatarIxsmiPath() string {
p := options.HostOptions.IluvatarIxsmiPath
if p != "" {
return p
}
return path.Join(iluvatarCorexHome(), "bin", "ixsmi")
return defaultIluvatarIxsmiPath
}
func iluvatarCorexHome() string {
return iluvatarCorexHomeWithReadlink(iluvatarIxsmiPath(), procutils.RemoteReadlink)
}
func iluvatarCorexHomeWithReadlink(ixsmiPath string, readlink func(string) (string, error)) string {
if ixsmiPath == "" {
ixsmiPath = defaultIluvatarIxsmiPath
}
resolved, err := readlink(ixsmiPath)
if err != nil || resolved == "" || resolved == ixsmiPath {
return defaultIluvatarCorexHome
}
home := path.Dir(path.Dir(resolved))
if home == "" || home == "/" {
return defaultIluvatarCorexHome
}
return home
}
func iluvatarLDLibraryPath() string {
pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go
+51
View File
@@ -15,6 +15,7 @@
package container_device
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -104,3 +105,53 @@ func TestIluvatarPCIAddrCandidates(t *testing.T) {
cands := iluvatarPCIAddrCandidates("00000000:26:00.0")
assert.Equal(t, []string{"00000000:26:00.0", "0000:26:00.0", "26:00.0"}, cands)
}
func TestIluvatarCorexHomeWithReadlink(t *testing.T) {
cases := []struct {
name string
ixsmi string
resolved string
readErr error
want string
}{
{
name: "resolve symlink",
ixsmi: "/usr/local/bin/ixsmi",
resolved: "/usr/local/corex-4.3.0/bin/ixsmi",
want: "/usr/local/corex-4.3.0",
},
{
name: "fallback on error",
ixsmi: "/usr/local/bin/ixsmi",
readErr: fmt.Errorf("no such file"),
want: defaultIluvatarCorexHome,
},
{
name: "fallback when not a symlink",
ixsmi: "/usr/local/bin/ixsmi",
resolved: "/usr/local/bin/ixsmi",
want: defaultIluvatarCorexHome,
},
{
name: "fallback on empty result",
ixsmi: "/usr/local/bin/ixsmi",
resolved: "",
want: defaultIluvatarCorexHome,
},
{
name: "fallback on empty input",
ixsmi: "",
want: defaultIluvatarCorexHome,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
readlink := func(string) (string, error) {
return c.resolved, c.readErr
}
if got := iluvatarCorexHomeWithReadlink(c.ixsmi, readlink); got != c.want {
t.Errorf("iluvatarCorexHomeWithReadlink(%q) = %q, expected %q", c.ixsmi, got, c.want)
}
})
}
}
pkg/hostman/options/options.go
+1 -2
View File
@@ -290,8 +290,7 @@ type SHostOptions struct {
HygonVgpuCacheDir string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"`
EnableContainerIluvatarGPU bool `help:"enable container iluvatar gpu" default:"true"`
IluvatarCorexHome string `help:"iluvatar corex home" default:"/usr/local/corex-4.4.0"`
IluvatarIxsmiPath string `help:"iluvatar ixsmi path" default:"/usr/local/corex-4.4.0/bin/ixsmi"`
IluvatarIxsmiPath string `help:"iluvatar ixsmi path" default:"/usr/local/bin/ixsmi"`
EnableContainerTHeadPPU bool `help:"enable container t-head ppu" default:"true"`
THeadPpuSdkHome string `help:"t-head ppu sdk home" default:"/usr/local/PPU_SDK"`
pkg/util/ansible/playbook.go
+5 -2
View File
@@ -103,10 +103,13 @@ func (pb *Playbook) Running() bool {
func (pb *Playbook) Run(ctx context.Context) error {
errs := make([]error, 0, len(pb.PrivateKeys))
for _, privateKey := range pb.PrivateKeys {
for i, privateKey := range pb.PrivateKeys {
err := pb.runOnce(ctx, privateKey)
if err != nil {
errs = append(errs, err)
if i != len(pb.PrivateKeys)-1 {
pb.state = pbStateInit
}
} else {
return nil
}
@@ -203,7 +206,7 @@ func (pb *Playbook) runOnce(ctx context.Context, privateKey string) error {
"all",
}
if privateKey != "" {
args = append(args, "--private-key", privateKey)
args = append(args, "--private-key", privateKeyFile)
}
cmd := exec.CommandContext(ctx, "ansible", args...)
cmd.Dir = pb.tmpdir