From d3d2cfc229ff958c57602a0f81f07aa46a028aea Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:14:05 +0800 Subject: [PATCH] Fix/baremetal hygpu detect check class code4 (#25763) * fix(baremetal-agent): bm register add check hygpu class code ob40 * fix(baremetal-agent): attach isolated devices after update bminfo - add host-baremetal-probe-isolated-devices --- cmd/climc/shell/compute/hosts.go | 1 + pkg/baremetal/handler/handlers.go | 6 ++ pkg/baremetal/manager.go | 15 +++ pkg/baremetal/tasks/baseprepare.go | 14 ++- pkg/baremetal/tasks/bm_register.go | 3 + pkg/baremetal/tasks/devProbe.go | 91 +++++++++++++++++++ .../utils/detect_storages/detect_storages.go | 2 +- pkg/compute/models/hosts.go | 36 ++++++++ pkg/hostman/isolated_device/gpu.go | 3 + 9 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 pkg/baremetal/tasks/devProbe.go diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index c776cfaa2f..c5470120f0 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -70,6 +70,7 @@ func init() { cmd.BatchPerform("set-host-files", &compute.HostSetHostFilesOptions{}) cmd.BatchPerform("create-from-import-baremetal", &compute.HostCreateFromImportBaremetalOptions{}) cmd.BatchPerform("attach-isolated-devices", &options.BaseIdsOptions{}) + cmd.BatchPerform("baremetal-probe-isolated-devices", &options.BaseIdsOptions{}) cmd.Get("ipmi", &options.BaseIdOptions{}) cmd.Get("vnc", &options.BaseIdOptions{}) diff --git a/pkg/baremetal/handler/handlers.go b/pkg/baremetal/handler/handlers.go index c34f855198..cc3eefa560 100644 --- a/pkg/baremetal/handler/handlers.go +++ b/pkg/baremetal/handler/handlers.go @@ -82,6 +82,7 @@ func initBaremetalsHandler(app *appsrv.Application) { AddHandler(app, "POST", bmActionPrefix("ipmi-probe"), bmObjMiddleware(handleBaremetalIpmiProbe)) AddHandler(app, "POST", bmActionPrefix("cdrom"), bmObjMiddleware(handleBaremetalCdromTask)) AddHandler(app, "POST", bmActionPrefix("jnlp"), bmObjMiddleware(handleBaremetalJnlpTask)) + AddHandler(app, "POST", bmActionPrefix("probe-isolated-devices"), bmObjMiddleware(handleBaremetalDetectIsolatedDevices)) AddHandler(app, "POST", "/baremetals/validate-ipmi", handleBaremetalValidateIPMI()) // server actions handler @@ -301,3 +302,8 @@ func handleBaremetalRegister(ctx *Context, input *baremetal.BmRegisterInput) { return nil, nil }, nil) } + +func handleBaremetalDetectIsolatedDevices(ctx *Context, bm *baremetal.SBaremetalInstance) { + bm.StartDetectIsolatedDevices(ctx.UserCred(), ctx.TaskId(), ctx.Data()) + ctx.ResponseOk() +} diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index 4d036937b2..4b11e321f3 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -929,6 +929,16 @@ func (b *SBaremetalInstance) SyncStatus(ctx context.Context, status string, reas log.Infof("Update baremetal %s to status %s", b.GetId(), status) } +func (b *SBaremetalInstance) AttachIsolatedDevices() { + params := jsonutils.NewDict() + _, err := modules.Hosts.PerformAction(b.GetClientSession(), b.GetId(), "attach_isolated_devices", params) + if err != nil { + log.Errorf("Attach baremetal %s isolated devices error: %v", b.GetId(), err) + return + } + log.Infof("Attach baremetal %s isolated devices", b.GetId()) +} + func (b *SBaremetalInstance) AutoSyncAllStatus(ctx context.Context) { b.SyncAllStatus(ctx, "") } @@ -2094,6 +2104,11 @@ func (b *SBaremetalInstance) StartBaremetalCdromTask(userCred mcclient.TokenCred return nil } +func (b *SBaremetalInstance) StartDetectIsolatedDevices(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error { + b.StartNewTask(tasks.NewBaremetalIsolatedDevicesProbeTask, userCred, taskId, data) + return nil +} + func (b *SBaremetalInstance) DelayedServerReset(ctx context.Context, _ jsonutils.JSONObject) (jsonutils.JSONObject, error) { err := b.DoPXEBoot() return nil, err diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index b03d89cebd..ee8ed36fc0 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -440,6 +440,7 @@ func (task *sBaremetalPrepareTask) doUpdateBmInfo( if len(i.isolatedDevicesInfo) > 0 { err = task.sendIsolatedDevicesInfo(task.getClientSession(), i.isolatedDevicesInfo) if err != nil { + log.Errorf("send isolated devices info") return errors.Wrap(err, "send isolated devices info") } } @@ -777,10 +778,21 @@ func getIsolatedDevicesInfo(cli *ssh.Client, ip net.IP) ([]*isolated_device.PCID for _, line := range lines { if len(line) > 0 { dev := isolated_device.NewPCIDevice2(line, cli) + //log.Infof("dev addr %s, vendor %s, class code %s", dev.Addr, dev.VendorId, dev.ClassCode) if len(dev.Addr) > 0 && utils.IsInArray(dev.ClassCode, isolated_device.GpuClassCodes) && !isBootVga(cli, dev, bootVgaPath) { + log.Infof("append dev %s", dev.Addr) devs = append(devs, dev) } - if o.Options.AutoRegisterBaremetal && utils.IsInStringArray(dev.VendorId, api.HeterogeneousVendors) { + if o.Options.AutoDetectHeterogeneousDevices && utils.IsInStringArray(dev.VendorId, api.HeterogeneousVendors) { + if dev.VendorId == api.HYGON_VENDOR_ID && dev.ClassCode != isolated_device.CLASS_CODE_ASSIST { + log.Infof("skip dev %s", dev.Addr) + continue + } + if dev.VendorId == api.ILUVATAR_VENDOR_ID && dev.ClassCode != isolated_device.CLASS_CODE_PA { + log.Infof("skip dev %s", dev.Addr) + continue + } + log.Infof("append dev %s", dev.Addr) devs = append(devs, dev) } } diff --git a/pkg/baremetal/tasks/bm_register.go b/pkg/baremetal/tasks/bm_register.go index 51ad3ef02c..72e5edb727 100644 --- a/pkg/baremetal/tasks/bm_register.go +++ b/pkg/baremetal/tasks/bm_register.go @@ -209,6 +209,7 @@ func (s *sBaremetalRegisterTask) UpdateBaremetal(ctx context.Context) (string, e } s.baremetal = pxeBm.(IBaremetal) + s.baremetal.SyncStatus(ctx, "init", "Register reprepare info") return s.baremetal.GetId(), nil } @@ -229,6 +230,7 @@ func (s *sBaremetalRegisterTask) doRedfishProbe(ctx context.Context) (redfishSup func (s *sBaremetalRegisterTask) DoPrepare(ctx context.Context, cli *ssh.Client, registered bool) error { infos, err := s.prepareBaremetalInfo(cli) if err != nil { + log.Errorf("prepareBaremetalInfo failed %s", err) return err } @@ -295,6 +297,7 @@ func (s *sBaremetalRegisterTask) updateBmInfo(ctx context.Context, cli *ssh.Clie } if registered { + s.baremetal.SyncStatus(ctx, "running", "Register update info success") return nil } return s.initBaremetalServer(ctx) diff --git a/pkg/baremetal/tasks/devProbe.go b/pkg/baremetal/tasks/devProbe.go new file mode 100644 index 0000000000..1f12da0ee1 --- /dev/null +++ b/pkg/baremetal/tasks/devProbe.go @@ -0,0 +1,91 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/mcclient" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/ssh" +) + +type SBaremetalIsolatedDevicesProbeTask struct { + SBaremetalTaskBase +} + +func (self *SBaremetalIsolatedDevicesProbeTask) GetName() string { + return "SBaremetalIsolatedDevicesProbeTask" +} + +func NewBaremetalIsolatedDevicesProbeTask( + userCred mcclient.TokenCredential, + baremetal IBaremetal, + taskId string, + data jsonutils.JSONObject, +) ITask { + task := &SBaremetalIsolatedDevicesProbeTask{ + SBaremetalTaskBase: newBaremetalTaskBase(userCred, baremetal, taskId, data), + } + task.SetVirtualObject(task) + + log.Debugf("SBaremetalIsolatedDevicesProbeTask: %s", data) + + task.SetStage(task.DoProbeIsolatedDevices) + return task +} + +func (self *SBaremetalIsolatedDevicesProbeTask) DoProbeIsolatedDevices(ctx context.Context, args interface{}) error { + accessIp, _ := self.data.GetString("access_ip") + sshPort := 22 + if self.data.Contains("ssh_port") { + sshPort64, _ := self.data.Int("ssh_port") + sshPort = int(sshPort64) + } + username := "cloudroot" + if self.data.Contains("username") { + username, _ = self.data.GetString("username") + } + //passwd, _ := self.data.GetString("password") + keys := make([]string, 0) + self.data.Unmarshal(&keys, "private_key") + log.Infof("sshinfo %v %v %v %v %v", accessIp, sshPort, username, "", keys) + + var sshCli *ssh.Client + var err error + for i := range keys { + sshCli, err = ssh.NewClient(accessIp, sshPort, username, "", keys[i]) + if err != nil { + log.Errorf("failed ssh.NewClient with %v %v %v %v %v: %s", + accessIp, sshPort, username, "", keys[i], err) + } else { + break + } + } + if sshCli == nil { + return errors.Errorf("no valid ssh key") + } + + devs, err := getIsolatedDevicesInfo(sshCli, nil) + if err != nil { + log.Errorf("failed getIsolatedDevicesInfo %s", err) + return errors.Wrap(err, "getIsolatedDevicesInfo") + } + if len(devs) == 0 { + return nil + } + bmPrepare := newBaremetalPrepareTask(self.Baremetal, self.userCred) + err = bmPrepare.sendIsolatedDevicesInfo(bmPrepare.baremetal.GetClientSession(), devs) + if err != nil { + log.Errorf("send isolated devices info") + return errors.Wrap(err, "send isolated devices info") + } + params := jsonutils.NewDict() + _, err = modules.Hosts.PerformAction(self.Baremetal.GetClientSession(), self.Baremetal.GetId(), "attach-isolated-devices", params) + if err != nil { + log.Errorf("Attach baremetal %s isolated devices error: %v", self.Baremetal.GetId(), err) + return errors.Wrap(err, "attach-isolated-devices") + } + log.Infof("Attach baremetal %s isolated devices success", self.Baremetal.GetId()) + return nil +} diff --git a/pkg/baremetal/utils/detect_storages/detect_storages.go b/pkg/baremetal/utils/detect_storages/detect_storages.go index 04296c1754..94f381d9b1 100644 --- a/pkg/baremetal/utils/detect_storages/detect_storages.go +++ b/pkg/baremetal/utils/detect_storages/detect_storages.go @@ -73,7 +73,7 @@ func DetectStorageInfo(term raid.IExecTerm, wait bool) ([]*baremetal.BaremetalSt log.Infof("Get Raid drivers: %v, collecting disks info ...", raidDrivers) pcieRet, err := term.Run("/lib/mos/lsdisk --pcie") if err != nil { - return nil, nil, nil, fmt.Errorf("Fail to retrieve PCIE DISK info") + return nil, nil, nil, fmt.Errorf("Fail to retrieve PCIE DISK info: %s", err) } pcieDiskInfo := sysutils.ParsePCIEDiskInfo(pcieRet) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 8866ba81a1..83e81dffb0 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -26,6 +26,7 @@ import ( "strings" "sync" "time" + "yunion.io/x/onecloud/pkg/compute/sshkeys" "golang.org/x/sync/errgroup" v1 "k8s.io/api/core/v1" @@ -5931,6 +5932,41 @@ func (hh *SHost) PerformAttachIsolatedDevices( return nil, nil } +func (hh *SHost) PerformBaremetalProbeIsolatedDevices( + ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject, data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + if hh.HostType != api.HOST_TYPE_BAREMETAL { + return nil, httperrors.NewBadRequestError("Not support host type %s", hh.HostType) + } + guest := hh.GetBaremetalServer() + if guest == nil { + return nil, httperrors.NewBadRequestError("baremetal not created") + } + username := "cloudroot" + accessIp := hh.AccessIp + private1, _, err := sshkeys.GetSshProjectKeypair(ctx, guest.ProjectId) + if err != nil { + return nil, errors.Wrap(err, "GetSshProjectKeypair") + } + private, _, err := sshkeys.GetSshAdminKeypair(ctx) + if err != nil { + return nil, errors.Wrap(err, "GetSshAdminKeypair") + } + privateKyes := append(private, private1...) + params := jsonutils.NewDict() + params.Set("access_ip", jsonutils.NewString(accessIp)) + params.Set("username", jsonutils.NewString(username)) + params.Set("private_key", jsonutils.NewStringArray(privateKyes)) + url := fmt.Sprintf("/baremetals/%s/probe-isolated-devices", hh.Id) + header := mcclient.GetTokenHeaders(userCred) + _, err = hh.BaremetalSyncRequest(ctx, "POST", url, header, params) + if err != nil { + return nil, errors.Wrap(err, "BaremetalSyncRequest") + } + return nil, nil +} + func validateHostNetif(input api.HostNetifInput) (api.HostNetifInput, error) { mac := input.Mac if len(input.Mac) > 0 { diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index 2daca609dd..f56b3865a6 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -42,6 +42,9 @@ const ( CLASS_CODE_3D = "0302" CLASS_CODE_DISP = "0380" + + CLASS_CODE_ASSIST = "0b40" + CLASS_CODE_PA = "1200" ) var (