From a270a0652c30002b3fa082b571ea1912cd595b3d Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Fri, 24 Nov 2023 21:06:54 +0800 Subject: [PATCH] feat(host): use IsolatedDeviceModels as GPU whitelist --- pkg/cloudcommon/options/options.go | 1 + pkg/hostman/hostinfo/hostinfo.go | 4 +- pkg/hostman/isolated_device/gpu.go | 36 +++++++++++-- .../isolated_device/isolated_device.go | 54 +++++++++---------- pkg/hostman/isolated_device/pci_device.go | 16 +++++- 5 files changed, 75 insertions(+), 36 deletions(-) diff --git a/pkg/cloudcommon/options/options.go b/pkg/cloudcommon/options/options.go index 0106001fb1..1cdf84c278 100644 --- a/pkg/cloudcommon/options/options.go +++ b/pkg/cloudcommon/options/options.go @@ -149,6 +149,7 @@ type HostCommonOptions struct { EnableRemoteExecutor bool `help:"Enable remote executor" default:"false"` + EnableIsolatedDeviceWhitelist bool `help:"enable isolated device white list" default:"false"` ExecutorConnectTimeoutSeconds int `help:"executor client connection timeout in seconds, default is 30" default:"30"` ImageDeployDriver string `help:"Image deploy driver" default:"qemu-kvm" choices:"qemu-kvm|nbd|libguestfs"` } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index e4c4a1b212..0fc7799d85 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2057,6 +2057,8 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { } } + enableDevWhitelist := options.HostOptions.EnableIsolatedDeviceWhitelist + offloadNics, err := h.getNicsOvsOffloadInterfaces(options.HostOptions.OvsOffloadNics) if err != nil { return nil, err @@ -2065,7 +2067,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { h.IsolatedDeviceMan.ProbePCIDevices( options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, options.HostOptions.DisableCustomDevice, sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs, options.HostOptions.AMDVgpuPFs, options.HostOptions.NVIDIAVgpuPFs, - ) + enableDevWhitelist) objs, err := h.getRemoteIsolatedDevices() if err != nil { diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index 9e10673384..e579461ff4 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -66,7 +66,16 @@ const ( DEFAULT_CPU_CMD = "host,kvm=off" ) -func getPassthroughGPUs(filteredAddrs []string) ([]*PCIDevice, error, []error) { +func isInWhitelistModels(models []IsolatedDeviceModel, dev *PCIDevice) bool { + for _, model := range models { + if model.VendorId == dev.VendorId && model.DeviceId == dev.DeviceId { + return true + } + } + return false +} + +func getPassthroughGPUs(filteredAddrs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) ([]*PCIDevice, error, []error) { lines, err := getGPUPCIStr() if err != nil { return nil, err, nil @@ -74,8 +83,11 @@ func getPassthroughGPUs(filteredAddrs []string) ([]*PCIDevice, error, []error) { warns := make([]error, 0) devs := []*PCIDevice{} - log.Infof("filter address %v", filteredAddrs) + log.Infof("filter address %v, enableWhiteList: %v", filteredAddrs, enableWhitelist) for _, line := range lines { + if len(line) == 0 { + continue + } dev := NewPCIDevice2(line) if utils.IsInStringArray(dev.Addr, filteredAddrs) { continue @@ -83,6 +95,12 @@ func getPassthroughGPUs(filteredAddrs []string) ([]*PCIDevice, error, []error) { if !utils.IsInArray(dev.ClassCode, GpuClassCodes) { continue } + if enableWhitelist { + if !isInWhitelistModels(whitelistModels, dev) { + log.Infof("skip add device %s cause of not in isolated_device_models", dev.String()) + continue + } + } if err := dev.checkSameIOMMUGroupDevice(); err != nil { warns = append(warns, errors.Wrapf(err, "get dev %s iommu group devices", dev.Addr)) continue @@ -143,6 +161,9 @@ type PCIDevice struct { } func NewPCIDevice(line string) (*PCIDevice, error) { + if len(line) == 0 { + return nil, errors.Errorf("input line is empty") + } dev := NewPCIDevice2(line) if err := dev.checkSameIOMMUGroupDevice(); err != nil { return nil, err @@ -156,7 +177,7 @@ func NewPCIDevice(line string) (*PCIDevice, error) { func NewPCIDevice2(line string) *PCIDevice { dev := parseLspci(line) if err := dev.fillPCIEInfo(); err != nil { - log.Warningf("fillPCIEInfo for device: %s, error: %v", dev.String(), err) + log.Warningf("fillPCIEInfo for line: %q, device: %s, error: %v", line, dev.String(), err) } return dev } @@ -384,6 +405,9 @@ func (d *PCIDevice) bindDriver() error { } func (d *PCIDevice) fillPCIEInfo() error { + if d.Addr == "" { + return errors.Errorf("device address is empty: %s", d.String()) + } cmd := fmt.Sprintf("lspci -vvv -s %s", d.Addr) lines, err := bashOutput(cmd) if err != nil { @@ -537,7 +561,11 @@ func detectPCIDevByAddrWithoutIOMMUGroup(addr string) (*PCIDevice, error) { if err != nil { return nil, err } - return NewPCIDevice2(strings.Join(ret, "")), nil + line := strings.Join(ret, "") + if line == "" { + return nil, nil + } + return NewPCIDevice2(line), nil } func getDeviceCmd(dev IDevice, index int) string { diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 7d5b976859..9a643a51d4 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -110,7 +110,7 @@ type IsolatedDeviceManager interface { GetDevices() []IDevice GetDeviceByIdent(vendorDevId, addr, mdevId string) IDevice GetDeviceByAddr(addr string) IDevice - ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string) + ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) StartDetachTask() BatchCustomProbe() AppendDetachedDevice(dev *CloudDeviceInfo) @@ -137,7 +137,7 @@ func (man *isolatedDeviceManager) GetDevices() []IDevice { return man.devices } -func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string) { +func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) { if skipGPUs { return } @@ -148,10 +148,10 @@ func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgp filteredAddrs = append(filteredAddrs, man.devices[i].GetAddr()) } - gpus, err, warns := getPassthroughGPUs(filteredAddrs) + gpus, err, warns := getPassthroughGPUs(filteredAddrs, enableWhitelist, whitelistModels) if err != nil { // ignore getPassthroughGPUS error on old machines without VGA devices - log.Errorf("getPassthroughGPUS: %v", err) + log.Errorf("getPassthroughGPUS error: %v", err) man.host.AppendError(fmt.Sprintf("get passhtrough gpus %s", err.Error()), "isolated_devices", "", " ") } else { if len(warns) > 0 { @@ -166,26 +166,20 @@ func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgp } } -func (man *isolatedDeviceManager) probeCustomPCIDevs(skipCustomDevs bool) { +func (man *isolatedDeviceManager) probeCustomPCIDevs(skipCustomDevs bool, devModels []IsolatedDeviceModel, filterClassCodes []string) { if skipCustomDevs { return } - devModels, err := man.getCustomIsolatedDeviceModels() - if err != nil { - log.Errorf("get custom isolated device models %s", err.Error()) - man.host.AppendError(fmt.Sprintf("get custom isolated device models %s", err.Error()), "isolated_devices", "", "") - } else { - for _, devModel := range devModels { - devs, err := getPassthroughPCIDevs(devModel) - if err != nil { - log.Errorf("getPassthroughPCIDevs %v: %s", devModel, err) - man.host.AppendError(fmt.Sprintf("get custom passthrough pci devices %s", err.Error()), "isolated_devices", "", "") - continue - } - for i, dev := range devs { - man.devices = append(man.devices, dev) - log.Infof("Add general pci device: %d => %#v", i, dev) - } + for _, devModel := range devModels { + devs, err := getPassthroughPCIDevs(devModel, filterClassCodes) + if err != nil { + log.Errorf("getPassthroughPCIDevs %v: %s", devModel, err) + man.host.AppendError(fmt.Sprintf("get custom passthrough pci devices %s", err.Error()), "isolated_devices", "", "") + continue + } + for i, dev := range devs { + man.devices = append(man.devices, dev) + log.Infof("Add general pci device: %d => %#v", i, dev) } } } @@ -306,19 +300,21 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) { } } -func (man *isolatedDeviceManager) ProbePCIDevices( - skipGPUs, skipUSBs, skipCustomDevs bool, - sriovNics, ovsOffloadNics []HostNic, - nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, -) { +func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) { man.devices = make([]IDevice, 0) + devModels, err := man.getCustomIsolatedDeviceModels() + if err != nil { + log.Errorf("get isolated device devModels %s", err.Error()) + man.host.AppendError(fmt.Sprintf("get custom isolated device devModels %s", err.Error()), "isolated_devices", "", "") + return + } man.probeUSBs(skipUSBs) - man.probeCustomPCIDevs(skipCustomDevs) + man.probeCustomPCIDevs(skipCustomDevs, devModels, GpuClassCodes) man.probeSRIOVNics(sriovNics) man.probeOffloadNICS(ovsOffloadNics) man.probeAMDVgpus(amdVgpuPFs) man.probeNVIDIAVgpus(nvidiaVgpuPFs) - man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs) + man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs, enableWhitelist, devModels) } type IsolatedDeviceModel struct { @@ -335,7 +331,7 @@ func (man *isolatedDeviceManager) getCustomIsolatedDeviceModels() ([]IsolatedDev params.Set("scope", jsonutils.NewString("system")) res, err := modules.IsolatedDeviceModels.List(man.getSession(), jsonutils.NewDict()) if err != nil { - return nil, err + return nil, errors.Wrap(err, "list isolated_device_models from compute service") } devModels := make([]IsolatedDeviceModel, len(res.Data)) for i, obj := range res.Data { diff --git a/pkg/hostman/isolated_device/pci_device.go b/pkg/hostman/isolated_device/pci_device.go index c8a3812651..783eee5c60 100644 --- a/pkg/hostman/isolated_device/pci_device.go +++ b/pkg/hostman/isolated_device/pci_device.go @@ -17,6 +17,10 @@ package isolated_device import ( "fmt" "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" ) type sGeneralPCIDevice struct { @@ -41,7 +45,7 @@ func newGeneralPCIDevice(dev *PCIDevice, devType string) *sGeneralPCIDevice { } } -func getPassthroughPCIDevs(devModel IsolatedDeviceModel) ([]*sGeneralPCIDevice, error) { +func getPassthroughPCIDevs(devModel IsolatedDeviceModel, filteredCodes []string) ([]*sGeneralPCIDevice, error) { ret, err := bashOutput(fmt.Sprintf("lspci -d %s:%s -nnmm", devModel.VendorId, devModel.DeviceId)) if err != nil { return nil, err @@ -54,12 +58,20 @@ func getPassthroughPCIDevs(devModel IsolatedDeviceModel) ([]*sGeneralPCIDevice, } devs := []*sGeneralPCIDevice{} + errs := make([]error, 0) for _, line := range lines { dev := NewPCIDevice2(line) if dev.ModelName == "" { dev.ModelName = devModel.Model } + if utils.IsInStringArray(dev.ClassCode, filteredCodes) { + continue + } + if err := dev.checkSameIOMMUGroupDevice(); err != nil { + errs = append(errs, errors.Wrapf(err, "get dev %s iommu group devices by model: %s", dev.Addr, jsonutils.Marshal(devModel))) + continue + } devs = append(devs, newGeneralPCIDevice(dev, devModel.DevType)) } - return devs, nil + return devs, errors.NewAggregate(errs) }