opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-10-31 02:38:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(host): use IsolatedDeviceModels as GPU whitelist

Browse Source
This commit is contained in:
Zexi Li
2023-12-04 15:23:08 +08:00
parent af4f1b3504
commit a270a0652c
5 changed files with 75 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cloudcommon/options/options.go
+1
View File
@@ -149,6 +149,7 @@ type HostCommonOptions struct {
EnableRemoteExecutor bool `help:"Enable remote executor" default:"false"`
EnableIsolatedDeviceWhitelist bool `help:"enable isolated device white list" default:"false"`
ExecutorConnectTimeoutSeconds int `help:"executor client connection timeout in seconds, default is 30" default:"30"`
ImageDeployDriver string `help:"Image deploy driver" default:"qemu-kvm" choices:"qemu-kvm|nbd|libguestfs"`
}
pkg/hostman/hostinfo/hostinfo.go
+3 -1
View File
@@ -2057,6 +2057,8 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
}
}
enableDevWhitelist := options.HostOptions.EnableIsolatedDeviceWhitelist
offloadNics, err := h.getNicsOvsOffloadInterfaces(options.HostOptions.OvsOffloadNics)
if err != nil {
return nil, err
@@ -2065,7 +2067,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
h.IsolatedDeviceMan.ProbePCIDevices(
options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, options.HostOptions.DisableCustomDevice,
sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs, options.HostOptions.AMDVgpuPFs, options.HostOptions.NVIDIAVgpuPFs,
)
enableDevWhitelist)
objs, err := h.getRemoteIsolatedDevices()
if err != nil {
pkg/hostman/isolated_device/gpu.go
+32 -4
View File
@@ -66,7 +66,16 @@ const (
DEFAULT_CPU_CMD = "host,kvm=off"
)
func getPassthroughGPUs(filteredAddrs []string) ([]*PCIDevice, error, []error) {
func isInWhitelistModels(models []IsolatedDeviceModel, dev *PCIDevice) bool {
for _, model := range models {
if model.VendorId == dev.VendorId && model.DeviceId == dev.DeviceId {
return true
}
}
return false
}
func getPassthroughGPUs(filteredAddrs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) ([]*PCIDevice, error, []error) {
lines, err := getGPUPCIStr()
if err != nil {
return nil, err, nil
@@ -74,8 +83,11 @@ func getPassthroughGPUs(filteredAddrs []string) ([]*PCIDevice, error, []error) {
warns := make([]error, 0)
devs := []*PCIDevice{}
log.Infof("filter address %v", filteredAddrs)
log.Infof("filter address %v, enableWhiteList: %v", filteredAddrs, enableWhitelist)
for _, line := range lines {
if len(line) == 0 {
continue
}
dev := NewPCIDevice2(line)
if utils.IsInStringArray(dev.Addr, filteredAddrs) {
continue
@@ -83,6 +95,12 @@ func getPassthroughGPUs(filteredAddrs []string) ([]*PCIDevice, error, []error) {
if !utils.IsInArray(dev.ClassCode, GpuClassCodes) {
continue
}
if enableWhitelist {
if !isInWhitelistModels(whitelistModels, dev) {
log.Infof("skip add device %s cause of not in isolated_device_models", dev.String())
continue
}
}
if err := dev.checkSameIOMMUGroupDevice(); err != nil {
warns = append(warns, errors.Wrapf(err, "get dev %s iommu group devices", dev.Addr))
continue
@@ -143,6 +161,9 @@ type PCIDevice struct {
}
func NewPCIDevice(line string) (*PCIDevice, error) {
if len(line) == 0 {
return nil, errors.Errorf("input line is empty")
}
dev := NewPCIDevice2(line)
if err := dev.checkSameIOMMUGroupDevice(); err != nil {
return nil, err
@@ -156,7 +177,7 @@ func NewPCIDevice(line string) (*PCIDevice, error) {
func NewPCIDevice2(line string) *PCIDevice {
dev := parseLspci(line)
if err := dev.fillPCIEInfo(); err != nil {
log.Warningf("fillPCIEInfo for device: %s, error: %v", dev.String(), err)
log.Warningf("fillPCIEInfo for line: %q, device: %s, error: %v", line, dev.String(), err)
}
return dev
}
@@ -384,6 +405,9 @@ func (d *PCIDevice) bindDriver() error {
}
func (d *PCIDevice) fillPCIEInfo() error {
if d.Addr == "" {
return errors.Errorf("device address is empty: %s", d.String())
}
cmd := fmt.Sprintf("lspci -vvv -s %s", d.Addr)
lines, err := bashOutput(cmd)
if err != nil {
@@ -537,7 +561,11 @@ func detectPCIDevByAddrWithoutIOMMUGroup(addr string) (*PCIDevice, error) {
if err != nil {
return nil, err
}
return NewPCIDevice2(strings.Join(ret, "")), nil
line := strings.Join(ret, "")
if line == "" {
return nil, nil
}
return NewPCIDevice2(line), nil
}
func getDeviceCmd(dev IDevice, index int) string {
pkg/hostman/isolated_device/isolated_device.go
+25 -29
View File
@@ -110,7 +110,7 @@ type IsolatedDeviceManager interface {
GetDevices() []IDevice
GetDeviceByIdent(vendorDevId, addr, mdevId string) IDevice
GetDeviceByAddr(addr string) IDevice
ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string)
ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool)
StartDetachTask()
BatchCustomProbe()
AppendDetachedDevice(dev *CloudDeviceInfo)
@@ -137,7 +137,7 @@ func (man *isolatedDeviceManager) GetDevices() []IDevice {
return man.devices
}
func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string) {
func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) {
if skipGPUs {
return
}
@@ -148,10 +148,10 @@ func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgp
filteredAddrs = append(filteredAddrs, man.devices[i].GetAddr())
}
gpus, err, warns := getPassthroughGPUs(filteredAddrs)
gpus, err, warns := getPassthroughGPUs(filteredAddrs, enableWhitelist, whitelistModels)
if err != nil {
// ignore getPassthroughGPUS error on old machines without VGA devices
log.Errorf("getPassthroughGPUS: %v", err)
log.Errorf("getPassthroughGPUS error: %v", err)
man.host.AppendError(fmt.Sprintf("get passhtrough gpus %s", err.Error()), "isolated_devices", "", " ")
} else {
if len(warns) > 0 {
@@ -166,26 +166,20 @@ func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgp
}
}
func (man *isolatedDeviceManager) probeCustomPCIDevs(skipCustomDevs bool) {
func (man *isolatedDeviceManager) probeCustomPCIDevs(skipCustomDevs bool, devModels []IsolatedDeviceModel, filterClassCodes []string) {
if skipCustomDevs {
return
}
devModels, err := man.getCustomIsolatedDeviceModels()
if err != nil {
log.Errorf("get custom isolated device models %s", err.Error())
man.host.AppendError(fmt.Sprintf("get custom isolated device models %s", err.Error()), "isolated_devices", "", "")
} else {
for _, devModel := range devModels {
devs, err := getPassthroughPCIDevs(devModel)
if err != nil {
log.Errorf("getPassthroughPCIDevs %v: %s", devModel, err)
man.host.AppendError(fmt.Sprintf("get custom passthrough pci devices %s", err.Error()), "isolated_devices", "", "")
continue
}
for i, dev := range devs {
man.devices = append(man.devices, dev)
log.Infof("Add general pci device: %d => %#v", i, dev)
}
for _, devModel := range devModels {
devs, err := getPassthroughPCIDevs(devModel, filterClassCodes)
if err != nil {
log.Errorf("getPassthroughPCIDevs %v: %s", devModel, err)
man.host.AppendError(fmt.Sprintf("get custom passthrough pci devices %s", err.Error()), "isolated_devices", "", "")
continue
}
for i, dev := range devs {
man.devices = append(man.devices, dev)
log.Infof("Add general pci device: %d => %#v", i, dev)
}
}
}
@@ -306,19 +300,21 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) {
}
}
func (man *isolatedDeviceManager) ProbePCIDevices(
skipGPUs, skipUSBs, skipCustomDevs bool,
sriovNics, ovsOffloadNics []HostNic,
nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string,
) {
func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) {
man.devices = make([]IDevice, 0)
devModels, err := man.getCustomIsolatedDeviceModels()
if err != nil {
log.Errorf("get isolated device devModels %s", err.Error())
man.host.AppendError(fmt.Sprintf("get custom isolated device devModels %s", err.Error()), "isolated_devices", "", "")
return
}
man.probeUSBs(skipUSBs)
man.probeCustomPCIDevs(skipCustomDevs)
man.probeCustomPCIDevs(skipCustomDevs, devModels, GpuClassCodes)
man.probeSRIOVNics(sriovNics)
man.probeOffloadNICS(ovsOffloadNics)
man.probeAMDVgpus(amdVgpuPFs)
man.probeNVIDIAVgpus(nvidiaVgpuPFs)
man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs)
man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs, enableWhitelist, devModels)
}
type IsolatedDeviceModel struct {
@@ -335,7 +331,7 @@ func (man *isolatedDeviceManager) getCustomIsolatedDeviceModels() ([]IsolatedDev
params.Set("scope", jsonutils.NewString("system"))
res, err := modules.IsolatedDeviceModels.List(man.getSession(), jsonutils.NewDict())
if err != nil {
return nil, err
return nil, errors.Wrap(err, "list isolated_device_models from compute service")
}
devModels := make([]IsolatedDeviceModel, len(res.Data))
for i, obj := range res.Data {
pkg/hostman/isolated_device/pci_device.go
+14 -2
View File
@@ -17,6 +17,10 @@ package isolated_device
import (
"fmt"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
)
type sGeneralPCIDevice struct {
@@ -41,7 +45,7 @@ func newGeneralPCIDevice(dev *PCIDevice, devType string) *sGeneralPCIDevice {
}
}
func getPassthroughPCIDevs(devModel IsolatedDeviceModel) ([]*sGeneralPCIDevice, error) {
func getPassthroughPCIDevs(devModel IsolatedDeviceModel, filteredCodes []string) ([]*sGeneralPCIDevice, error) {
ret, err := bashOutput(fmt.Sprintf("lspci -d %s:%s -nnmm", devModel.VendorId, devModel.DeviceId))
if err != nil {
return nil, err
@@ -54,12 +58,20 @@ func getPassthroughPCIDevs(devModel IsolatedDeviceModel) ([]*sGeneralPCIDevice,
}
devs := []*sGeneralPCIDevice{}
errs := make([]error, 0)
for _, line := range lines {
dev := NewPCIDevice2(line)
if dev.ModelName == "" {
dev.ModelName = devModel.Model
}
if utils.IsInStringArray(dev.ClassCode, filteredCodes) {
continue
}
if err := dev.checkSameIOMMUGroupDevice(); err != nil {
errs = append(errs, errors.Wrapf(err, "get dev %s iommu group devices by model: %s", dev.Addr, jsonutils.Marshal(devModel)))
continue
}
devs = append(devs, newGeneralPCIDevice(dev, devModel.DevType))
}
return devs, nil
return devs, errors.NewAggregate(errs)
}