feat(host): container add nvidia gpu share device type (#22432)

pkg/apis/compute/container.go

@@ -39,6 +39,7 @@ const (
 	CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA"
 	CONTAINER_DEV_NVIDIA_GPU       = "NVIDIA_GPU"
 	CONTAINER_DEV_NVIDIA_MPS       = "NVIDIA_MPS"
+	CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE"
 	CONTAINER_DEV_ASCEND_NPU       = "ASCEND_NPU"
 	CONTAINER_DEV_VASTAITECH_GPU   = "VASTAITECH_GPU"
 )
@@ -48,10 +49,17 @@ var (
 		CONTAINER_DEV_CPH_AMD_GPU,
 		CONTAINER_DEV_NVIDIA_GPU,
 		CONTAINER_DEV_NVIDIA_MPS,
+		CONTAINER_DEV_NVIDIA_GPU_SHARE,
 		CONTAINER_DEV_VASTAITECH_GPU,
 	}
 )
 
+var NVIDIA_GPU_TYPES = []string{
+	CONTAINER_DEV_NVIDIA_GPU,
+	CONTAINER_DEV_NVIDIA_MPS,
+	CONTAINER_DEV_NVIDIA_GPU_SHARE,
+}
+
 const (
 	CONTAINER_STORAGE_LOCAL_RAW = "local_raw"
 )

pkg/apis/compute/isolated_device_const.go

@@ -33,7 +33,7 @@ var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VG
 
 var VALID_CONTAINER_DEVICE_TYPES = []string{
 	CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA,
-	CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS,
+	CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE,
 	CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU,
 }
 
@@ -50,6 +50,7 @@ var VITRUAL_DEVICE_TYPES = []string{
 	CONTAINER_DEV_NETINT_CA_ASIC,
 	CONTAINER_DEV_NVIDIA_MPS,
 	CONTAINER_DEV_ASCEND_NPU,
+	CONTAINER_DEV_NVIDIA_GPU_SHARE,
 }
 
 var ID_VENDOR_MAP = map[string]string{

pkg/apis/compute/zz_generated.model.go

@@ -1472,6 +1472,8 @@ type SIsolatedDevice struct {
 	// # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr`
 	Addr       string `json:"addr"`
 	DevicePath string `json:"device_path"`
+	// GPU card path, like /dev/dri/cardX
+	CardPath string `json:"card_path"`
 	// Is vgpu physical funcion, That means it cannot be attached to guest
 	// VGPUPhysicalFunction bool `nullable:"true" default:"false" list:"domain" create:"domain_optional"`
 	// nvidia vgpu config

pkg/compute/models/isolated_devices.go

@@ -124,6 +124,9 @@ type SIsolatedDevice struct {
 	Addr       string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
 	DevicePath string `width:"128" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"optional"`
 
+	// GPU card path, like /dev/dri/cardX
+	CardPath string `width:"128" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"optional"`
+
 	// Is vgpu physical funcion, That means it cannot be attached to guest
 	// VGPUPhysicalFunction bool `nullable:"true" default:"false" list:"domain" create:"domain_optional"`
 	// nvidia vgpu config

pkg/hostman/guestman/pod.go

@@ -35,6 +35,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/sets"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
@@ -644,8 +645,7 @@ func (s *sPodGuestInstance) GetPodContainerCriIds() []string {
 
 func (s *sPodGuestInstance) HasContainerNvidiaGpu() bool {
 	for i := range s.Desc.IsolatedDevices {
-		if s.Desc.IsolatedDevices[i].DevType == computeapi.CONTAINER_DEV_NVIDIA_MPS ||
-			s.Desc.IsolatedDevices[i].DevType == computeapi.CONTAINER_DEV_NVIDIA_GPU {
+		if utils.IsInStringArray(s.Desc.IsolatedDevices[i].DevType, computeapi.NVIDIA_GPU_TYPES) {
 			return true
 		}
 	}
@@ -1846,6 +1846,7 @@ func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.Container
 	devTypes := []isolated_device.ContainerDeviceType{
 		isolated_device.ContainerDeviceTypeNvidiaGpu,
 		isolated_device.ContainerDeviceTypeNvidiaMps,
+		isolated_device.ContainerDeviceTypeNvidiaGpuShare,
 		isolated_device.ContainerDeviceTypeAscendNpu,
 	}
 	for _, devType := range devTypes {

pkg/hostman/hostinfo/container.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
 
 	apis "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
@@ -136,7 +137,7 @@ func (h *SHostInfo) HasContainerNvidiaGpu() bool {
 	nvDevs := make([]isolated_device.IDevice, 0)
 	devs := h.IsolatedDeviceMan.GetDevices()
 	for i := range devs {
-		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_NVIDIA_GPU || devs[i].GetDeviceType() == apis.CONTAINER_DEV_NVIDIA_MPS {
+		if utils.IsInStringArray(devs[i].GetDeviceType(), apis.NVIDIA_GPU_TYPES) {
 			hasNvidiaGpus = true
 			nvDevs = append(nvDevs, devs[i])
 		}

pkg/hostman/hostinfo/hostinfo.go

@@ -2570,7 +2570,7 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 		case string(isolated_device.ContainerDeviceTypeVastaitechGpu):
 			hasVasmi = true
 			continue
-		case string(isolated_device.ContainerDeviceTypeNvidiaGpu), string(isolated_device.ContainerDeviceTypeNvidiaMps):
+		case string(isolated_device.ContainerDeviceTypeNvidiaGpu), string(isolated_device.ContainerDeviceTypeNvidiaMps), string(isolated_device.ContainerDeviceTypeNvidiaGpuShare):
 			hasNvidiasmi = true
 		}
 	}

pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go

@@ -159,7 +159,7 @@ func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string
 		podDesc := pod.GetDesc()
 		hasGpu := false
 		for i := range podDesc.IsolatedDevices {
-			if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, []string{compute.CONTAINER_DEV_NVIDIA_GPU, compute.CONTAINER_DEV_NVIDIA_MPS, compute.CONTAINER_DEV_VASTAITECH_GPU, compute.CONTAINER_DEV_CPH_AMD_GPU}) {
+			if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, compute.CONTAINER_GPU_TYPES) {
 				hasGpu = true
 				break
 			}

pkg/hostman/hostmetrics/hostmetrics.go

@@ -627,7 +627,7 @@ func NewGuestPodMonitor(
 	hasCphAmdGpu := false
 	hasVastaitechGpu := false
 	for i := range podDesc.IsolatedDevices {
-		if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, []string{compute.CONTAINER_DEV_NVIDIA_MPS, compute.CONTAINER_DEV_NVIDIA_GPU}) {
+		if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, compute.NVIDIA_GPU_TYPES) {
 			hasNvGpu = true
 		} else if podDesc.IsolatedDevices[i].DevType == compute.CONTAINER_DEV_VASTAITECH_GPU {
 			hasVastaitechGpu = true

pkg/hostman/isolated_device/container_device.go

@@ -32,14 +32,15 @@ var (
 type ContainerDeviceType string
 
 const (
-	ContainerDeviceTypeCphAMDGPU     ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU
-	ContainerDeviceTypeCphASOPBinder ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER
-	ContainerNetintCAASIC            ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC
-	ContainerNetintCAQuadra          ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA
-	ContainerDeviceTypeNvidiaGpu     ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
-	ContainerDeviceTypeNvidiaMps     ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS
-	ContainerDeviceTypeAscendNpu     ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
-	ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
+	ContainerDeviceTypeCphAMDGPU      ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU
+	ContainerDeviceTypeCphASOPBinder  ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER
+	ContainerNetintCAASIC             ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC
+	ContainerNetintCAQuadra           ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA
+	ContainerDeviceTypeNvidiaGpu      ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
+	ContainerDeviceTypeNvidiaMps      ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS
+	ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE
+	ContainerDeviceTypeAscendNpu      ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
+	ContainerDeviceTypeVastaitechGpu  ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
 )
 
 func GetContainerDeviceManager(t ContainerDeviceType) (IContainerDeviceManager, error) {

pkg/hostman/isolated_device/container_device/base_dev.go

@@ -71,6 +71,10 @@ func (c BaseDevice) GetNvidiaMpsThreadPercentage() int {
 	return -1
 }
 
+func (c BaseDevice) GetCardPath() string {
+	return ""
+}
+
 func (c BaseDevice) GetNumaNode() (int, error) {
 	if c.SBaseDevice == nil {
 		return -1, nil

pkg/hostman/isolated_device/container_device/nvidia_gpu.go

@@ -42,7 +42,7 @@ func (m *nvidiaGPUManager) GetType() isolated_device.ContainerDeviceType {
 }
 
 func (m *nvidiaGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
-	return getNvidiaGPUs()
+	return probeNvidiaGpus()
 }
 
 func (m *nvidiaGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
@@ -95,8 +95,32 @@ func (dev *nvidiaGPU) GetNvidiaDevIndex() string {
 	return dev.gpuIndex
 }
 
-func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
-	devs := make([]isolated_device.IDevice, 0)
+func probeNvidiaGpus() ([]isolated_device.IDevice, error) {
+	if nvidiaGpuUsages != nil {
+		res := make([]isolated_device.IDevice, 0)
+		for pciAddr, dev := range nvidiaGpuUsages {
+			if dev.Used {
+				continue
+			}
+			res = append(res, nvidiaGpuUsages[pciAddr].nvidiaGPU)
+		}
+		nvidiaGpuUsages = nil
+		return res, nil
+	}
+
+	devs, err := getNvidiaGPUs()
+	if err != nil {
+		return nil, err
+	}
+	res := make([]isolated_device.IDevice, 0)
+	for i := range devs {
+		res = append(res, devs[i])
+	}
+	return res, nil
+}
+
+func getNvidiaGPUs() ([]*nvidiaGPU, error) {
+	devs := make([]*nvidiaGPU, 0)
 	// nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id --format=csv
 	// uuid, name, pci.bus_id
 	// GPU-bc1a3bb9-55cb-8c52-c374-4f8b4f388a20, NVIDIA A800-SXM4-80GB, 00000000:10:00.0
@@ -137,7 +161,6 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
 			gpuIndex:   index,
 		}
 		gpuDev.SetModelName(gpuName)
-
 		devs = append(devs, gpuDev)
 	}
 	if len(devs) == 0 {

pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go

@@ -0,0 +1,124 @@
+package container_device
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+)
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newNvidiaGPUShareManager())
+}
+
+type nvidiaGPUShareManager struct {
+	nvidiaGPUManager
+}
+
+func newNvidiaGPUShareManager() *nvidiaGPUShareManager {
+	return &nvidiaGPUShareManager{}
+}
+
+func (m *nvidiaGPUShareManager) GetType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeNvidiaGpuShare
+}
+
+func (m *nvidiaGPUShareManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *nvidiaGPUShareManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	if !strings.HasPrefix(dev.Path, "/dev/dri/renderD") {
+		return nil, errors.Errorf("device path %q doesn't start with /dev/dri/renderD", dev.Path)
+	}
+	if err := CheckVirtualNumber(dev); err != nil {
+		return nil, err
+	}
+
+	gpuDevs := make([]isolated_device.IDevice, 0)
+	for i := 0; i < dev.VirtualNumber; i++ {
+		gpuDev, err := newNvidiaGpuShare(dev.Path, i)
+		if err != nil {
+			return nil, errors.Wrapf(err, "new CPH AMD GPU with index %d", i)
+		}
+		gpuDevs = append(gpuDevs, gpuDev)
+	}
+	return gpuDevs, nil
+}
+
+type nvidiaGpuShareDev struct {
+	nvidiaGPU
+
+	CardPath string
+}
+
+func (dev *nvidiaGpuShareDev) GetCardPath() string {
+	return dev.CardPath
+}
+
+type nvidiaGpuUsage struct {
+	*nvidiaGPU
+
+	Used bool
+}
+
+var nvidiaGpuUsages map[string]*nvidiaGpuUsage = nil
+
+func getNvidiaGpuUsage() (map[string]*nvidiaGpuUsage, error) {
+	if nvidiaGpuUsages != nil {
+		return nvidiaGpuUsages, nil
+	}
+	devs, err := getNvidiaGPUs()
+	if err != nil {
+		return nil, err
+	}
+	if len(devs) == 0 {
+		return nil, nil
+	}
+	gpuUsages := map[string]*nvidiaGpuUsage{}
+	for i := range devs {
+		gpuUsages[devs[i].GetAddr()] = &nvidiaGpuUsage{
+			nvidiaGPU: devs[i],
+			Used:      false,
+		}
+	}
+	nvidiaGpuUsages = gpuUsages
+	return nvidiaGpuUsages, nil
+}
+
+func newNvidiaGpuShare(devPath string, index int) (*nvidiaGpuShareDev, error) {
+	devUsages, err := getNvidiaGpuUsage()
+	if err != nil {
+		return nil, errors.Wrap(err, "getNvidiaGpuUsage")
+	}
+
+	dev, err := newPCIGPURenderBaseDevice(devPath, index, isolated_device.ContainerDeviceTypeNvidiaGpuShare)
+	if err != nil {
+		return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice")
+	}
+	devAddr := dev.GetOriginAddr()
+	cardPath := path.Join("/dev/dri/by-path", fmt.Sprintf("pci-0000:%s-card", devAddr))
+	cardLinkPath, err := filepath.EvalSymlinks(cardPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read link of %s", cardPath)
+	}
+
+	_, ok := devUsages[devAddr]
+	if !ok {
+		return nil, errors.Errorf("newNvidiaGpuShare dev addr not found %s", devAddr)
+	}
+	devUsages[devAddr].Used = true
+
+	return &nvidiaGpuShareDev{
+		nvidiaGPU: nvidiaGPU{
+			BaseDevice: dev,
+			memSize:    devUsages[devAddr].memSize,
+			gpuIndex:   devUsages[devAddr].gpuIndex,
+		},
+		CardPath: cardLinkPath,
+	}, nil
+}

pkg/hostman/isolated_device/isolated_device.go

@@ -52,6 +52,7 @@ type CloudDeviceInfo struct {
 	IsInfinibandNic     bool                        `json:"is_infiniband_nic"`
 	NvmeSizeMB          int                         `json:"nvme_size_mb"`
 	DevicePath          string                      `json:"device_path"`
+	CardPath            string                      `json:"card_path"`
 	MpsMemoryLimit      int                         `json:"mps_memory_limit"`
 	MpsMemoryTotal      int                         `json:"mps_memory_total"`
 	MpsThreadPercentage int                         `json:"mps_thread_percentage"`
@@ -136,6 +137,7 @@ type IDevice interface {
 	// Get extra PCIE information
 	GetPCIEInfo() *api.IsolatedDevicePCIEInfo
 	GetDevicePath() string
+	GetCardPath() string
 
 	// mps infos
 	GetNvidiaMpsMemoryLimit() int
@@ -430,9 +432,9 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) {
 func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableCudaMps, enableContainerNPU, enableWhitelist bool) {
 	man.devices = make([]IDevice, 0)
 	if man.host.IsContainerHost() {
+		man.probeContainerDevices()
 		man.probeContainerNvidiaGPUs(enableCudaMps)
 		man.probeContainerAscendNPUs(enableContainerNPU)
-		man.probeContainerDevices()
 	} else {
 		devModels, err := man.getCustomIsolatedDeviceModels()
 		if err != nil {
@@ -487,6 +489,9 @@ func (man *isolatedDeviceManager) CheckDevIsNeedUpdate(dev IDevice, devInfo *Clo
 	if dev.GetDevicePath() != devInfo.DevicePath {
 		return true
 	}
+	if dev.GetCardPath() != devInfo.CardPath {
+		return true
+	}
 	if dev.GetModelName() != devInfo.Model {
 		return true
 	}
@@ -781,6 +786,10 @@ func (dev *SBaseDevice) GetNvidiaMpsThreadPercentage() int {
 	return -1
 }
 
+func (dev *SBaseDevice) GetCardPath() string {
+	return ""
+}
+
 func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	data := map[string]interface{}{
 		"dev_type":         dev.GetDeviceType(),
@@ -835,6 +844,10 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	if devPath != "" {
 		data["device_path"] = devPath
 	}
+	cardPath := dev.GetCardPath()
+	if cardPath != "" {
+		data["card_path"] = cardPath
+	}
 
 	if mpsMemTotal := dev.GetNvidiaMpsMemoryTotal(); mpsMemTotal > 0 {
 		data["mps_memory_total"] = mpsMemTotal

pkg/hostman/isolated_device/nvidia_vgpu.go

@@ -113,6 +113,10 @@ func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsThreadPercentage() int {
 	return -1
 }
 
+func (dev *sNVIDIAVgpuDevice) GetCardPath() string {
+	return ""
+}
+
 func (dev *sNVIDIAVgpuDevice) SetDeviceInfo(info CloudDeviceInfo) {
 	if len(info.Id) != 0 {
 		dev.cloudId = info.Id

pkg/hostman/options/options.go

@@ -237,7 +237,7 @@ type SHostOptions struct {
 	EnableCudaMPS        bool   `help:"enable cuda mps" default:"false"`
 	CudaMPSPipeDirectory string `help:"cuda mps pipe dir" default:"/tmp/nvidia-mps/pipe"`
 	CudaMPSLogDirectory  string `help:"cuda mps log dir" default:"/tmp/nvidia-mps/log"`
-	CudaMPSReplicas      int    `help:"cuda mps replias" default:"10"`
+	CudaMPSReplicas      int    `help:"cuda mps replicas" default:"10"`
 
 	EnableContainerAscendNPU bool `help:"enable container npu" default:"false"`
 

pkg/scheduler/algorithm/predicates/isolated_device_predicate.go

@@ -63,7 +63,7 @@ func (f *IsolatedDevicePredicate) PreExecute(ctx context.Context, u *core.Unit,
 
 func (f *IsolatedDevicePredicate) getIsolatedDeviceCountByType(getter core.CandidatePropertyGetter, devType string) int {
 	devs := getter.UnusedIsolatedDevicesByType(devType)
-	if devType != compute.CONTAINER_DEV_NVIDIA_MPS {
+	if devType != compute.CONTAINER_DEV_NVIDIA_MPS && devType != compute.CONTAINER_DEV_NVIDIA_GPU_SHARE {
 		return len(devs)
 	} else {
 		devMap := map[string]struct{}{}