From 2858568942f27531ca513a36638f22ff09e9abd8 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 22 Apr 2025 16:45:12 +0800 Subject: [PATCH] feat: container env ref (#22449) --- cmd/container-examples/create_steam.go | 252 +++++++++++++----- pkg/apis/compute/container.go | 5 +- pkg/apis/container.go | 11 + pkg/apis/host/container.go | 11 +- .../device/isolated_device.go | 3 + .../container/device/isolated_device.go | 4 + pkg/hostman/guestman/pod.go | 1 + pkg/hostman/guestman/pod_sync_loop.go | 6 +- .../container_device/nvidia_gpu.go | 48 +++- 9 files changed, 253 insertions(+), 88 deletions(-) diff --git a/cmd/container-examples/create_steam.go b/cmd/container-examples/create_steam.go index 62d54a326e..e6a528790d 100644 --- a/cmd/container-examples/create_steam.go +++ b/cmd/container-examples/create_steam.go @@ -44,23 +44,25 @@ var ( password string region string - podNet string - podIP string - podName string - diskSizeGB int - ncpu int - mem int - wolfBasePort int - accessPort int + podNet string + podIP string + podName string + diskSizeGB int + ncpu int + mem int + basePort int + accessPort int wolfImage string steamImage string externalIP string enableLxcfs bool - gpu string - gpuEnvId string - renderNode string + gpu string + gpuModel string + gpuType string + gpuEnvId string + // renderNode string overlay string alwaysMountDriverVol bool @@ -88,14 +90,17 @@ func init() { flag.IntVar(&mem, "mem", 16, "memory in GB") flag.IntVar(&diskSizeGB, "disk-size", 10, "disk size in GB") flag.IntVar(&accessPort, "port", 20105, "moonlight access http port") - flag.StringVar(&wolfImage, "wolf-image", "registry.cn-beijing.aliyuncs.com/zexi/wolf:hook-0408.0", "wolf image") - flag.StringVar(&steamImage, "steam-image", "registry.cn-beijing.aliyuncs.com/zexi/steam:custom.3", "steam image") + // - registry.cn-beijing.aliyuncs.com/zexi/wolf:hook-0408.0: stable version + flag.StringVar(&wolfImage, "wolf-image", "registry.cn-beijing.aliyuncs.com/zexi/wolf:patch-191-0420.0", "wolf image") + flag.StringVar(&steamImage, "steam-image", "registry.cn-beijing.aliyuncs.com/zexi/steam:custom.4", "steam image") flag.StringVar(&externalIP, "eip", "", "external ip") flag.BoolVar(&enableLxcfs, "lxcfs", false, "enable lxcfs") flag.BoolVar(&alwaysMountDriverVol, "mount-driver-vol", false, "always mount driver volume") flag.StringVar(&gpu, "gpu", "", "gpu") flag.StringVar(&gpuEnvId, "gpu-env-id", "", "gpu env id") - flag.StringVar(&renderNode, "render-node", "/dev/dri/renderD128", "render node") + flag.StringVar(&gpuModel, "gpu-model", "", "gpu model") + flag.StringVar(&gpuType, "gpu-type", "", "gpu type") + // flag.StringVar(&renderNode, "render-node", "/dev/dri/renderD128", "render node") flag.StringVar(&overlay, "overlay", "", "overlay") flag.StringVar(&devs, "devs", "", "devs") flag.StringVar(&mounts, "mounts", "", "mounts") @@ -104,7 +109,7 @@ func init() { flag.BoolVar(&steamNoBigScreen, "steam-no-big-screen", false, "steam no big screen") flag.Parse() - wolfBasePort = accessPort - 5 + basePort = accessPort - 5 initAuthInfo() log.Infof("Connecting to %s as %s", authUrl, user) @@ -171,7 +176,7 @@ func getTmpSocketsHostPath(name string) string { return fmt.Sprintf("/tmp/%s/sockets", name) } -func NewPulseAudioContainer() *compute.PodContainerCreateInput { +func NewPulseAudioContainer(podName string, enableLxcfs bool) *compute.PodContainerCreateInput { return &compute.PodContainerCreateInput{ ContainerSpec: compute.ContainerSpec{ ContainerSpec: api.ContainerSpec{ @@ -230,24 +235,25 @@ func getNvidiaAppDevs(idx int) []*compute.ContainerDevice { return devs } -func NewWolfContainer(gpu string) *compute.PodContainerCreateInput { +func NewWolfContainer(i CreateInput) *compute.PodContainerCreateInput { zero := 0 - eip := podIP - if externalIP != "" { - eip = externalIP + eip := i.IP + if i.ExternalIP != "" { + eip = i.ExternalIP } envs := []*api.ContainerKeyValue{ // NewEnv("WOLF_LOG_LEVEL", "DEBUG"), - NewEnv("WOLF_BASE_PORT", fmt.Sprintf("%d", wolfBasePort)), + NewEnv("WOLF_BASE_PORT", fmt.Sprintf("%d", i.BasePort)), NewEnv("WOLF_EXTERNAL_IP", eip), NewEnv("HOST_APPS_STATE_FOLDER", "/etc/wolf"), NewEnv("XDG_RUNTIME_DIR", "/tmp/sockets"), - NewEnv("WOLF_RENDER_NODE", renderNode), + // NewEnv("WOLF_RENDER_NODE", i.RenderNode), } - if gpu == "" || wolfAllGpu { + envs = append(envs, getPortEnvs(i.BasePort)...) + if i.GPU == "" || i.WolfAllGpu { envs = append(envs, NewEnv("NVIDIA_DRIVER_VOLUME_NAME", "nvidia-driver-vol")) } - if wolfAllGpu { + if i.WolfAllGpu { envs = append(envs, NewEnv("NVIDIA_VISIBLE_DEVICES", "all"), NewEnv("NVIDIA_DRIVER_CAPABILITIES", "all")) @@ -257,20 +263,33 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput { NewHostDev(DEV_UINPUT), NewHostDev(DEV_UHID), } - if gpu == "" { - if !wolfAllGpu { + if i.GPU == "" && i.GPUModel == "" && i.GPUType == "" { + if !i.WolfAllGpu { devs = append(devs, NewHostDev(DEV_DRI)) devs = append(devs, getNvidiaNvDevs(0)...) } } else { - if !wolfAllGpu { - id0 := 0 + id0 := 0 + if !i.WolfAllGpu { devs = append(devs, &compute.ContainerDevice{ Type: api.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, IsolatedDevice: &compute.ContainerIsolatedDevice{ Index: &id0, }, }) + } else { + devs = append(devs, &compute.ContainerDevice{ + Type: api.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, + IsolatedDevice: &compute.ContainerIsolatedDevice{ + Index: &id0, + OnlyEnv: []*api.ContainerIsolatedDeviceOnlyEnv{ + { + Key: "WOLF_RENDER_NODE", + FromRenderPath: true, + }, + }, + }, + }) } } vms := []*api.ContainerVolumeMount{ @@ -321,7 +340,7 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput { HostPath: &api.ContainerVolumeMountHostPath{ Type: api.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY, //Path: "/tmp/sockets", - Path: getTmpSocketsHostPath(podName), + Path: getTmpSocketsHostPath(i.Name), }, Propagation: api.MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL, }, @@ -345,8 +364,8 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput { }, }, } - vms = append(vms, getMounts(mountList)...) - if gpu == "" { + vms = append(vms, getMounts(i.MountList)...) + if i.GPU == "" { vms = append(vms, &api.ContainerVolumeMount{ UniqueName: "nvidia-driver-vol", @@ -362,8 +381,8 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput { return &compute.PodContainerCreateInput{ ContainerSpec: compute.ContainerSpec{ ContainerSpec: api.ContainerSpec{ - EnableLxcfs: enableLxcfs, - Image: wolfImage, + EnableLxcfs: i.EnableLxcfs, + Image: i.WolfImage, ImagePullPolicy: api.ImagePullPolicyAlways, CgroupDevicesAllow: []string{ CGROUP_RULE_13, @@ -376,7 +395,7 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput { } } -func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput { +func NewAppSteamContainer(i CreateInput) *compute.PodContainerCreateInput { // TODO: 设置 ulimit 和 ipc host // --ipc host --ulimit nofile=10240:10240 zero := 0 @@ -403,11 +422,11 @@ func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput envs = append(envs, NewEnv("STEAM_STARTUP_FLAGS", "-fullscreen")) } - if gpu == "" && gpuEnvId == "" { + if i.GPU == "" && i.GPUEnvId == "" && i.GPUModel == "" && i.GPUType == "" { devs = append(devs, getNvidiaAppDevs(0)...) - } else if gpuEnvId != "" { + } else if i.GPUEnvId != "" { envs = append(envs, - NewEnv("NVIDIA_VISIBLE_DEVICES", gpuEnvId), + NewEnv("NVIDIA_VISIBLE_DEVICES", i.GPUEnvId), NewEnv("NVIDIA_DRIVER_CAPABILITIES", "all")) } else { devs = append(devs, &compute.ContainerDevice{ @@ -427,8 +446,8 @@ func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput SubDirectory: "home", }, } - if overlay != "" { - overlayParts := strings.Split(overlay, ":") + if i.Overlay != "" { + overlayParts := strings.Split(i.Overlay, ":") dataVol.Disk.Overlay = &api.ContainerVolumeMountDiskOverlay{ LowerDir: overlayParts, } @@ -461,7 +480,7 @@ func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput MountPath: "/tmp/sockets", HostPath: &api.ContainerVolumeMountHostPath{ Type: api.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY, - Path: getTmpSocketsHostPath(podName), + Path: getTmpSocketsHostPath(i.Name), }, }, // { @@ -484,9 +503,9 @@ func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput }, }, } - vols = append(vols, getMounts(mountList)...) - vols = append(vols, getMounts(appMountList)...) - if gpu == "" || alwaysMountDriverVol { + vols = append(vols, getMounts(i.MountList)...) + vols = append(vols, getMounts(i.AppMountList)...) + if i.GPU == "" || i.AlwaysMountDriverVol { vols = append(vols, &api.ContainerVolumeMount{ UniqueName: "steam-nvidia-driver-vol", @@ -503,8 +522,9 @@ func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput return &compute.PodContainerCreateInput{ ContainerSpec: compute.ContainerSpec{ ContainerSpec: api.ContainerSpec{ - EnableLxcfs: enableLxcfs, - Image: steamImage, + AlwaysRestart: true, + EnableLxcfs: i.EnableLxcfs, + Image: i.SteamImage, ImagePullPolicy: api.ImagePullPolicyAlways, Command: []string{"/opt/bin/wolf-hook", "-addr", "127.0.0.1", @@ -526,13 +546,62 @@ func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput } } -func getPortMappings() compute.GuestPortMappings { - httpsPort := wolfBasePort - httpPort := wolfBasePort + 5 - controlUDPPort := wolfBasePort + 15 - videoUDPPingPort := wolfBasePort + 116 - audioUDPPingPort := wolfBasePort + 216 - rtspTCPSetupPort := wolfBasePort + 26 +func HTTPSPort(basePort int) int { + return basePort +} + +func HTTPPort(basePort int) int { + return basePort + 5 +} + +func ControlUDPPort(basePort int) int { + return basePort + 15 +} + +func VideoUDPPingPort(basePort int) int { + return basePort + 116 +} + +func AudioUDPPingPort(basePort int) int { + return basePort + 216 +} + +func RTSPTCPSetupPort(basePort int) int { + return basePort + 26 +} + +func getPortEnvs(basePort int) []*api.ContainerKeyValue { + httpsPort := HTTPSPort(basePort) + httpPort := HTTPPort(basePort) + controlUDPPort := ControlUDPPort(basePort) + videoUDPPingPort := VideoUDPPingPort(basePort) + audioUDPPingPort := AudioUDPPingPort(basePort) + rtspTCPSetupPort := RTSPTCPSetupPort(basePort) + + newV := func(key string, port int) *api.ContainerKeyValue { + return &api.ContainerKeyValue{ + Key: key, + Value: fmt.Sprintf("%d", port), + } + } + + return []*api.ContainerKeyValue{ + newV("WOLF_HTTP_PORT", httpPort), + newV("WOLF_HTTPS_PORT", httpsPort), + newV("WOLF_CONTROL_PORT", controlUDPPort), + newV("WOLF_RTSP_SETUP_PORT", rtspTCPSetupPort), + newV("WOLF_VIDEO_PING_PORT", videoUDPPingPort), + newV("WOLF_AUDIO_PING_PORT", audioUDPPingPort), + } +} + +func getPortMappings(basePort int) compute.GuestPortMappings { + httpsPort := HTTPSPort(basePort) + httpPort := HTTPPort(basePort) + controlUDPPort := ControlUDPPort(basePort) + videoUDPPingPort := VideoUDPPingPort(basePort) + audioUDPPingPort := AudioUDPPingPort(basePort) + rtspTCPSetupPort := RTSPTCPSetupPort(basePort) return compute.GuestPortMappings{ { // HTTPS @@ -573,43 +642,69 @@ func getPortMappings() compute.GuestPortMappings { } } -func GetCreateParams(name string) *compute.ServerCreateInput { +type CreateInput struct { + Name string + BasePort int + NCPU int + MemGB int + DiskSizeGB int + Network string + IP string + GPU string + GPUType string + GPUModel string + GPUEnvId string + EnableLxcfs bool + WolfImage string + WolfAllGpu bool + MountList []string + AppMountList []string + // RenderNode string + ExternalIP string + Overlay string + AlwaysMountDriverVol bool + SteamImage string +} + +func GetCreateParams(i CreateInput) *compute.ServerCreateInput { input := &compute.ServerCreateInput{ ServerConfigs: &compute.ServerConfigs{ Hypervisor: compute.HYPERVISOR_POD, }, } - input.Name = name - input.VcpuCount = ncpu - input.VmemSize = mem * 1024 + input.Name = i.Name + input.VcpuCount = i.NCPU + input.VmemSize = i.MemGB * 1024 fv := false input.DisableDelete = &fv input.AutoStart = true input.Disks = []*compute.DiskConfig{ { - SizeMb: diskSizeGB * 1024, + SizeMb: i.DiskSizeGB * 1024, Format: "raw", Fs: "ext4", }, } net := &compute.NetworkConfig{ - Network: podNet, - Address: podIP, - PortMappings: getPortMappings(), + Network: i.Network, + Address: i.IP, + PortMappings: getPortMappings(i.BasePort), } input.Networks = []*compute.NetworkConfig{net} input.Pod = &compute.PodCreateInput{ HostIPC: true, Containers: []*compute.PodContainerCreateInput{ - NewPulseAudioContainer(), - NewWolfContainer(gpu), - NewAppSteamContainer(gpu, gpuEnvId), + NewPulseAudioContainer(i.Name, i.EnableLxcfs), + NewWolfContainer(i), + NewAppSteamContainer(i), }, } - if gpu != "" { + if i.GPU != "" || i.GPUModel != "" || i.GPUType != "" { input.IsolatedDevices = []*compute.IsolatedDeviceConfig{ { - Id: gpu, + Id: i.GPU, + DevType: i.GPUType, + Model: i.GPUModel, }, } } @@ -628,7 +723,30 @@ func getSession() *mcclient.ClientSession { func main() { s := getSession() - obj, err := modules.Servers.Create(s, jsonutils.Marshal(GetCreateParams(podName))) + input := CreateInput{ + Name: podName, + BasePort: basePort, + NCPU: ncpu, + MemGB: mem, + DiskSizeGB: diskSizeGB, + Network: podNet, + IP: podIP, + GPU: gpu, + GPUEnvId: gpuEnvId, + GPUModel: gpuModel, + GPUType: gpuType, + EnableLxcfs: enableLxcfs, + WolfImage: wolfImage, + WolfAllGpu: wolfAllGpu, + MountList: mountList, + AppMountList: appMountList, + // RenderNode: renderNode, + ExternalIP: externalIP, + Overlay: overlay, + AlwaysMountDriverVol: alwaysMountDriverVol, + SteamImage: steamImage, + } + obj, err := modules.Servers.Create(s, jsonutils.Marshal(GetCreateParams(input))) if err != nil { log.Errorf("Failed to create server: %v", err) return @@ -666,5 +784,5 @@ func main() { if accessIp == "" { accessIp = srvDetails.IPs } - log.Infof("Access URL: %s:%d , port_mappings: %s", accessIp, accessPort, getPortMappings().String()) + log.Infof("Access URL: %s:%d , port_mappings: %s", accessIp, accessPort, getPortMappings(input.BasePort).String()) } diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 940f8f6f57..ba9c76c892 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -189,8 +189,9 @@ type ContainerHostDevice struct { } type ContainerIsolatedDevice struct { - Index *int `json:"index"` - Id string `json:"id"` + Index *int `json:"index"` + Id string `json:"id"` + OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` } type ContainerDevice struct { diff --git a/pkg/apis/container.go b/pkg/apis/container.go index 32a7d425b4..c7d8d6d2d4 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -83,6 +83,17 @@ type ContainerResources struct { CpusetCloneChildren bool `json:"cpuset_clone_children"` } +type ContainerEnvRefValueType string + +const ( + ContainerEnvRefValueTypeIsolatedDevice ContainerEnvRefValueType = "isolated_device" +) + +type ContainerIsolatedDeviceOnlyEnv struct { + Key string `json:"key"` + FromRenderPath bool `json:"from_render_path"` +} + type ContainerSpec struct { // Image to use. Image string `json:"image"` diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index 3c20a55335..593f857fc5 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -79,10 +79,13 @@ type ContainerDevice struct { } type ContainerIsolatedDevice struct { - Id string `json:"id"` - Addr string `json:"addr"` - Path string `json:"path"` - DeviceType string `json:"device_type"` + Id string `json:"id"` + Addr string `json:"addr"` + Path string `json:"path"` + DeviceType string `json:"device_type"` + CardPath string `json:"card_path"` + RenderPath string `json:"render_path"` + OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` } type ContainerHostDevice struct { diff --git a/pkg/compute/container_drivers/device/isolated_device.go b/pkg/compute/container_drivers/device/isolated_device.go index d88243a580..5b2a4de86a 100644 --- a/pkg/compute/container_drivers/device/isolated_device.go +++ b/pkg/compute/container_drivers/device/isolated_device.go @@ -131,7 +131,10 @@ func (i isolatedDevice) ToHostDevice(dev *api.ContainerDevice) (*hostapi.Contain Id: isoDev.GetId(), Addr: isoDev.Addr, Path: isoDev.DevicePath, + CardPath: isoDev.CardPath, DeviceType: isoDev.DevType, + RenderPath: isoDev.RenderPath, + OnlyEnv: input.OnlyEnv, }, }, nil } diff --git a/pkg/hostman/container/device/isolated_device.go b/pkg/hostman/container/device/isolated_device.go index ba6c4e3446..85f3841418 100644 --- a/pkg/hostman/container/device/isolated_device.go +++ b/pkg/hostman/container/device/isolated_device.go @@ -59,6 +59,10 @@ func (i isolatedDevice) GetRuntimeDevices(input *hostapi.ContainerCreateInput, d } for idx := range mappedDevs { + mDev := mappedDevs[idx] + if mDev.IsolatedDevice != nil && mDev.IsolatedDevice.OnlyEnv != nil { + continue + } ctrDevs, commonDevs, err := man.NewContainerDevices(input, mappedDevs[idx]) if err != nil { return nil, errors.Wrapf(err, "NewContainerDevices with %#v", devs) diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 1c8e6b9ba3..d885bb23b6 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -1774,6 +1774,7 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie Value: env.Value, }) } + pms, err := s.GetPortMappings() if err != nil { return "", errors.Wrapf(err, "get pod port mappings") diff --git a/pkg/hostman/guestman/pod_sync_loop.go b/pkg/hostman/guestman/pod_sync_loop.go index 2f559f2e9f..03c905313d 100644 --- a/pkg/hostman/guestman/pod_sync_loop.go +++ b/pkg/hostman/guestman/pod_sync_loop.go @@ -199,7 +199,11 @@ func (m *SGuestManager) syncContainerLoopIteration(plegCh chan *pleg.PodLifecycl log.Infof("container %s exited", ctrCriId) reason = fmt.Sprintf("container %s exited", ctrCriId) } else { - reason = fmt.Sprintf("exit code of died container %s is %d", ctr.Id, ctrStatus.ExitCode) + ctrId := ctrCriId + if ctr != nil { + ctrId = ctr.Id + } + reason = fmt.Sprintf("exit code of died container %s is %d", ctrId, ctrStatus.ExitCode) } } log.Infof("sync pod %s container %s status: %s", e.Id, ctrCriId, reason) diff --git a/pkg/hostman/isolated_device/container_device/nvidia_gpu.go b/pkg/hostman/isolated_device/container_device/nvidia_gpu.go index 8283348324..7ba54fe45b 100644 --- a/pkg/hostman/isolated_device/container_device/nvidia_gpu.go +++ b/pkg/hostman/isolated_device/container_device/nvidia_gpu.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/sets" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" @@ -55,29 +56,48 @@ func (m *nvidiaGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInp func (m *nvidiaGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { gpuIds := []string{} + retEnvs := []*runtimeapi.KeyValue{} for _, dev := range devs { if dev.IsolatedDevice == nil { continue } - if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNvidiaGpu { + types := sets.NewString( + string(isolated_device.ContainerDeviceTypeNvidiaGpu), + string(isolated_device.ContainerDeviceTypeNvidiaGpuShare), + ) + if !types.Has(dev.IsolatedDevice.DeviceType) { continue } - gpuIds = append(gpuIds, dev.IsolatedDevice.Path) + if len(dev.IsolatedDevice.OnlyEnv) > 0 { + for _, oe := range dev.IsolatedDevice.OnlyEnv { + if !oe.FromRenderPath { + continue + } + retEnvs = append(retEnvs, &runtimeapi.KeyValue{ + Key: oe.Key, + Value: dev.IsolatedDevice.RenderPath, + }) + } + } else { + gpuIds = append(gpuIds, dev.IsolatedDevice.Path) + } } - if len(gpuIds) == 0 { + if len(gpuIds) == 0 && len(retEnvs) == 0 { return nil, nil } - - return []*runtimeapi.KeyValue{ - { - Key: "NVIDIA_VISIBLE_DEVICES", - Value: strings.Join(gpuIds, ","), - }, - { - Key: "NVIDIA_DRIVER_CAPABILITIES", - Value: "all", - }, - }, nil + if len(gpuIds) > 0 { + retEnvs = append(retEnvs, []*runtimeapi.KeyValue{ + { + Key: "NVIDIA_VISIBLE_DEVICES", + Value: strings.Join(gpuIds, ","), + }, + { + Key: "NVIDIA_DRIVER_CAPABILITIES", + Value: "all", + }, + }...) + } + return retEnvs, nil } type nvidiaGPU struct {