opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-01 15:07:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(host,region): support vastaitech gpu for container (#20987)

Browse Source
This commit is contained in:
Zexi Li
2024-08-12 09:33:50 +08:00
committed by GitHub
parent b4861abe59
commit 33da931b74
9 changed files with 237 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/compute/container.go
+2
View File
@@ -37,6 +37,7 @@ const (
CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU"
CONTAINER_DEV_NVIDIA_MPS = "NVIDIA_MPS"
CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU"
CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU"
)
var (
@@ -44,6 +45,7 @@ var (
CONTAINER_DEV_CPH_AMD_GPU,
CONTAINER_DEV_NVIDIA_GPU,
CONTAINER_DEV_NVIDIA_MPS,
CONTAINER_DEV_VASTAITECH_GPU,
}
)
pkg/apis/compute/isolated_device_const.go
+1 -1
View File
@@ -34,7 +34,7 @@ var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VG
var VALID_CONTAINER_DEVICE_TYPES = []string{
CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA,
CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS,
CONTAINER_DEV_ASCEND_NPU,
CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU,
}
var VALID_PASSTHROUGH_TYPES = []string{
pkg/compute/models/guestdisks.go
+2 -1
View File
@@ -20,6 +20,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
"yunion.io/x/sqlchemy"
api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -215,7 +216,7 @@ func (self *SGuestdisk) GetDiskJsonDescAtHost(ctx context.Context, host *SHost,
desc.ImagePath = storagecacheimg.Path
}
}
if host.HostType == api.HOST_TYPE_HYPERVISOR {
if utils.IsInStringArray(host.HostType, []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER}) {
desc.StorageId = disk.StorageId
localpath := disk.GetPathAtHost(host)
if len(localpath) == 0 {
pkg/hostman/hostinfo/hostinfo.go
+2 -1
View File
@@ -2199,7 +2199,8 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
dev := devs[i]
eg.Go(func() error {
if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev); err != nil {
return errors.Wrapf(err, "Sync device %s", dev)
log.Errorf("Sync deviceInfo %s error: %v", dev.String(), err)
return errors.Wrapf(err, "Sync device %s", dev.String())
} else {
mtx.Lock()
updateDevs.Add(obj)
pkg/hostman/isolated_device/container_device.go
+1
View File
@@ -39,6 +39,7 @@ const (
ContainerDeviceTypeNvidiaGpu ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
ContainerDeviceTypeNvidiaMps ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS
ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
)
func GetContainerDeviceManager(t ContainerDeviceType) (IContainerDeviceManager, error) {
pkg/hostman/isolated_device/container_device/base_dev.go
+52
View File
@@ -15,6 +15,11 @@
package container_device
import (
"fmt"
"os"
"path"
"strings"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
@@ -74,3 +79,50 @@ func CheckVirtualNumber(dev *isolated_device.ContainerDevice) error {
}
return nil
}
func getGPUPCIAddr(linkPartName string) (string, error) {
if !strings.HasPrefix(linkPartName, "pci-") {
return "", errors.Errorf("wrong link name: %s", linkPartName)
}
segs := strings.Split(linkPartName, "-")
if len(segs) < 3 {
return "", errors.Errorf("%s: segments length is less than 3 after splited by -", linkPartName)
}
fullAddr := segs[1]
return fullAddr, nil
}
func newPCIGPURenderBaseDevice(devPath string, index int, devType isolated_device.ContainerDeviceType) (*BaseDevice, error) {
dir := "/dev/dri/by-path/"
entries, err := os.ReadDir(dir)
if err != nil {
return nil, errors.Wrap(err, "read dir")
}
for _, entry := range entries {
entryName := entry.Name()
fp := path.Join(dir, entryName)
linkPath, err := os.Readlink(fp)
if err != nil {
return nil, errors.Wrapf(err, "read link of %s", entry.Name())
}
linkDevPath := path.Join(dir, linkPath)
if linkDevPath == devPath {
// get pci address
if !strings.HasSuffix(entryName, "-render") {
return nil, errors.Errorf("%s isn't render device", devPath)
}
pciAddr, err := getGPUPCIAddr(entryName)
if err != nil {
return nil, errors.Wrapf(err, "get pci address of %s", devPath)
}
pciOutput, err := isolated_device.GetPCIStrByAddr(pciAddr)
if err != nil {
return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", pciAddr)
}
dev := isolated_device.NewPCIDevice2(pciOutput[0])
dev.Addr = fmt.Sprintf("%s-%d", dev.Addr, index)
return NewBaseDevice(dev, devType, devPath), nil
}
}
return nil, errors.Wrapf(errors.ErrNotFound, "%s doesn't exist in %s", devPath, dir)
}
pkg/hostman/isolated_device/container_device/cph_amd_gpu.go
+5 -47
View File
@@ -15,9 +15,6 @@
package container_device
import (
"fmt"
"os"
"path"
"strings"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -90,50 +87,11 @@ type cphAMDGPU struct {
}
func newCphAMDGPU(devPath string, index int) (*cphAMDGPU, error) {
dir := "/dev/dri/by-path/"
entries, err := os.ReadDir(dir)
dev, err := newPCIGPURenderBaseDevice(devPath, index, isolated_device.ContainerDeviceTypeCphAMDGPU)
if err != nil {
return nil, errors.Wrap(err, "read dir")
return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice")
}
for _, entry := range entries {
entryName := entry.Name()
fp := path.Join(dir, entryName)
linkPath, err := os.Readlink(fp)
if err != nil {
return nil, errors.Wrapf(err, "read link of %s", entry.Name())
}
linkDevPath := path.Join(dir, linkPath)
if linkDevPath == devPath {
// get pci address
if !strings.HasSuffix(entryName, "-render") {
return nil, errors.Errorf("%s isn't render device", devPath)
}
pciAddr, err := getCphAMDGPUPCIAddr(entryName)
if err != nil {
return nil, errors.Wrapf(err, "get pci address of %s", devPath)
}
pciOutput, err := isolated_device.GetPCIStrByAddr(pciAddr)
if err != nil {
return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", pciAddr)
}
dev := isolated_device.NewPCIDevice2(pciOutput[0])
dev.Addr = fmt.Sprintf("%s-%d", dev.Addr, index)
return &cphAMDGPU{
BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeCphAMDGPU, devPath),
}, nil
}
}
return nil, errors.Wrapf(errors.ErrNotFound, "%s doesn't exist in %s", devPath, dir)
}
func getCphAMDGPUPCIAddr(linkPartName string) (string, error) {
if !strings.HasPrefix(linkPartName, "pci-") {
return "", errors.Errorf("wrong link name: %s", linkPartName)
}
segs := strings.Split(linkPartName, "-")
if len(segs) != 3 {
return "", errors.Errorf("segments is not 3 after splited by -")
}
fullAddr := segs[1]
return fullAddr, nil
return &cphAMDGPU{
BaseDevice: dev,
}, nil
}
pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go
+1 -1
View File
@@ -40,7 +40,7 @@ func Test_getCphAMDGPUPCIAddr(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.linkPartName, func(t *testing.T) {
got, err := getCphAMDGPUPCIAddr(tt.linkPartName)
got, err := getGPUPCIAddr(tt.linkPartName)
if (err != nil) != tt.wantErr {
t.Errorf("getCphAMDGPUPCIAddr() error = %v, wantErr %v", err, tt.wantErr)
return
pkg/hostman/isolated_device/container_device/vastaitech_gpu.go
+171
View File
@@ -0,0 +1,171 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package container_device
import (
"fmt"
"strconv"
"strings"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/pkg/errors"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
fileutils "yunion.io/x/onecloud/pkg/util/fileutils2"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newVastaitechGPUManager())
}
type vastaitechGPUManager struct{}
func newVastaitechGPUManager() isolated_device.IContainerDeviceManager {
return &vastaitechGPUManager{}
}
func (v vastaitechGPUManager) GetType() isolated_device.ContainerDeviceType {
return isolated_device.ContainerDeviceTypeVastaitechGpu
}
const (
VASTAITECH_VA_CTL = "va_ctl"
VASTAITECH_VA_VIDEO = "va_video"
VASTAITECH_VACC = "vacc"
)
var vastaitechRelatedDevices = map[string]string{
VASTAITECH_VA_CTL: "/dev/va%d_ctl",
VASTAITECH_VA_VIDEO: "/dev/va_video%d",
VASTAITECH_VACC: "/dev/vacc%d",
}
func (v vastaitechGPUManager) getRelatedDevices(index int) map[string]string {
devs := make(map[string]string)
for key, devFmt := range vastaitechRelatedDevices {
devs[key] = fmt.Sprintf(devFmt, index)
}
return devs
}
func (v vastaitechGPUManager) getDriRenderPrefix() string {
return "/dev/dri/renderD"
}
func (v vastaitechGPUManager) getDriStartIndex() int {
return 128
}
func (v vastaitechGPUManager) getRelatedDeviceStartIndex(driPath string) (int, error) {
prefix := v.getDriRenderPrefix()
if !strings.HasPrefix(driPath, prefix) {
return -1, errors.Errorf("device path %q doesn't start with /dev/dri/renderD", driPath)
}
idxStr := strings.ReplaceAll(driPath, prefix, "")
driIdx, err := strconv.Atoi(idxStr)
if err != nil {
return -1, errors.Wrapf(err, "convert %s to int", idxStr)
}
idx := driIdx - v.getDriStartIndex()
if idx < 0 {
return -1, errors.Errorf("%s index is less than %d", driPath, v.getDriStartIndex())
}
return idx, nil
}
func (v vastaitechGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
idx, err := v.getRelatedDeviceStartIndex(dev.Path)
if err != nil {
return nil, errors.Wrap(err, "get related device start index")
}
// check related devices
for _, devPath := range v.getRelatedDevices(idx) {
if !fileutils.Exists(devPath) {
return nil, errors.Wrapf(errors.ErrNotFound, "related device %s not found of %s", devPath, dev.Path)
}
}
if err := CheckVirtualNumber(dev); err != nil {
return nil, err
}
gpuDevs := make([]isolated_device.IDevice, 0)
for i := 0; i < dev.VirtualNumber; i++ {
gpuDev, err := newVastaitechGPU(dev.Path, i)
if err != nil {
return nil, errors.Wrapf(err, "new CPH AMD GPU with index %d", i)
}
gpuDevs = append(gpuDevs, gpuDev)
}
return gpuDevs, nil
}
func (v vastaitechGPUManager) getCommonDevices() []*runtimeapi.Device {
vatools := "/dev/vatools"
vaSync := "/dev/va_sync"
devs := []*runtimeapi.Device{}
for _, devPath := range []string{vatools, vaSync} {
devs = append(devs, &runtimeapi.Device{
ContainerPath: devPath,
HostPath: devPath,
Permissions: "rwm",
})
}
return devs
}
func (v vastaitechGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
driHostPath := dev.IsolatedDevice.Path
idx, err := v.getRelatedDeviceStartIndex(driHostPath)
if err != nil {
return nil, nil, errors.Wrapf(err, "get related device start index by %s", driHostPath)
}
perms := "rwm"
devs := []*runtimeapi.Device{
{
HostPath: driHostPath,
ContainerPath: driHostPath,
Permissions: perms,
},
}
for _, devPath := range v.getRelatedDevices(idx) {
devs = append(devs, &runtimeapi.Device{
HostPath: devPath,
ContainerPath: devPath,
Permissions: perms,
})
}
return devs, v.getCommonDevices(), nil
}
func (v vastaitechGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return nil, nil
}
func (v vastaitechGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
return nil, nil
}
type vastaitechGPU struct {
*BaseDevice
}
func newVastaitechGPU(devPath string, index int) (*vastaitechGPU, error) {
dev, err := newPCIGPURenderBaseDevice(devPath, index, isolated_device.ContainerDeviceTypeVastaitechGpu)
if err != nil {
return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice")
}
return &vastaitechGPU{BaseDevice: dev}, nil
}