opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-19 10:46:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(host): support ascend npu hami (#25235)

Browse Source
This commit is contained in:
wanyaoqi
2026-07-28 10:55:37 +08:00
committed by GitHub
parent 9fbf557c31
commit 285a7ff633
7 changed files with 204 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/compute/container.go
+1
View File
@@ -42,6 +42,7 @@ const (
CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE"
CONTAINER_DEV_NVIDIA_HAMI = "NVIDIA_HAMI"
CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU"
CONTAINER_DEV_ASCEND_NPU_HAMI = "ASCEND_NPU_HAMI"
CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU"
)
pkg/hostman/hostinfo/hostinfo.go
+13 -12
View File
@@ -2324,18 +2324,19 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
return nil, err
}
probeOpts := &isolated_device.SIsolatedDeviceProbeOptions{
SkipGPUs: options.HostOptions.DisableGPU,
SkipUSBs: options.HostOptions.DisableUSB,
SkipCustomDevs: options.HostOptions.DisableCustomDevice,
EnableCudaHAMI: options.HostOptions.EnableCudaHAMI,
EnableCudaMps: options.HostOptions.EnableCudaMPS,
EnableContainerNPU: options.HostOptions.EnableContainerAscendNPU,
EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist,
SriovNics: sriovNics,
OvsOffloadNics: offloadNics,
NvmePciDisks: options.HostOptions.PTNVMEConfigs,
AmdVgpuPFs: options.HostOptions.AMDVgpuPFs,
NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs,
SkipGPUs: options.HostOptions.DisableGPU,
SkipUSBs: options.HostOptions.DisableUSB,
SkipCustomDevs: options.HostOptions.DisableCustomDevice,
EnableCudaHAMI: options.HostOptions.EnableCudaHAMI,
EnableCudaMps: options.HostOptions.EnableCudaMPS,
EnableContainerAscendNpu: options.HostOptions.EnableContainerAscendNPU,
EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami,
EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist,
SriovNics: sriovNics,
OvsOffloadNics: offloadNics,
NvmePciDisks: options.HostOptions.PTNVMEConfigs,
AmdVgpuPFs: options.HostOptions.AMDVgpuPFs,
NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs,
}
h.IsolatedDeviceMan.ProbePCIDevices(probeOpts)
pkg/hostman/isolated_device/container_device.go
+1
View File
@@ -46,6 +46,7 @@ const (
ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE
ContainerDeviceTypeNvidiaHAMI ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI
ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
ContainerDeviceTypeAscendNpuHami ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI
ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
)
pkg/hostman/isolated_device/container_device/ascend_npu.go
+43 -76
View File
@@ -16,6 +16,7 @@ package container_device
import (
"fmt"
"regexp"
"strconv"
"strings"
@@ -37,6 +38,19 @@ func init() {
type ascendNPUManager struct{}
func extractPartitionNumber(device string) (int, error) {
re := regexp.MustCompile(`(\d+)$`)
matches := re.FindStringSubmatch(device)
if len(matches) < 2 {
return 0, fmt.Errorf("no partition number found in %s", device)
}
num, err := strconv.Atoi(matches[1])
if err != nil {
return 0, err
}
return num, nil
}
func (m *ascendNPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
npus := []string{}
for _, dev := range devs {
@@ -48,59 +62,22 @@ func (m *ascendNPUManager) GetContainerExtraConfigures(devs []*hostapi.Container
if _, ok := devMan.(*ascendNPUManager); !ok {
continue
}
npus = append(npus, dev.IsolatedDevice.Path)
idx, err := extractPartitionNumber(dev.IsolatedDevice.Path)
if err != nil {
npus = append(npus, strconv.Itoa(idx))
}
}
if len(npus) == 0 {
return nil, nil
}
var (
ASCEND_TOOLKIT_HOME = "/usr/local/Ascend/ascend-toolkit/latest"
LD_LIBRARY_PATH = fmt.Sprintf("/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:"+
"%s/lib64:%s/lib64/plugin/opskernel:%s/lib64/plugin/nnengine", ASCEND_TOOLKIT_HOME, ASCEND_TOOLKIT_HOME, ASCEND_TOOLKIT_HOME)
ASCEND_AICPU_PATH = ASCEND_TOOLKIT_HOME
ASCEND_OPP_PATH = fmt.Sprintf("%s/opp", ASCEND_TOOLKIT_HOME)
TOOLCHAIN_HOME = fmt.Sprintf("%s/toolkit", ASCEND_TOOLKIT_HOME)
ASCEND_HOME_PATH = ASCEND_AICPU_PATH
)
return []*runtimeapi.KeyValue{
{
Key: "ASCEND_TOOLKIT_HOME",
Value: ASCEND_TOOLKIT_HOME,
}, {
Key: "LD_LIBRARY_PATH",
Value: LD_LIBRARY_PATH,
}, {
Key: "ASCEND_AICPU_PATH",
Value: ASCEND_AICPU_PATH,
}, {
Key: "ASCEND_OPP_PATH",
Value: ASCEND_OPP_PATH,
}, {
Key: "TOOLCHAIN_HOME",
Value: TOOLCHAIN_HOME,
}, {
Key: "ASCEND_HOME_PATH",
Value: ASCEND_HOME_PATH,
},
}, []*runtimeapi.Mount{
{
ContainerPath: "/usr/local/Ascend",
HostPath: "/usr/local/Ascend",
Readonly: true,
},
{
ContainerPath: "/usr/local/dcmi",
HostPath: "/usr/local/dcmi",
Readonly: true,
},
{
ContainerPath: "/usr/local/bin/npu-smi",
HostPath: "/usr/local/bin/npu-smi",
Readonly: true,
},
}
{
Key: "ASCEND_VISIBLE_DEVICES",
Value: strings.Join(npus, ","),
},
}, nil
}
func newAscendNPUManager() *ascendNPUManager {
@@ -108,7 +85,7 @@ func newAscendNPUManager() *ascendNPUManager {
}
func (m *ascendNPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return m.getAscendNpus()
return getAscendNpus(m, computeapi.DEVICE_SHARING_MODE_UNLIMITED)
}
func (m *ascendNPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
@@ -116,29 +93,7 @@ func (m *ascendNPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]i
}
func (m *ascendNPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
return []*runtimeapi.Device{
&runtimeapi.Device{
ContainerPath: dev.IsolatedDevice.Path,
HostPath: dev.IsolatedDevice.Path,
Permissions: "rwm",
},
}, []*runtimeapi.Device{
&runtimeapi.Device{
ContainerPath: "/dev/davinci_manager",
HostPath: "/dev/davinci_manager",
Permissions: "rwm",
},
&runtimeapi.Device{
ContainerPath: "/dev/devmm_svm",
HostPath: "/dev/devmm_svm",
Permissions: "rwm",
},
&runtimeapi.Device{
ContainerPath: "/dev/hisi_hdc",
HostPath: "/dev/hisi_hdc",
Permissions: "rwm",
},
}, nil
return nil, nil, nil
}
func (m *ascendNPUManager) GetRegisterType() isolated_device.ContainerDeviceType {
@@ -154,16 +109,21 @@ func (m *ascendNPUManager) GetSharingMode() string {
}
type ascnedNPU struct {
manager *ascendNPUManager
manager isolated_device.IContainerDeviceManager
*BaseDevice
memorySize int
}
func (dev *ascnedNPU) GetMemorySize() int {
return dev.memorySize
}
func (dev *ascnedNPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager {
return dev.manager
}
func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) {
func getAscendNpus(m isolated_device.IContainerDeviceManager, sharingMode string) ([]isolated_device.IDevice, error) {
devs := make([]isolated_device.IDevice, 0)
// Show all device's topology information
out, err := procutils.NewRemoteCommandAsFarAsPossible("npu-smi", "info").Output()
@@ -195,11 +155,17 @@ func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) {
devPath := fmt.Sprintf("/dev/davinci%d", npuId)
fileds2 := strings.Fields(lines[i+1])
if len(fileds2) < 4 {
return nil, errors.Errorf("failed parse npu-smi unknonw chip line get busid")
if len(fileds2) < 12 {
return nil, errors.Errorf("failed parse npu-smi unknonw chip line get busid, memory size")
}
log.Debugf("fileds2 %v", fileds2)
busID := fileds2[3]
hbmMemStr := fileds2[11]
hbmMem, err := strconv.Atoi(strings.TrimSpace(hbmMemStr))
if err != nil {
return nil, errors.Errorf("failed parse npu-smi unknonw chip line get hbm memory size %s: %s. break", hbmMemStr, err)
}
pciOutput, err := isolated_device.GetPCIStrByAddr(busID)
if err != nil {
return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", busID)
@@ -207,9 +173,10 @@ func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) {
dev := isolated_device.NewPCIDevice2(pciOutput[0])
npuDev := &ascnedNPU{
manager: m,
BaseDevice: NewBaseDevice(dev, computeapi.NPU_TYPE, devPath, computeapi.DEVICE_SHARING_MODE_UNLIMITED, 1),
BaseDevice: NewBaseDevice(dev, computeapi.NPU_TYPE, devPath, sharingMode, 1),
}
npuDev.SetModelName(npuName)
npuDev.memorySize = hbmMem
devs = append(devs, npuDev)
}
pkg/hostman/isolated_device/container_device/ascend_npu_hami.go
+129
View File
@@ -0,0 +1,129 @@
package container_device
import (
"fmt"
"os"
"strconv"
"strings"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/hostinfo"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
"yunion.io/x/onecloud/pkg/hostman/options"
fileutils "yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newAscendNPUHamiManager())
}
type ascendNPUHamiManager struct {
*ascendNPUManager
}
func newAscendNPUHamiManager() *ascendNPUHamiManager {
return &ascendNPUHamiManager{}
}
func (m *ascendNPUHamiManager) GetRegisterType() isolated_device.ContainerDeviceType {
return isolated_device.ContainerDeviceTypeAscendNpuHami
}
func (m *ascendNPUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
npus := []string{}
memoryLimit := ""
smLimit := ""
for _, dev := range devs {
if dev.IsolatedDevice == nil {
continue
}
iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
devMan := iDev.GetContainerDeviceManager()
if _, ok := devMan.(*ascendNPUHamiManager); !ok {
continue
}
idx, err := extractPartitionNumber(dev.IsolatedDevice.Path)
if err != nil {
log.Errorf("failed to extract partition number %s: %s", dev.IsolatedDevice.Path, err)
}
npus = append(npus, strconv.Itoa(idx))
if memoryLimit == "" {
memoryLimit = strconv.Itoa(dev.IsolatedDevice.MemoryLimit)
}
if smLimit == "" && dev.IsolatedDevice.SmUtilLimit > 0 {
smLimit = fmt.Sprintf("%d", dev.IsolatedDevice.SmUtilLimit)
}
}
if len(npus) == 0 {
return nil, nil
}
if !fileutils.Exists(options.HostOptions.AscendNpuHamiShmPath) {
err := os.MkdirAll(options.HostOptions.AscendNpuHamiShmPath, 0755)
if err != nil {
log.Errorf("failed to create shm dir %s: %s", options.HostOptions.AscendNpuHamiShmPath, err)
}
}
retEnvs := []*runtimeapi.KeyValue{
{
Key: "ASCEND_VISIBLE_DEVICES",
Value: strings.Join(npus, ","),
},
{
Key: "LD_PRELOAD",
Value: options.HostOptions.AscendNpuHamiLibvnpuPath,
},
{
Key: "NPU_MEM_QUOTA",
Value: memoryLimit,
},
{
Key: "NPU_GLOBAL_SHM_PATH",
Value: "/hami-shared-region/global_registry",
},
}
if len(smLimit) > 0 {
retEnvs = append(retEnvs, &runtimeapi.KeyValue{
Key: "NPU_PRIORITY",
Value: smLimit,
})
}
return retEnvs, []*runtimeapi.Mount{
{
ContainerPath: "/hami-shared-region",
HostPath: options.HostOptions.AscendNpuHamiShmPath,
Readonly: false,
},
{
ContainerPath: options.HostOptions.AscendNpuHamiLibvnpuPath,
HostPath: options.HostOptions.AscendNpuHamiLibvnpuPath,
Readonly: true,
},
}
}
func (m *ascendNPUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error) {
devs, err := getAscendNpus(m, computeapi.DEVICE_SHARING_MODE_HAMI)
if err != nil {
return nil, err
}
for i := range devs {
devPath := devs[i].GetDevicePath()
idx, err := extractPartitionNumber(devPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to extract partition number %s", devPath)
}
out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", fmt.Sprintf("echo y | npu-smi set -t device-share -i %d -d 1", idx)).Output()
if err != nil {
return nil, errors.Wrapf(err, "failed to set device-share %s: %s", devPath, out)
}
}
return devs, nil
}
pkg/hostman/isolated_device/isolated_device.go
+13 -9
View File
@@ -267,14 +267,17 @@ func (man *isolatedDeviceManager) probeContainerNvidiaGPUs(enableCudaHAMI, enabl
}
}
func (man *isolatedDeviceManager) probeContainerAscendNPUs(enable bool) {
if !enable {
func (man *isolatedDeviceManager) probeContainerAscendNPUs(enable, enableHami bool) {
devType := ContainerDeviceTypeAscendNpu
if enableHami {
devType = ContainerDeviceTypeAscendNpuHami
} else if !enable {
return
}
devman, err := GetContainerDeviceManager(ContainerDeviceTypeAscendNpu)
devman, err := GetContainerDeviceManager(devType)
if err != nil {
log.Errorf("no container device manager %s found", ContainerDeviceTypeAscendNpu)
log.Errorf("no container device manager %s found", devType)
return
}
devs, err := devman.ProbeDevices()
@@ -457,10 +460,11 @@ type SIsolatedDeviceProbeOptions struct {
SkipUSBs bool
SkipCustomDevs bool
EnableCudaHAMI bool
EnableCudaMps bool
EnableContainerNPU bool
EnableWhitelist bool
EnableCudaHAMI bool
EnableCudaMps bool
EnableContainerAscendNpu bool
EnableContainerAscendNpuHAMI bool
EnableWhitelist bool
SriovNics, OvsOffloadNics []HostNic
@@ -472,7 +476,7 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti
if man.host.IsContainerHost() {
man.probeContainerDevices()
man.probeContainerNvidiaGPUs(opts.EnableCudaHAMI, opts.EnableCudaMps)
man.probeContainerAscendNPUs(opts.EnableContainerNPU)
man.probeContainerAscendNPUs(opts.EnableContainerAscendNpu, opts.EnableContainerAscendNpuHAMI)
} else {
devModels, err := man.getCustomIsolatedDeviceModels()
if err != nil {
pkg/hostman/options/options.go
+4 -1
View File
@@ -274,7 +274,10 @@ type SHostOptions struct {
SkipCheckKernelMods []string `help:"skip check kernel modules"`
EnableContainerAscendNPU bool `help:"enable container npu" default:"true"`
EnableContainerAscendNPU bool `help:"enable container npu" default:"false"`
EnableContainerAscendNPUHami bool `help:"enable container npu hami" default:"true"`
AscendNpuHamiShmPath string `help:"ascend npu hami shm path" default:"/opt/cloud/hami-shared-region"`
AscendNpuHamiLibvnpuPath string `help:"ascend npu hami libvnpu.so path" default:"/opt/cloud/hami/libvnpu.so"`
EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"`
EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"`