From 285a7ff633cb3a3d302d3a18a06cbff381c68568 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:55:37 +0800 Subject: [PATCH] feat(host): support ascend npu hami (#25235) --- pkg/apis/compute/container.go | 1 + pkg/hostman/hostinfo/hostinfo.go | 25 ++-- .../isolated_device/container_device.go | 1 + .../container_device/ascend_npu.go | 119 ++++++---------- .../container_device/ascend_npu_hami.go | 129 ++++++++++++++++++ .../isolated_device/isolated_device.go | 22 +-- pkg/hostman/options/options.go | 5 +- 7 files changed, 204 insertions(+), 98 deletions(-) create mode 100644 pkg/hostman/isolated_device/container_device/ascend_npu_hami.go diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 6e1c46b1d2..59746cab76 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -42,6 +42,7 @@ const ( CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE" CONTAINER_DEV_NVIDIA_HAMI = "NVIDIA_HAMI" CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU" + CONTAINER_DEV_ASCEND_NPU_HAMI = "ASCEND_NPU_HAMI" CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU" ) diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index fbb4ec950b..af45e9f2aa 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2324,18 +2324,19 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { return nil, err } probeOpts := &isolated_device.SIsolatedDeviceProbeOptions{ - SkipGPUs: options.HostOptions.DisableGPU, - SkipUSBs: options.HostOptions.DisableUSB, - SkipCustomDevs: options.HostOptions.DisableCustomDevice, - EnableCudaHAMI: options.HostOptions.EnableCudaHAMI, - EnableCudaMps: options.HostOptions.EnableCudaMPS, - EnableContainerNPU: options.HostOptions.EnableContainerAscendNPU, - EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, - SriovNics: sriovNics, - OvsOffloadNics: offloadNics, - NvmePciDisks: options.HostOptions.PTNVMEConfigs, - AmdVgpuPFs: options.HostOptions.AMDVgpuPFs, - NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs, + SkipGPUs: options.HostOptions.DisableGPU, + SkipUSBs: options.HostOptions.DisableUSB, + SkipCustomDevs: options.HostOptions.DisableCustomDevice, + EnableCudaHAMI: options.HostOptions.EnableCudaHAMI, + EnableCudaMps: options.HostOptions.EnableCudaMPS, + EnableContainerAscendNpu: options.HostOptions.EnableContainerAscendNPU, + EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami, + EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, + SriovNics: sriovNics, + OvsOffloadNics: offloadNics, + NvmePciDisks: options.HostOptions.PTNVMEConfigs, + AmdVgpuPFs: options.HostOptions.AMDVgpuPFs, + NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs, } h.IsolatedDeviceMan.ProbePCIDevices(probeOpts) diff --git a/pkg/hostman/isolated_device/container_device.go b/pkg/hostman/isolated_device/container_device.go index f3eb5b49f3..29da032f2b 100644 --- a/pkg/hostman/isolated_device/container_device.go +++ b/pkg/hostman/isolated_device/container_device.go @@ -46,6 +46,7 @@ const ( ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE ContainerDeviceTypeNvidiaHAMI ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU + ContainerDeviceTypeAscendNpuHami ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU ) diff --git a/pkg/hostman/isolated_device/container_device/ascend_npu.go b/pkg/hostman/isolated_device/container_device/ascend_npu.go index 1b01af6415..9b7eef4dce 100644 --- a/pkg/hostman/isolated_device/container_device/ascend_npu.go +++ b/pkg/hostman/isolated_device/container_device/ascend_npu.go @@ -16,6 +16,7 @@ package container_device import ( "fmt" + "regexp" "strconv" "strings" @@ -37,6 +38,19 @@ func init() { type ascendNPUManager struct{} +func extractPartitionNumber(device string) (int, error) { + re := regexp.MustCompile(`(\d+)$`) + matches := re.FindStringSubmatch(device) + if len(matches) < 2 { + return 0, fmt.Errorf("no partition number found in %s", device) + } + num, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, err + } + return num, nil +} + func (m *ascendNPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { npus := []string{} for _, dev := range devs { @@ -48,59 +62,22 @@ func (m *ascendNPUManager) GetContainerExtraConfigures(devs []*hostapi.Container if _, ok := devMan.(*ascendNPUManager); !ok { continue } - npus = append(npus, dev.IsolatedDevice.Path) + idx, err := extractPartitionNumber(dev.IsolatedDevice.Path) + if err != nil { + npus = append(npus, strconv.Itoa(idx)) + } + } if len(npus) == 0 { return nil, nil } - var ( - ASCEND_TOOLKIT_HOME = "/usr/local/Ascend/ascend-toolkit/latest" - LD_LIBRARY_PATH = fmt.Sprintf("/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:"+ - "%s/lib64:%s/lib64/plugin/opskernel:%s/lib64/plugin/nnengine", ASCEND_TOOLKIT_HOME, ASCEND_TOOLKIT_HOME, ASCEND_TOOLKIT_HOME) - ASCEND_AICPU_PATH = ASCEND_TOOLKIT_HOME - ASCEND_OPP_PATH = fmt.Sprintf("%s/opp", ASCEND_TOOLKIT_HOME) - TOOLCHAIN_HOME = fmt.Sprintf("%s/toolkit", ASCEND_TOOLKIT_HOME) - ASCEND_HOME_PATH = ASCEND_AICPU_PATH - ) - return []*runtimeapi.KeyValue{ - { - Key: "ASCEND_TOOLKIT_HOME", - Value: ASCEND_TOOLKIT_HOME, - }, { - Key: "LD_LIBRARY_PATH", - Value: LD_LIBRARY_PATH, - }, { - Key: "ASCEND_AICPU_PATH", - Value: ASCEND_AICPU_PATH, - }, { - Key: "ASCEND_OPP_PATH", - Value: ASCEND_OPP_PATH, - }, { - Key: "TOOLCHAIN_HOME", - Value: TOOLCHAIN_HOME, - }, { - Key: "ASCEND_HOME_PATH", - Value: ASCEND_HOME_PATH, - }, - }, []*runtimeapi.Mount{ - { - ContainerPath: "/usr/local/Ascend", - HostPath: "/usr/local/Ascend", - Readonly: true, - }, - { - ContainerPath: "/usr/local/dcmi", - HostPath: "/usr/local/dcmi", - Readonly: true, - }, - { - ContainerPath: "/usr/local/bin/npu-smi", - HostPath: "/usr/local/bin/npu-smi", - Readonly: true, - }, - } + { + Key: "ASCEND_VISIBLE_DEVICES", + Value: strings.Join(npus, ","), + }, + }, nil } func newAscendNPUManager() *ascendNPUManager { @@ -108,7 +85,7 @@ func newAscendNPUManager() *ascendNPUManager { } func (m *ascendNPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { - return m.getAscendNpus() + return getAscendNpus(m, computeapi.DEVICE_SHARING_MODE_UNLIMITED) } func (m *ascendNPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { @@ -116,29 +93,7 @@ func (m *ascendNPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]i } func (m *ascendNPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { - return []*runtimeapi.Device{ - &runtimeapi.Device{ - ContainerPath: dev.IsolatedDevice.Path, - HostPath: dev.IsolatedDevice.Path, - Permissions: "rwm", - }, - }, []*runtimeapi.Device{ - &runtimeapi.Device{ - ContainerPath: "/dev/davinci_manager", - HostPath: "/dev/davinci_manager", - Permissions: "rwm", - }, - &runtimeapi.Device{ - ContainerPath: "/dev/devmm_svm", - HostPath: "/dev/devmm_svm", - Permissions: "rwm", - }, - &runtimeapi.Device{ - ContainerPath: "/dev/hisi_hdc", - HostPath: "/dev/hisi_hdc", - Permissions: "rwm", - }, - }, nil + return nil, nil, nil } func (m *ascendNPUManager) GetRegisterType() isolated_device.ContainerDeviceType { @@ -154,16 +109,21 @@ func (m *ascendNPUManager) GetSharingMode() string { } type ascnedNPU struct { - manager *ascendNPUManager + manager isolated_device.IContainerDeviceManager *BaseDevice + memorySize int +} + +func (dev *ascnedNPU) GetMemorySize() int { + return dev.memorySize } func (dev *ascnedNPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { return dev.manager } -func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) { +func getAscendNpus(m isolated_device.IContainerDeviceManager, sharingMode string) ([]isolated_device.IDevice, error) { devs := make([]isolated_device.IDevice, 0) // Show all device's topology information out, err := procutils.NewRemoteCommandAsFarAsPossible("npu-smi", "info").Output() @@ -195,11 +155,17 @@ func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) { devPath := fmt.Sprintf("/dev/davinci%d", npuId) fileds2 := strings.Fields(lines[i+1]) - if len(fileds2) < 4 { - return nil, errors.Errorf("failed parse npu-smi unknonw chip line get busid") + if len(fileds2) < 12 { + return nil, errors.Errorf("failed parse npu-smi unknonw chip line get busid, memory size") } log.Debugf("fileds2 %v", fileds2) busID := fileds2[3] + hbmMemStr := fileds2[11] + hbmMem, err := strconv.Atoi(strings.TrimSpace(hbmMemStr)) + if err != nil { + return nil, errors.Errorf("failed parse npu-smi unknonw chip line get hbm memory size %s: %s. break", hbmMemStr, err) + } + pciOutput, err := isolated_device.GetPCIStrByAddr(busID) if err != nil { return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", busID) @@ -207,9 +173,10 @@ func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) { dev := isolated_device.NewPCIDevice2(pciOutput[0]) npuDev := &ascnedNPU{ manager: m, - BaseDevice: NewBaseDevice(dev, computeapi.NPU_TYPE, devPath, computeapi.DEVICE_SHARING_MODE_UNLIMITED, 1), + BaseDevice: NewBaseDevice(dev, computeapi.NPU_TYPE, devPath, sharingMode, 1), } npuDev.SetModelName(npuName) + npuDev.memorySize = hbmMem devs = append(devs, npuDev) } diff --git a/pkg/hostman/isolated_device/container_device/ascend_npu_hami.go b/pkg/hostman/isolated_device/container_device/ascend_npu_hami.go new file mode 100644 index 0000000000..14b20f9306 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/ascend_npu_hami.go @@ -0,0 +1,129 @@ +package container_device + +import ( + "fmt" + "os" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" + fileutils "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + isolated_device.RegisterContainerDeviceManager(newAscendNPUHamiManager()) +} + +type ascendNPUHamiManager struct { + *ascendNPUManager +} + +func newAscendNPUHamiManager() *ascendNPUHamiManager { + return &ascendNPUHamiManager{} +} + +func (m *ascendNPUHamiManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeAscendNpuHami +} + +func (m *ascendNPUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + npus := []string{} + memoryLimit := "" + smLimit := "" + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*ascendNPUHamiManager); !ok { + continue + } + idx, err := extractPartitionNumber(dev.IsolatedDevice.Path) + if err != nil { + log.Errorf("failed to extract partition number %s: %s", dev.IsolatedDevice.Path, err) + } + npus = append(npus, strconv.Itoa(idx)) + if memoryLimit == "" { + memoryLimit = strconv.Itoa(dev.IsolatedDevice.MemoryLimit) + } + if smLimit == "" && dev.IsolatedDevice.SmUtilLimit > 0 { + smLimit = fmt.Sprintf("%d", dev.IsolatedDevice.SmUtilLimit) + } + } + if len(npus) == 0 { + return nil, nil + } + + if !fileutils.Exists(options.HostOptions.AscendNpuHamiShmPath) { + err := os.MkdirAll(options.HostOptions.AscendNpuHamiShmPath, 0755) + if err != nil { + log.Errorf("failed to create shm dir %s: %s", options.HostOptions.AscendNpuHamiShmPath, err) + } + } + retEnvs := []*runtimeapi.KeyValue{ + { + Key: "ASCEND_VISIBLE_DEVICES", + Value: strings.Join(npus, ","), + }, + { + Key: "LD_PRELOAD", + Value: options.HostOptions.AscendNpuHamiLibvnpuPath, + }, + { + Key: "NPU_MEM_QUOTA", + Value: memoryLimit, + }, + { + Key: "NPU_GLOBAL_SHM_PATH", + Value: "/hami-shared-region/global_registry", + }, + } + if len(smLimit) > 0 { + retEnvs = append(retEnvs, &runtimeapi.KeyValue{ + Key: "NPU_PRIORITY", + Value: smLimit, + }) + } + return retEnvs, []*runtimeapi.Mount{ + { + ContainerPath: "/hami-shared-region", + HostPath: options.HostOptions.AscendNpuHamiShmPath, + Readonly: false, + }, + { + ContainerPath: options.HostOptions.AscendNpuHamiLibvnpuPath, + HostPath: options.HostOptions.AscendNpuHamiLibvnpuPath, + Readonly: true, + }, + } +} + +func (m *ascendNPUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error) { + devs, err := getAscendNpus(m, computeapi.DEVICE_SHARING_MODE_HAMI) + if err != nil { + return nil, err + } + for i := range devs { + devPath := devs[i].GetDevicePath() + idx, err := extractPartitionNumber(devPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to extract partition number %s", devPath) + } + out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", fmt.Sprintf("echo y | npu-smi set -t device-share -i %d -d 1", idx)).Output() + if err != nil { + return nil, errors.Wrapf(err, "failed to set device-share %s: %s", devPath, out) + } + } + return devs, nil +} diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index c2e4ec3ca2..e2df587d95 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -267,14 +267,17 @@ func (man *isolatedDeviceManager) probeContainerNvidiaGPUs(enableCudaHAMI, enabl } } -func (man *isolatedDeviceManager) probeContainerAscendNPUs(enable bool) { - if !enable { +func (man *isolatedDeviceManager) probeContainerAscendNPUs(enable, enableHami bool) { + devType := ContainerDeviceTypeAscendNpu + if enableHami { + devType = ContainerDeviceTypeAscendNpuHami + } else if !enable { return } - devman, err := GetContainerDeviceManager(ContainerDeviceTypeAscendNpu) + devman, err := GetContainerDeviceManager(devType) if err != nil { - log.Errorf("no container device manager %s found", ContainerDeviceTypeAscendNpu) + log.Errorf("no container device manager %s found", devType) return } devs, err := devman.ProbeDevices() @@ -457,10 +460,11 @@ type SIsolatedDeviceProbeOptions struct { SkipUSBs bool SkipCustomDevs bool - EnableCudaHAMI bool - EnableCudaMps bool - EnableContainerNPU bool - EnableWhitelist bool + EnableCudaHAMI bool + EnableCudaMps bool + EnableContainerAscendNpu bool + EnableContainerAscendNpuHAMI bool + EnableWhitelist bool SriovNics, OvsOffloadNics []HostNic @@ -472,7 +476,7 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti if man.host.IsContainerHost() { man.probeContainerDevices() man.probeContainerNvidiaGPUs(opts.EnableCudaHAMI, opts.EnableCudaMps) - man.probeContainerAscendNPUs(opts.EnableContainerNPU) + man.probeContainerAscendNPUs(opts.EnableContainerAscendNpu, opts.EnableContainerAscendNpuHAMI) } else { devModels, err := man.getCustomIsolatedDeviceModels() if err != nil { diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index d88223c078..a90b435930 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -274,7 +274,10 @@ type SHostOptions struct { SkipCheckKernelMods []string `help:"skip check kernel modules"` - EnableContainerAscendNPU bool `help:"enable container npu" default:"true"` + EnableContainerAscendNPU bool `help:"enable container npu" default:"false"` + EnableContainerAscendNPUHami bool `help:"enable container npu hami" default:"true"` + AscendNpuHamiShmPath string `help:"ascend npu hami shm path" default:"/opt/cloud/hami-shared-region"` + AscendNpuHamiLibvnpuPath string `help:"ascend npu hami libvnpu.so path" default:"/opt/cloud/hami/libvnpu.so"` EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"`