mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-08-29 03:51:54 +08:00
feat(host): support Hygon DCU isolated device and HAMI (#25247)
Add Hygon DCU container device passthrough, HAMI vGPU sharing, vendor field/filter on isolated devices, remote path helper, and LLM SKU mapping.
This commit is contained in:
Zexi Li
@@ -44,6 +44,8 @@ const (
|
||||
CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU"
|
||||
CONTAINER_DEV_ASCEND_NPU_HAMI = "ASCEND_NPU_HAMI"
|
||||
CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU"
|
||||
CONTAINER_DEV_HYGON_DCU = "HYGON_DCU"
|
||||
CONTAINER_DEV_HYGON_DCU_HAMI = "HYGON_DCU_HAMI"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -54,6 +56,8 @@ var (
|
||||
CONTAINER_DEV_NVIDIA_HAMI,
|
||||
CONTAINER_DEV_NVIDIA_GPU_SHARE,
|
||||
CONTAINER_DEV_VASTAITECH_GPU,
|
||||
CONTAINER_DEV_HYGON_DCU,
|
||||
CONTAINER_DEV_HYGON_DCU_HAMI,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -34,6 +34,9 @@ type IsolateDeviceDetails struct {
|
||||
|
||||
SIsolatedDevice
|
||||
|
||||
// 设备厂商，由 vendor_device_id 经 ID_VENDOR_MAP 翻译
|
||||
Vendor string `json:"vendor"`
|
||||
|
||||
MemoryAllocated int
|
||||
AllocatedCount int
|
||||
|
||||
@@ -73,6 +76,9 @@ type IsolatedDeviceListInput struct {
|
||||
// 设备VENDOE编号
|
||||
VendorDeviceId []string `json:"vendor_device_id"`
|
||||
|
||||
// 设备厂商，如 HYGON / NVIDIA
|
||||
Vendor []string `json:"vendor"`
|
||||
|
||||
// NUMA节点序号
|
||||
NumaNode []uint8 `json:"numa_node"`
|
||||
|
||||
@@ -413,4 +419,7 @@ type GuestIsolatedDeviceDetails struct {
|
||||
SIsolatedDevice
|
||||
HostResourceInfo
|
||||
apis.SharableResourceBaseInfo
|
||||
|
||||
// 设备厂商，由 vendor_device_id 经 ID_VENDOR_MAP 翻译
|
||||
Vendor string `json:"vendor"`
|
||||
}
|
||||
|
||||
@@ -48,17 +48,20 @@ const (
|
||||
NVIDIA_VENDOR_ID = "10de"
|
||||
AMD_VENDOR_ID = "1002"
|
||||
VASTAITECH_VENDOR_ID = "1ec6"
|
||||
HYGON_VENDOR_ID = "1d94"
|
||||
)
|
||||
|
||||
var ID_VENDOR_MAP = map[string]string{
|
||||
NVIDIA_VENDOR_ID: "NVIDIA",
|
||||
AMD_VENDOR_ID: "AMD",
|
||||
VASTAITECH_VENDOR_ID: "VASTAITECH",
|
||||
HYGON_VENDOR_ID: "HYGON",
|
||||
}
|
||||
|
||||
var VENDOR_ID_MAP = map[string]string{
|
||||
"NVIDIA": NVIDIA_VENDOR_ID,
|
||||
"AMD": AMD_VENDOR_ID,
|
||||
"HYGON": HYGON_VENDOR_ID,
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -83,6 +86,7 @@ var GPU_TYPES = []string{
|
||||
GPU_HPC_TYPE, GPU_VGA_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE,
|
||||
CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE,
|
||||
CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_VASTAITECH_GPU,
|
||||
CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI,
|
||||
}
|
||||
|
||||
var NETINT_TYPES = []string{
|
||||
@@ -93,6 +97,7 @@ var VALID_CONTAINER_DEVICE_TYPES = []string{
|
||||
CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA,
|
||||
CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_NVIDIA_HAMI,
|
||||
CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU,
|
||||
CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI,
|
||||
}
|
||||
|
||||
var VALID_PASSTHROUGH_TYPES = []string{
|
||||
@@ -111,6 +116,8 @@ var VITRUAL_DEVICE_TYPES = []string{
|
||||
CONTAINER_DEV_ASCEND_NPU,
|
||||
CONTAINER_DEV_NVIDIA_GPU_SHARE,
|
||||
CONTAINER_DEV_VASTAITECH_GPU,
|
||||
CONTAINER_DEV_HYGON_DCU,
|
||||
CONTAINER_DEV_HYGON_DCU_HAMI,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -132,6 +132,7 @@ func (manager *SGuestIsolatedDeviceManager) FetchCustomizeColumns(
|
||||
rows[i].SIsolatedDevice = isolatedDeviceRows[i].SIsolatedDevice
|
||||
rows[i].HostResourceInfo = isolatedDeviceRows[i].HostResourceInfo
|
||||
rows[i].SharableResourceBaseInfo = isolatedDeviceRows[i].SharableResourceBaseInfo
|
||||
rows[i].Vendor = isolatedDeviceRows[i].Vendor
|
||||
}
|
||||
|
||||
return rows
|
||||
|
||||
@@ -346,6 +346,13 @@ func (manager *SIsolatedDeviceManager) ListItemFilter(
|
||||
if len(query.VendorDeviceId) > 0 {
|
||||
q = q.In("vendor_device_id", query.VendorDeviceId)
|
||||
}
|
||||
if len(query.Vendor) > 0 {
|
||||
conds := make([]sqlchemy.ICondition, 0, len(query.Vendor))
|
||||
for _, v := range query.Vendor {
|
||||
conds = append(conds, sqlchemy.Startswith(q.Field("vendor_device_id"), vendorDeviceIdPrefixForFilter(v)))
|
||||
}
|
||||
q = q.Filter(sqlchemy.OR(conds...))
|
||||
}
|
||||
if len(query.NumaNode) > 0 {
|
||||
q = q.In("numa_node", query.NumaNode)
|
||||
}
|
||||
@@ -505,6 +512,22 @@ func GetVendorByVendorDeviceId(vendorDeviceId string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func vendorDeviceIdPrefixForFilter(vendor string) string {
|
||||
return resolveVendorIdForFilter(vendor) + ":"
|
||||
}
|
||||
|
||||
func resolveVendorIdForFilter(vendor string) string {
|
||||
if id, ok := api.VENDOR_ID_MAP[vendor]; ok {
|
||||
return id
|
||||
}
|
||||
for name, id := range api.VENDOR_ID_MAP {
|
||||
if strings.EqualFold(name, vendor) {
|
||||
return id
|
||||
}
|
||||
}
|
||||
return strings.ToLower(vendor)
|
||||
}
|
||||
|
||||
func (self *SIsolatedDevice) IsGPU() bool {
|
||||
return self.DevType == api.GPU_TYPE
|
||||
}
|
||||
@@ -1502,6 +1525,7 @@ func (manager *SIsolatedDeviceManager) FetchCustomizeColumns(
|
||||
SharableResourceBaseInfo: shareRows[i],
|
||||
}
|
||||
dev := objs[i].(*SIsolatedDevice)
|
||||
rows[i].Vendor = dev.getVendor()
|
||||
if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI {
|
||||
rows[i].MemoryAllocated, _ = dev.getAllocatedMemorySize()
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetVendorByVendorDeviceId(t *testing.T) {
|
||||
assert.Equal(t, "HYGON", GetVendorByVendorDeviceId("1d94:6320"))
|
||||
assert.Equal(t, "NVIDIA", GetVendorByVendorDeviceId("10de:1c82"))
|
||||
assert.Equal(t, "AMD", GetVendorByVendorDeviceId("1002:6611"))
|
||||
assert.Equal(t, "abcd", GetVendorByVendorDeviceId("abcd:0001"))
|
||||
}
|
||||
|
||||
func TestSIsolatedDeviceGetVendor(t *testing.T) {
|
||||
dev := &SIsolatedDevice{VendorDeviceId: "1d94:6320"}
|
||||
assert.Equal(t, "HYGON", dev.getVendor())
|
||||
}
|
||||
|
||||
func TestVendorDeviceIdPrefixForFilter(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"HYGON": "1d94:",
|
||||
"NVIDIA": "10de:",
|
||||
"1d94": "1d94:",
|
||||
"hygon": "1d94:",
|
||||
"Hygon": "1d94:",
|
||||
"nvidia": "10de:",
|
||||
"1D94": "1d94:",
|
||||
}
|
||||
for vendor, want := range cases {
|
||||
assert.Equal(t, want, vendorDeviceIdPrefixForFilter(vendor), "vendor=%s", vendor)
|
||||
}
|
||||
}
|
||||
@@ -83,6 +83,7 @@ func (manager *SIsolatedDeviceResourceBaseManager) FetchCustomizeColumns(
|
||||
rows[i].HostResourceInfo = devRows[i].HostResourceInfo
|
||||
rows[i].Guest = devRows[i].Guest
|
||||
rows[i].GuestStatus = devRows[i].GuestStatus
|
||||
rows[i].Vendor = devRows[i].Vendor
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
@@ -2325,6 +2325,15 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Infof("==== probeSyncIsolatedDevices: hostType=%s isContainerHost=%v isKvmSupport=%v",
|
||||
options.HostOptions.HostType, h.IsContainerHost(), h.IsKvmSupport())
|
||||
log.Infof("==== probeSyncIsolatedDevices hygon config: enableDCU=%v enableHAMI=%v hySmiPath=%s hyhalPath=%s dtkPath=%s",
|
||||
options.HostOptions.EnableContainerHygonDCU,
|
||||
options.HostOptions.EnableContainerHygonDCUHAMI,
|
||||
options.HostOptions.HygonHySmiPath,
|
||||
options.HostOptions.HygonHyhalPath,
|
||||
options.HostOptions.HygonDtkPath,
|
||||
)
|
||||
probeOpts := &isolated_device.SIsolatedDeviceProbeOptions{
|
||||
SkipGPUs: options.HostOptions.DisableGPU,
|
||||
SkipUSBs: options.HostOptions.DisableUSB,
|
||||
@@ -2333,6 +2342,8 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
|
||||
EnableCudaMps: options.HostOptions.EnableCudaMPS,
|
||||
EnableContainerAscendNpu: options.HostOptions.EnableContainerAscendNPU,
|
||||
EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami,
|
||||
EnableContainerHygonDCU: options.HostOptions.EnableContainerHygonDCU,
|
||||
EnableContainerHygonDCUHAMI: options.HostOptions.EnableContainerHygonDCUHAMI,
|
||||
EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist,
|
||||
SriovNics: sriovNics,
|
||||
OvsOffloadNics: offloadNics,
|
||||
@@ -2374,6 +2385,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
|
||||
mtx := sync.Mutex{}
|
||||
updateDevs := jsonutils.NewArray()
|
||||
devs := h.IsolatedDeviceMan.GetDevices()
|
||||
log.Infof("==== probeSyncIsolatedDevices: local isolated devices count=%d, syncing to region", len(devs))
|
||||
for i := range devs {
|
||||
dev := devs[i]
|
||||
eg.Go(func() error {
|
||||
|
||||
@@ -48,6 +48,8 @@ const (
|
||||
ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
|
||||
ContainerDeviceTypeAscendNpuHami ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI
|
||||
ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
|
||||
ContainerDeviceTypeHygonDcu ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU
|
||||
ContainerDeviceTypeHygonDcuHami ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI
|
||||
)
|
||||
|
||||
func GetContainerDeviceManager(devType ContainerDeviceType) (IContainerDeviceManager, error) {
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package container_device
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
|
||||
hostapi "yunion.io/x/onecloud/pkg/apis/host"
|
||||
"yunion.io/x/onecloud/pkg/hostman/hostinfo"
|
||||
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
|
||||
"yunion.io/x/onecloud/pkg/hostman/options"
|
||||
"yunion.io/x/onecloud/pkg/util/procutils"
|
||||
)
|
||||
|
||||
func init() {
|
||||
isolated_device.RegisterContainerDeviceManager(newHygonDCUManager())
|
||||
}
|
||||
|
||||
type hygonDCUManager struct{}
|
||||
|
||||
func newHygonDCUManager() *hygonDCUManager {
|
||||
return &hygonDCUManager{}
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) GetRegisterType() isolated_device.ContainerDeviceType {
|
||||
return isolated_device.ContainerDeviceTypeHygonDcu
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
|
||||
return getHygonDCUs(m, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE)
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) GetDevType() string {
|
||||
return computeapi.GPU_TYPE
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) GetSharingMode() string {
|
||||
return computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
|
||||
}
|
||||
|
||||
type hygonDCU struct {
|
||||
manager isolated_device.IContainerDeviceManager
|
||||
|
||||
*BaseDevice
|
||||
gpuIndex int
|
||||
renderPath string
|
||||
memorySize int
|
||||
computeUnits int
|
||||
}
|
||||
|
||||
func (dev *hygonDCU) GetMemorySize() int {
|
||||
return dev.memorySize
|
||||
}
|
||||
|
||||
func (dev *hygonDCU) GetIndex() int {
|
||||
return dev.gpuIndex
|
||||
}
|
||||
|
||||
func (dev *hygonDCU) GetRenderPath() string {
|
||||
return dev.renderPath
|
||||
}
|
||||
|
||||
func (dev *hygonDCU) GetComputeUnits() int {
|
||||
return dev.computeUnits
|
||||
}
|
||||
|
||||
func (dev *hygonDCU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager {
|
||||
return dev.manager
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||
indices := []string{}
|
||||
for _, dev := range devs {
|
||||
if dev.IsolatedDevice == nil {
|
||||
continue
|
||||
}
|
||||
iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
|
||||
if iDev == nil {
|
||||
continue
|
||||
}
|
||||
devMan := iDev.GetContainerDeviceManager()
|
||||
if _, ok := devMan.(*hygonDCUManager); !ok {
|
||||
continue
|
||||
}
|
||||
indices = append(indices, dev.IsolatedDevice.Path)
|
||||
}
|
||||
if len(indices) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return m.buildHygonExtraConfigures(indices)
|
||||
}
|
||||
|
||||
func buildHygonRuntimeMounts() []*runtimeapi.Mount {
|
||||
hyhalPath := options.HostOptions.HygonHyhalPath
|
||||
dtkPath := options.HostOptions.HygonDtkPath
|
||||
mounts := []*runtimeapi.Mount{}
|
||||
if hygonPathExists(hyhalPath) {
|
||||
mounts = append(mounts, &runtimeapi.Mount{
|
||||
ContainerPath: hyhalPath,
|
||||
HostPath: hyhalPath,
|
||||
Readonly: true,
|
||||
})
|
||||
}
|
||||
if hygonPathExists(dtkPath) {
|
||||
mounts = append(mounts, &runtimeapi.Mount{
|
||||
ContainerPath: dtkPath,
|
||||
HostPath: dtkPath,
|
||||
Readonly: true,
|
||||
})
|
||||
}
|
||||
return mounts
|
||||
}
|
||||
|
||||
func buildHygonRuntimeEnvs(indices []string) []*runtimeapi.KeyValue {
|
||||
hyhalPath := options.HostOptions.HygonHyhalPath
|
||||
dtkPath := options.HostOptions.HygonDtkPath
|
||||
envs := []*runtimeapi.KeyValue{
|
||||
{
|
||||
Key: "HYGON_VISIBLE_DEVICES",
|
||||
Value: strings.Join(indices, ","),
|
||||
},
|
||||
{
|
||||
Key: "HIP_VISIBLE_DEVICES",
|
||||
Value: strings.Join(indices, ","),
|
||||
},
|
||||
{
|
||||
Key: "ROCM_PATH",
|
||||
Value: dtkPath,
|
||||
},
|
||||
{
|
||||
Key: "DTK_HOME",
|
||||
Value: dtkPath,
|
||||
},
|
||||
{
|
||||
Key: "ROCM_SMI_LIB_PATH",
|
||||
Value: path.Join(hyhalPath, "lib"),
|
||||
},
|
||||
}
|
||||
return envs
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) buildHygonExtraConfigures(indices []string) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||
return buildHygonRuntimeEnvs(indices), buildHygonRuntimeMounts()
|
||||
}
|
||||
|
||||
func hygonCommonDevices() []*runtimeapi.Device {
|
||||
perms := "rwm"
|
||||
devs := []*runtimeapi.Device{}
|
||||
for _, devPath := range []string{"/dev/kfd", "/dev/mkfd"} {
|
||||
if hygonPathExists(devPath) {
|
||||
devs = append(devs, &runtimeapi.Device{
|
||||
ContainerPath: devPath,
|
||||
HostPath: devPath,
|
||||
Permissions: perms,
|
||||
})
|
||||
}
|
||||
}
|
||||
return devs
|
||||
}
|
||||
|
||||
func (m *hygonDCUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
|
||||
if dev.IsolatedDevice == nil {
|
||||
return nil, nil, errors.Errorf("isolated device is nil")
|
||||
}
|
||||
iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
|
||||
if iDev == nil {
|
||||
return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id)
|
||||
}
|
||||
dcuDev, ok := iDev.(*hygonDCU)
|
||||
if !ok {
|
||||
return nil, nil, errors.Errorf("device %s is not hygon dcu", dev.IsolatedDevice.Id)
|
||||
}
|
||||
renderPath := dcuDev.GetRenderPath()
|
||||
if renderPath == "" {
|
||||
return nil, nil, errors.Errorf("hygon dcu %s has empty render path", dev.IsolatedDevice.Id)
|
||||
}
|
||||
perms := "rwm"
|
||||
ctrDevs := []*runtimeapi.Device{
|
||||
{
|
||||
ContainerPath: renderPath,
|
||||
HostPath: renderPath,
|
||||
Permissions: perms,
|
||||
},
|
||||
}
|
||||
return ctrDevs, hygonCommonDevices(), nil
|
||||
}
|
||||
|
||||
func hygonDebugOutputSnippet(output string, maxLines int) string {
|
||||
lines := strings.Split(output, "\n")
|
||||
if len(lines) > maxLines {
|
||||
lines = lines[:maxLines]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func getHygonDCUs(manager isolated_device.IContainerDeviceManager, sharingMode string) ([]isolated_device.IDevice, error) {
|
||||
log.Infof("==== getHygonDCUs start: sharingMode=%s", sharingMode)
|
||||
hySmiPath := options.HostOptions.HygonHySmiPath
|
||||
log.Infof("==== getHygonDCUs config: hySmiPath=%s hyhalPath=%s dtkPath=%s remoteExecutor=%v",
|
||||
hySmiPath, options.HostOptions.HygonHyhalPath, options.HostOptions.HygonDtkPath, hygonUseRemoteFS())
|
||||
if !hygonPathExists(hySmiPath) {
|
||||
log.Infof("==== getHygonDCUs abort: hy-smi not found at %s (remote=%v)", hySmiPath, hygonUseRemoteFS())
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
memOut, err := procutils.NewRemoteCommandAsFarAsPossible(hySmiPath, "--showmeminfo", "vram").Output()
|
||||
if err != nil {
|
||||
log.Warningf("==== getHygonDCUs: hy-smi --showmeminfo vram failed: %v, fallback to default output", err)
|
||||
memOut, err = procutils.NewRemoteCommandAsFarAsPossible(hySmiPath).Output()
|
||||
if err != nil {
|
||||
log.Warningf("==== getHygonDCUs abort: hy-smi failed: %v", err)
|
||||
return nil, errors.Wrap(err, "hy-smi")
|
||||
}
|
||||
}
|
||||
log.Infof("==== getHygonDCUs hy-smi output (first 10 lines):\n%s", hygonDebugOutputSnippet(string(memOut), 10))
|
||||
|
||||
indices := parseHySmiDeviceIndices(string(memOut))
|
||||
log.Infof("==== getHygonDCUs parsed indices: %v", indices)
|
||||
if len(indices) == 0 {
|
||||
log.Infof("==== getHygonDCUs abort: no HCU indices parsed from hy-smi output")
|
||||
return nil, nil
|
||||
}
|
||||
memMap := parseHySmiMemInfoVram(string(memOut))
|
||||
log.Infof("==== getHygonDCUs parsed memMap: %v", memMap)
|
||||
renderToPCI, err := getHygonRenderPathToPCIMap()
|
||||
if err != nil {
|
||||
log.Warningf("==== getHygonDCUs abort: getHygonRenderPathToPCIMap failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
log.Infof("==== getHygonDCUs render to pci map: %v", renderToPCI)
|
||||
computeUnitsMap := parseHySmiComputeUnits(string(memOut))
|
||||
|
||||
devs := make([]isolated_device.IDevice, 0, len(indices))
|
||||
for i, idx := range indices {
|
||||
renderPath := hygonRenderPathForHCUIndex(idx)
|
||||
pciAddr := renderToPCI[renderPath]
|
||||
if pciAddr == "" {
|
||||
log.Warningf("==== getHygonDCUs: no pci addr for render path %s (HCU idx=%d)", renderPath, idx)
|
||||
}
|
||||
|
||||
var pciDev *isolated_device.PCIDevice
|
||||
if pciAddr != "" {
|
||||
pciOutput, err := isolated_device.GetPCIStrByAddr(pciAddr)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", pciAddr)
|
||||
}
|
||||
pciDev = isolated_device.NewPCIDevice2(pciOutput[0])
|
||||
} else {
|
||||
pciDev = &isolated_device.PCIDevice{
|
||||
VendorId: computeapi.HYGON_VENDOR_ID,
|
||||
Addr: fmt.Sprintf("hygon-%d", idx),
|
||||
}
|
||||
}
|
||||
|
||||
modelName := hygonModelNameFromPCIDevice(pciDev)
|
||||
|
||||
memSize := memMap[idx]
|
||||
if memSize == 0 {
|
||||
memSize = memMap[i]
|
||||
}
|
||||
computeUnits := computeUnitsMap[idx]
|
||||
if computeUnits == 0 {
|
||||
computeUnits = 60
|
||||
}
|
||||
|
||||
dcuDev := &hygonDCU{
|
||||
manager: manager,
|
||||
BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, strconv.Itoa(idx), sharingMode, 1),
|
||||
gpuIndex: idx,
|
||||
renderPath: renderPath,
|
||||
memorySize: memSize,
|
||||
computeUnits: computeUnits,
|
||||
}
|
||||
dcuDev.SetModelName(modelName)
|
||||
log.Infof("==== getHygonDCUs device[%d]: idx=%d model=%s pci=%s render=%s vendorDeviceId=%s memMiB=%d cu=%d",
|
||||
i, idx, modelName, pciAddr, renderPath, pciDev.GetVendorDeviceId(), memSize, computeUnits)
|
||||
devs = append(devs, dcuDev)
|
||||
}
|
||||
if len(devs) == 0 {
|
||||
log.Infof("==== getHygonDCUs finished: no devices built")
|
||||
return nil, nil
|
||||
}
|
||||
log.Infof("==== getHygonDCUs finished: built %d devices", len(devs))
|
||||
return devs, nil
|
||||
}
|
||||
|
||||
func hygonRenderPathForHCUIndex(idx int) string {
|
||||
return fmt.Sprintf("/dev/dri/renderD%d", 128+idx)
|
||||
}
|
||||
|
||||
func hygonModelNameFromPCIDevice(pciDev *isolated_device.PCIDevice) string {
|
||||
if pciDev == nil {
|
||||
return "Hygon DCU"
|
||||
}
|
||||
if pciDev.ModelName != "" {
|
||||
return pciDev.ModelName
|
||||
}
|
||||
if pciDev.DeviceName != "" {
|
||||
return pciDev.DeviceName
|
||||
}
|
||||
return "Hygon DCU"
|
||||
}
|
||||
|
||||
func buildHygonRenderPathToPCIMap(pciToRender map[string]string) map[string]string {
|
||||
ret := make(map[string]string, len(pciToRender))
|
||||
for pciAddr, renderPath := range pciToRender {
|
||||
ret[renderPath] = pciAddr
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func getHygonRenderPathToPCIMap() (map[string]string, error) {
|
||||
pciToRender, err := getHygonRenderPathMap()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildHygonRenderPathToPCIMap(pciToRender), nil
|
||||
}
|
||||
|
||||
func getHygonRenderPathMap() (map[string]string, error) {
|
||||
const byPathDir = "/dev/dri/by-path"
|
||||
ret := map[string]string{}
|
||||
if !hygonPathExists(byPathDir) {
|
||||
return ret, nil
|
||||
}
|
||||
entries, err := hygonReadDir(byPathDir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "read %s", byPathDir)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
entryName := entry.Name()
|
||||
if !strings.HasSuffix(entryName, "-render") {
|
||||
continue
|
||||
}
|
||||
pciAddr, err := getGPUPCIAddr(entryName)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fp := path.Join(byPathDir, entryName)
|
||||
linkPath, err := hygonReadlink(fp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
renderPath := hygonRenderPathFromLink(linkPath)
|
||||
ret[pciAddr] = renderPath
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
var (
|
||||
hySmiIndexRe = regexp.MustCompile(`(?m)^\s*(\d+)\s+`)
|
||||
hySmiHCUBracketRe = regexp.MustCompile(`HCU\[(\d+)\]`)
|
||||
hySmiHCUMemTotalRe = regexp.MustCompile(`HCU\[(\d+)\]\s*:\s*vram Total Memory \(MiB\):\s*(\d+)`)
|
||||
hySmiComputeRe = regexp.MustCompile(`(?i)(?:Compute units|CU)\s*:\s*(\d+)`)
|
||||
)
|
||||
|
||||
func parseHySmiDeviceIndices(output string) []int {
|
||||
indices := []int{}
|
||||
seen := map[int]bool{}
|
||||
addIndex := func(idx int) {
|
||||
if !seen[idx] {
|
||||
seen[idx] = true
|
||||
indices = append(indices, idx)
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if m := hySmiHCUBracketRe.FindStringSubmatch(line); len(m) == 2 {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
addIndex(idx)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "GPU[") {
|
||||
if m := regexp.MustCompile(`GPU\[(\d+)\]`).FindStringSubmatch(line); len(m) == 2 {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
addIndex(idx)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "HCU") || strings.HasPrefix(line, "GPU") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(fields[0])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
addIndex(idx)
|
||||
}
|
||||
if len(indices) == 0 {
|
||||
for i, line := range strings.Split(output, "\n") {
|
||||
if m := hySmiIndexRe.FindStringSubmatch(line); len(m) == 2 {
|
||||
idx, err := strconv.Atoi(m[1])
|
||||
if err == nil && i > 0 {
|
||||
addIndex(idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return indices
|
||||
}
|
||||
|
||||
func parseHySmiMemInfoVram(output string) map[int]int {
|
||||
ret := map[int]int{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
// HCU[0] : vram Total Memory (MiB): 65520
|
||||
if m := hySmiHCUMemTotalRe.FindStringSubmatch(line); len(m) == 3 {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
mem, _ := strconv.Atoi(m[2])
|
||||
ret[idx] = mem
|
||||
continue
|
||||
}
|
||||
if m := regexp.MustCompile(`GPU\[(\d+)\]\s*:\s*(?:Total Memory|VRAM Total Memory).*?(\d+)`).FindStringSubmatch(line); len(m) == 3 {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
val, _ := strconv.ParseInt(m[2], 10, 64)
|
||||
if val > 1024*1024 {
|
||||
ret[idx] = int(val / 1024 / 1024)
|
||||
} else {
|
||||
ret[idx] = int(val)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if m := regexp.MustCompile(`(?i)^(\d+)\s+.*?(\d+)\s*MiB`).FindStringSubmatch(line); len(m) == 3 {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
mem, _ := strconv.Atoi(m[2])
|
||||
ret[idx] = mem
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func parseHySmiComputeUnits(output string) map[int]int {
|
||||
ret := map[int]int{}
|
||||
currentIdx := -1
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if m := regexp.MustCompile(`(?i)(?:Device|GPU|HCU)\s*(\d+)`).FindStringSubmatch(line); len(m) == 2 {
|
||||
currentIdx, _ = strconv.Atoi(m[1])
|
||||
}
|
||||
if m := hySmiComputeRe.FindStringSubmatch(line); len(m) == 2 && currentIdx >= 0 {
|
||||
cu, _ := strconv.Atoi(m[1])
|
||||
ret[currentIdx] = cu
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package container_device
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
|
||||
hostapi "yunion.io/x/onecloud/pkg/apis/host"
|
||||
"yunion.io/x/onecloud/pkg/hostman/hostinfo"
|
||||
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
|
||||
"yunion.io/x/onecloud/pkg/hostman/options"
|
||||
"yunion.io/x/onecloud/pkg/util/procutils"
|
||||
)
|
||||
|
||||
func init() {
|
||||
isolated_device.RegisterContainerDeviceManager(newHygonDCUHamiManager())
|
||||
}
|
||||
|
||||
type hygonDCUHamiManager struct {
|
||||
*hygonDCUManager
|
||||
|
||||
vdevMu sync.Mutex
|
||||
allocated map[string]int // containerDeviceId -> vdevIndex
|
||||
vdevInUse map[int]string // vdevIndex -> containerDeviceId
|
||||
}
|
||||
|
||||
func newHygonDCUHamiManager() *hygonDCUHamiManager {
|
||||
return &hygonDCUHamiManager{
|
||||
hygonDCUManager: newHygonDCUManager(),
|
||||
allocated: make(map[string]int),
|
||||
vdevInUse: make(map[int]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) GetRegisterType() isolated_device.ContainerDeviceType {
|
||||
return isolated_device.ContainerDeviceTypeHygonDcuHami
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error) {
|
||||
return getHygonDCUs(m, computeapi.DEVICE_SHARING_MODE_HAMI)
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||
if len(devs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
dev := devs[0]
|
||||
if dev.IsolatedDevice == nil {
|
||||
return nil, nil
|
||||
}
|
||||
iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
|
||||
if iDev == nil {
|
||||
return nil, nil
|
||||
}
|
||||
dcuDev, ok := iDev.(*hygonDCU)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
memLimitMiB := dev.IsolatedDevice.MemoryLimit
|
||||
if memLimitMiB <= 0 {
|
||||
memLimitMiB = dcuDev.GetMemorySize()
|
||||
}
|
||||
smLimit := dev.IsolatedDevice.SmUtilLimit
|
||||
if smLimit <= 0 {
|
||||
smLimit = 100
|
||||
}
|
||||
computeUnits := dcuDev.GetComputeUnits()
|
||||
if computeUnits <= 0 {
|
||||
computeUnits = 60
|
||||
}
|
||||
cuCount := computeUnits * smLimit / 100
|
||||
if cuCount <= 0 {
|
||||
cuCount = 1
|
||||
}
|
||||
|
||||
vdevIdx, err := m.ensureVdev(dev.IsolatedDevice.Id, dcuDev.GetIndex(), memLimitMiB, cuCount)
|
||||
if err != nil {
|
||||
log.Errorf("ensure hygon vdev for device %s: %v", dev.IsolatedDevice.Id, err)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
envs := buildHygonRuntimeEnvs([]string{strconv.Itoa(vdevIdx)})
|
||||
mounts := buildHygonRuntimeMounts()
|
||||
|
||||
vdevConfHost := path.Join(options.HostOptions.HygonVdevConfDir, fmt.Sprintf("vdev%d.conf", vdevIdx))
|
||||
vdevConfContainer := path.Join("/etc/vdev/docker", fmt.Sprintf("vdev%d.conf", vdevIdx))
|
||||
if hygonPathExists(vdevConfHost) {
|
||||
mounts = append(mounts, &runtimeapi.Mount{
|
||||
ContainerPath: vdevConfContainer,
|
||||
HostPath: vdevConfHost,
|
||||
Readonly: true,
|
||||
})
|
||||
}
|
||||
return envs, mounts
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
|
||||
return m.hygonDCUManager.NewContainerDevices(input, dev)
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) ensureVdev(containerDevId string, physIdx, memMiB, computeUnits int) (int, error) {
|
||||
m.vdevMu.Lock()
|
||||
defer m.vdevMu.Unlock()
|
||||
|
||||
if vdevIdx, ok := m.allocated[containerDevId]; ok {
|
||||
return vdevIdx, nil
|
||||
}
|
||||
|
||||
vdevIdx, err := m.findOrCreateVdev(physIdx, memMiB, computeUnits)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
m.allocated[containerDevId] = vdevIdx
|
||||
m.vdevInUse[vdevIdx] = containerDevId
|
||||
return vdevIdx, nil
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) findOrCreateVdev(physIdx, memMiB, computeUnits int) (int, error) {
|
||||
existing, err := m.listAvailableVdevIndices(physIdx)
|
||||
if err != nil {
|
||||
log.Warningf("list hygon vdev indices: %v", err)
|
||||
}
|
||||
for _, idx := range existing {
|
||||
if _, used := m.vdevInUse[idx]; !used {
|
||||
return idx, nil
|
||||
}
|
||||
}
|
||||
|
||||
hySmiPath := options.HostOptions.HygonHySmiPath
|
||||
out, err := procutils.NewRemoteCommandAsFarAsPossible(
|
||||
hySmiPath, "virtual", "-create-vdevices", "1",
|
||||
"-d", strconv.Itoa(physIdx),
|
||||
"-vdevice-compute-units", strconv.Itoa(computeUnits),
|
||||
"-vdevice-memory-size", strconv.Itoa(memMiB),
|
||||
).Output()
|
||||
if err != nil {
|
||||
return -1, errors.Wrapf(err, "create vdev on dcu %d: %s", physIdx, out)
|
||||
}
|
||||
|
||||
created, err := m.listAvailableVdevIndices(physIdx)
|
||||
if err != nil || len(created) == 0 {
|
||||
return physIdx, nil
|
||||
}
|
||||
for _, idx := range created {
|
||||
if _, used := m.vdevInUse[idx]; !used {
|
||||
return idx, nil
|
||||
}
|
||||
}
|
||||
return created[len(created)-1], nil
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) listAvailableVdevIndices(physIdx int) ([]int, error) {
|
||||
hySmiPath := options.HostOptions.HygonHySmiPath
|
||||
out, err := procutils.NewRemoteCommandAsFarAsPossible(hySmiPath, "virtual", "-show-vdevice-info").Output()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "hy-smi virtual -show-vdevice-info")
|
||||
}
|
||||
return parseHySmiVdeviceIndices(string(out), physIdx), nil
|
||||
}
|
||||
|
||||
func parseHySmiVdeviceIndices(output string, physIdx int) []int {
|
||||
indices := []int{}
|
||||
currentVdev := -1
|
||||
currentPhys := -1
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "Virtual Device") {
|
||||
if m := regexp.MustCompile(`Virtual Device\s*(\d+)`).FindStringSubmatch(line); len(m) == 2 {
|
||||
currentVdev, _ = strconv.Atoi(m[1])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "Actual Device:") {
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) == 2 {
|
||||
currentPhys, _ = strconv.Atoi(strings.TrimSpace(parts[1]))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if currentVdev >= 0 && currentPhys == physIdx {
|
||||
indices = append(indices, currentVdev)
|
||||
currentVdev = -1
|
||||
currentPhys = -1
|
||||
}
|
||||
}
|
||||
if len(indices) == 0 {
|
||||
vdevDir := options.HostOptions.HygonVdevConfDir
|
||||
if hygonPathExists(vdevDir) {
|
||||
entries, err := hygonReadDir(vdevDir)
|
||||
if err == nil {
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if strings.HasPrefix(name, "vdev") && strings.HasSuffix(name, ".conf") {
|
||||
numStr := strings.TrimSuffix(strings.TrimPrefix(name, "vdev"), ".conf")
|
||||
if idx, err := strconv.Atoi(numStr); err == nil {
|
||||
indices = append(indices, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return indices
|
||||
}
|
||||
|
||||
func (m *hygonDCUHamiManager) releaseVdev(containerDevId string) {
|
||||
m.vdevMu.Lock()
|
||||
defer m.vdevMu.Unlock()
|
||||
vdevIdx, ok := m.allocated[containerDevId]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(m.allocated, containerDevId)
|
||||
delete(m.vdevInUse, vdevIdx)
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package container_device
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
|
||||
)
|
||||
|
||||
func TestParseHySmiDeviceIndices(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
input string
|
||||
expect []int
|
||||
}{
|
||||
{
|
||||
name: "default table",
|
||||
input: `HCU Temp AvgPwr Perf PwrCap VRAM% HCU% Mode
|
||||
0 61.0C 108.0W auto 400.0W 0% 0.0% Normal
|
||||
1 62.0C 112.0W auto 400.0W 0% 0.0% Normal`,
|
||||
expect: []int{0, 1},
|
||||
},
|
||||
{
|
||||
name: "gpu bracket format",
|
||||
input: `GPU[0] : VRAM Total Memory (B): 34342961152
|
||||
GPU[1] : VRAM Total Memory (B): 34342961152`,
|
||||
expect: []int{0, 1},
|
||||
},
|
||||
{
|
||||
name: "hcu vram meminfo format",
|
||||
input: `HCU[0] : vram Total Memory (MiB): 65520
|
||||
HCU[0] : vram Total Used Memory (MiB): 2
|
||||
HCU[1] : vram Total Memory (MiB): 65520
|
||||
HCU[7] : vram Total Memory (MiB): 65520`,
|
||||
expect: []int{0, 1, 7},
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := parseHySmiDeviceIndices(c.input)
|
||||
assert.Equal(t, c.expect, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHySmiMemInfoVram(t *testing.T) {
|
||||
t.Run("gpu bytes format", func(t *testing.T) {
|
||||
input := `GPU[0] : VRAM Total Memory (B): 34342961152
|
||||
GPU[1] : VRAM Total Memory (B): 34342961152`
|
||||
memMap := parseHySmiMemInfoVram(input)
|
||||
assert.Greater(t, memMap[0], 30000)
|
||||
assert.Greater(t, memMap[1], 30000)
|
||||
})
|
||||
t.Run("hcu vram meminfo format", func(t *testing.T) {
|
||||
input := `HCU[0] : vram Total Memory (MiB): 65520
|
||||
HCU[0] : vram Total Used Memory (MiB): 2
|
||||
HCU[1] : vram Total Memory (MiB): 65520
|
||||
HCU[7] : vram Total Memory (MiB): 65520`
|
||||
memMap := parseHySmiMemInfoVram(input)
|
||||
assert.Equal(t, 65520, memMap[0])
|
||||
assert.Equal(t, 65520, memMap[1])
|
||||
assert.Equal(t, 65520, memMap[7])
|
||||
assert.NotContains(t, memMap, 2) // should not pick up "Used Memory" value
|
||||
})
|
||||
}
|
||||
|
||||
func TestHygonModelNameFromPCIDevice(t *testing.T) {
|
||||
t.Run("from lspci model name", func(t *testing.T) {
|
||||
line := `09:00.0 "Display controller [0380]" "Chengdu Haiguang IC Design Co., Ltd. [1d94]" "Z100 [K100_AI] [6320]" -rxx "Chengdu Haiguang IC Design Co., Ltd. [1d94]" "Z100 [K100_AI] [6320]"`
|
||||
pciDev := isolated_device.NewPCIDevice2(line)
|
||||
assert.Equal(t, "K100_AI", hygonModelNameFromPCIDevice(pciDev))
|
||||
})
|
||||
t.Run("fallback to device name", func(t *testing.T) {
|
||||
pciDev := &isolated_device.PCIDevice{
|
||||
DeviceName: "Z100 [K100_AI]",
|
||||
}
|
||||
assert.Equal(t, "Z100 [K100_AI]", hygonModelNameFromPCIDevice(pciDev))
|
||||
})
|
||||
t.Run("fallback to default", func(t *testing.T) {
|
||||
assert.Equal(t, "Hygon DCU", hygonModelNameFromPCIDevice(nil))
|
||||
assert.Equal(t, "Hygon DCU", hygonModelNameFromPCIDevice(&isolated_device.PCIDevice{}))
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseHySmiVdeviceIndices(t *testing.T) {
|
||||
input := `Virtual Device 0:
|
||||
Actual Device: 0
|
||||
Compute units: 5
|
||||
Global memory: 4294967296 bytes
|
||||
Virtual Device 1:
|
||||
Actual Device: 0
|
||||
Compute units: 15
|
||||
Global memory: 8589934592 bytes`
|
||||
indices := parseHySmiVdeviceIndices(input, 0)
|
||||
assert.Equal(t, []int{0, 1}, indices)
|
||||
}
|
||||
|
||||
func TestHygonRenderPathFromLinkWithRemote(t *testing.T) {
|
||||
assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathFromLinkWithRemote("../renderD128", false))
|
||||
assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathFromLinkWithRemote("/dev/dri/renderD128", true))
|
||||
}
|
||||
|
||||
func TestBuildHygonRenderPathMapFromEntry(t *testing.T) {
|
||||
entryName := "pci-0000:03:00.0-render"
|
||||
pciAddr, err := getGPUPCIAddr(entryName)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "0000:03:00.0", pciAddr)
|
||||
renderPath := hygonRenderPathFromLinkWithRemote("/dev/dri/renderD128", true)
|
||||
assert.Equal(t, "/dev/dri/renderD128", renderPath)
|
||||
}
|
||||
|
||||
func TestHygonRenderPathForHCUIndex(t *testing.T) {
|
||||
assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathForHCUIndex(0))
|
||||
assert.Equal(t, "/dev/dri/renderD129", hygonRenderPathForHCUIndex(1))
|
||||
assert.Equal(t, "/dev/dri/renderD135", hygonRenderPathForHCUIndex(7))
|
||||
}
|
||||
|
||||
func TestBuildHygonRenderPathToPCIMapFromEntries(t *testing.T) {
|
||||
// Sample by-path entries from Hygon node m06r2n19
|
||||
entries := []struct {
|
||||
entryName string
|
||||
linkPath string
|
||||
renderPath string
|
||||
pciAddr string
|
||||
}{
|
||||
{"pci-0000:09:00.0-render", "../renderD128", "/dev/dri/renderD128", "0000:09:00.0"},
|
||||
{"pci-0000:36:00.0-render", "../renderD129", "/dev/dri/renderD129", "0000:36:00.0"},
|
||||
{"pci-0000:55:00.0-render", "../renderD130", "/dev/dri/renderD130", "0000:55:00.0"},
|
||||
{"pci-0000:77:00.0-render", "../renderD131", "/dev/dri/renderD131", "0000:77:00.0"},
|
||||
{"pci-0000:85:00.0-render", "../renderD132", "/dev/dri/renderD132", "0000:85:00.0"},
|
||||
{"pci-0000:b5:00.0-render", "../renderD133", "/dev/dri/renderD133", "0000:b5:00.0"},
|
||||
{"pci-0000:d5:00.0-render", "../renderD134", "/dev/dri/renderD134", "0000:d5:00.0"},
|
||||
{"pci-0000:f5:00.0-render", "../renderD135", "/dev/dri/renderD135", "0000:f5:00.0"},
|
||||
}
|
||||
|
||||
pciToRender := map[string]string{}
|
||||
for _, e := range entries {
|
||||
pciAddr, err := getGPUPCIAddr(e.entryName)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, e.pciAddr, pciAddr)
|
||||
renderPath := hygonRenderPathFromLinkWithRemote(e.linkPath, false)
|
||||
assert.Equal(t, e.renderPath, renderPath)
|
||||
pciToRender[pciAddr] = renderPath
|
||||
}
|
||||
|
||||
renderToPCI := buildHygonRenderPathToPCIMap(pciToRender)
|
||||
assert.Len(t, renderToPCI, 8)
|
||||
|
||||
for hcuIdx := 0; hcuIdx < 8; hcuIdx++ {
|
||||
renderPath := hygonRenderPathForHCUIndex(hcuIdx)
|
||||
pciAddr, ok := renderToPCI[renderPath]
|
||||
assert.True(t, ok, "HCU[%d] render path %s should exist in map", hcuIdx, renderPath)
|
||||
assert.Equal(t, entries[hcuIdx].pciAddr, pciAddr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the permissions and limitations under the License.
|
||||
|
||||
package container_device
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/hostman/options"
|
||||
"yunion.io/x/onecloud/pkg/util/fileutils2"
|
||||
"yunion.io/x/onecloud/pkg/util/procutils"
|
||||
)
|
||||
|
||||
func hygonUseRemoteFS() bool {
|
||||
return options.HostOptions.EnableRemoteExecutor
|
||||
}
|
||||
|
||||
func hygonPathExists(path string) bool {
|
||||
if hygonUseRemoteFS() {
|
||||
return procutils.RemotePathExists(path)
|
||||
}
|
||||
return fileutils2.Exists(path)
|
||||
}
|
||||
|
||||
func hygonReadDir(dirname string) ([]os.FileInfo, error) {
|
||||
if hygonUseRemoteFS() {
|
||||
return procutils.RemoteReadDir(dirname)
|
||||
}
|
||||
entries, err := os.ReadDir(dirname)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := make([]os.FileInfo, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
files = append(files, info)
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func hygonReadlink(path string) (string, error) {
|
||||
if hygonUseRemoteFS() {
|
||||
return procutils.RemoteReadlink(path)
|
||||
}
|
||||
return os.Readlink(path)
|
||||
}
|
||||
|
||||
func hygonRenderPathFromLink(linkPath string) string {
|
||||
return hygonRenderPathFromLinkWithRemote(linkPath, hygonUseRemoteFS())
|
||||
}
|
||||
|
||||
func hygonRenderPathFromLinkWithRemote(linkPath string, remote bool) string {
|
||||
if remote {
|
||||
return linkPath
|
||||
}
|
||||
return path.Join("/dev/dri", path.Base(linkPath))
|
||||
}
|
||||
@@ -292,6 +292,38 @@ func (man *isolatedDeviceManager) probeContainerAscendNPUs(enable, enableHami bo
|
||||
}
|
||||
}
|
||||
|
||||
func (man *isolatedDeviceManager) probeContainerHygonDCUs(enable, enableHami bool) {
|
||||
log.Infof("==== hygon dcu probe start: enable=%v enableHami=%v", enable, enableHami)
|
||||
devType := ContainerDeviceTypeHygonDcu
|
||||
if enableHami {
|
||||
devType = ContainerDeviceTypeHygonDcuHami
|
||||
} else if !enable {
|
||||
log.Infof("==== hygon dcu probe skipped: enable_container_hygon_dcu=false and enable_container_hygon_dcu_hami=false")
|
||||
return
|
||||
}
|
||||
log.Infof("==== hygon dcu probe using manager type: %s", devType)
|
||||
|
||||
devman, err := GetContainerDeviceManager(devType)
|
||||
if err != nil {
|
||||
log.Errorf("==== hygon dcu probe failed: no container device manager %s found: %v", devType, err)
|
||||
return
|
||||
}
|
||||
devs, err := devman.ProbeDevices()
|
||||
if err != nil {
|
||||
log.Warningf("==== hygon dcu probe failed: ProbeDevices error: %v", err)
|
||||
return
|
||||
}
|
||||
if len(devs) == 0 {
|
||||
log.Infof("==== hygon dcu probe finished: no devices found")
|
||||
return
|
||||
}
|
||||
for idx, dev := range devs {
|
||||
man.devices = append(man.devices, dev)
|
||||
log.Infof("==== hygon dcu probe add device: idx=%d dev=%#v", idx, dev)
|
||||
}
|
||||
log.Infof("==== hygon dcu probe finished: total %d devices", len(devs))
|
||||
}
|
||||
|
||||
func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) {
|
||||
if skipGPUs {
|
||||
return
|
||||
@@ -464,6 +496,8 @@ type SIsolatedDeviceProbeOptions struct {
|
||||
EnableCudaMps bool
|
||||
EnableContainerAscendNpu bool
|
||||
EnableContainerAscendNpuHAMI bool
|
||||
EnableContainerHygonDCU bool
|
||||
EnableContainerHygonDCUHAMI bool
|
||||
EnableWhitelist bool
|
||||
|
||||
SriovNics, OvsOffloadNics []HostNic
|
||||
@@ -473,11 +507,16 @@ type SIsolatedDeviceProbeOptions struct {
|
||||
|
||||
func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOptions) {
|
||||
man.devices = make([]IDevice, 0)
|
||||
if man.host.IsContainerHost() {
|
||||
isContainerHost := man.host.IsContainerHost()
|
||||
log.Infof("==== ProbePCIDevices start: isContainerHost=%v hygonEnable=%v hygonHami=%v skipGPUs=%v",
|
||||
isContainerHost, opts.EnableContainerHygonDCU, opts.EnableContainerHygonDCUHAMI, opts.SkipGPUs)
|
||||
if isContainerHost {
|
||||
man.probeContainerDevices()
|
||||
man.probeContainerNvidiaGPUs(opts.EnableCudaHAMI, opts.EnableCudaMps)
|
||||
man.probeContainerAscendNPUs(opts.EnableContainerAscendNpu, opts.EnableContainerAscendNpuHAMI)
|
||||
man.probeContainerHygonDCUs(opts.EnableContainerHygonDCU, opts.EnableContainerHygonDCUHAMI)
|
||||
} else {
|
||||
log.Infof("==== ProbePCIDevices: not container host, hygon container probe will NOT run (use host_type=container for hygon dcu)")
|
||||
devModels, err := man.getCustomIsolatedDeviceModels()
|
||||
if err != nil {
|
||||
log.Errorf("get isolated device devModels %s", err.Error())
|
||||
@@ -492,6 +531,7 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti
|
||||
man.probeNVIDIAVgpus(opts.NvidiaVgpuPFs)
|
||||
man.probeGPUS(opts.SkipGPUs, opts.AmdVgpuPFs, opts.NvidiaVgpuPFs, opts.EnableWhitelist, devModels)
|
||||
}
|
||||
log.Infof("==== ProbePCIDevices finished: total isolated devices=%d", len(man.devices))
|
||||
}
|
||||
|
||||
type IsolatedDeviceModel struct {
|
||||
|
||||
@@ -279,6 +279,13 @@ type SHostOptions struct {
|
||||
AscendNpuHamiShmPath string `help:"ascend npu hami shm path" default:"/opt/cloud/hami-shared-region"`
|
||||
AscendNpuHamiLibvnpuPath string `help:"ascend npu hami libvnpu.so path" default:"/opt/cloud/hami/libvnpu.so"`
|
||||
|
||||
EnableContainerHygonDCU bool `help:"enable container hygon dcu" default:"true"`
|
||||
EnableContainerHygonDCUHAMI bool `help:"enable container hygon dcu hami" default:"false"`
|
||||
HygonHyhalPath string `help:"hygon hyhal driver path" default:"/opt/hyhal"`
|
||||
HygonDtkPath string `help:"hygon dtk toolkit path" default:"/opt/dtk"`
|
||||
HygonHySmiPath string `help:"hygon hy-smi path" default:"/opt/hyhal/bin/hy-smi"`
|
||||
HygonVdevConfDir string `help:"hygon vdcu config directory" default:"/etc/vdev"`
|
||||
|
||||
EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"`
|
||||
EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"`
|
||||
DisableReconcileContainer bool `help:"disable reconcile container" default:"false"`
|
||||
|
||||
@@ -119,6 +119,16 @@ func normalizeLLMSkuDevice(dev *api.Device) {
|
||||
if dev.SharingMode == "" {
|
||||
dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
|
||||
}
|
||||
case computeapi.CONTAINER_DEV_HYGON_DCU:
|
||||
dev.DevType = computeapi.GPU_TYPE
|
||||
if dev.SharingMode == "" {
|
||||
dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
|
||||
}
|
||||
case computeapi.CONTAINER_DEV_HYGON_DCU_HAMI:
|
||||
dev.DevType = computeapi.GPU_TYPE
|
||||
if dev.SharingMode == "" {
|
||||
dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
|
||||
}
|
||||
}
|
||||
if dev.SharingMode == "" {
|
||||
dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
|
||||
|
||||
@@ -50,6 +50,18 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) {
|
||||
wantDevType: computeapi.GPU_TYPE,
|
||||
wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI,
|
||||
},
|
||||
{
|
||||
name: "HYGON_DCU",
|
||||
in: api.Device{DevType: computeapi.CONTAINER_DEV_HYGON_DCU},
|
||||
wantDevType: computeapi.GPU_TYPE,
|
||||
wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE,
|
||||
},
|
||||
{
|
||||
name: "HYGON_DCU_HAMI",
|
||||
in: api.Device{DevType: computeapi.CONTAINER_DEV_HYGON_DCU_HAMI},
|
||||
wantDevType: computeapi.GPU_TYPE,
|
||||
wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI,
|
||||
},
|
||||
{
|
||||
name: "explicit sharing_mode preserved",
|
||||
in: api.Device{DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU_SHARE, SharingMode: computeapi.DEVICE_SHARING_MODE_MPS},
|
||||
|
||||
@@ -27,7 +27,7 @@ var (
|
||||
func init() {
|
||||
IsolatedDevices = modules.NewComputeManager("isolated_device", "isolated_devices",
|
||||
[]string{"ID", "Dev_type", "Sharing_mode",
|
||||
"Model", "Addr", "Vendor_device_id", "Mdev_id",
|
||||
"Model", "Vendor", "Addr", "Vendor_device_id", "Mdev_id",
|
||||
"Host_id", "Host", "numa_node",
|
||||
"Guest_id", "Guest", "Guest_status", "Device_path", "Render_path", "PCIE_Info", "Index", "Device_minor"},
|
||||
[]string{})
|
||||
@@ -36,7 +36,7 @@ func init() {
|
||||
ServerIsolatedDevices = modules.NewJointComputeManager(
|
||||
"guestisolateddevice",
|
||||
"guestisolateddevices",
|
||||
[]string{"Guest_ID", "Guest", "Isolated_device_ID", "Index", "Dev_type", "Sharing_mode",
|
||||
[]string{"Guest_ID", "Guest", "Isolated_device_ID", "Index", "Dev_type", "Sharing_mode", "Vendor",
|
||||
"Device_memory_size", "Sm_util_limit", "Network_index", "Disk_index"},
|
||||
[]string{},
|
||||
&Servers,
|
||||
|
||||
@@ -31,6 +31,7 @@ type DeviceListOptions struct {
|
||||
|
||||
DevType []string `help:"filter by dev_type"`
|
||||
Model []string `help:"filter by model"`
|
||||
Vendor []string `help:"filter by vendor, e.g. HYGON, NVIDIA"`
|
||||
Addr []string `help:"filter by addr"`
|
||||
DevicePath []string `help:"filter by device path"`
|
||||
VendorDeviceId []string `help:"filter by vendor device id(PCIID)"`
|
||||
|
||||
@@ -19,6 +19,15 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRemotePathExists(t *testing.T) {
|
||||
if !RemotePathExists("/") {
|
||||
t.Errorf("RemotePathExists(\"/\") want true")
|
||||
}
|
||||
if RemotePathExists("/tmp/__0_1_2_3_a_b_c_d____") {
|
||||
t.Errorf("RemotePathExists missing file want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStat(t *testing.T) {
|
||||
cases := []struct {
|
||||
filename string
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package procutils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
)
|
||||
|
||||
// RemotePathExists checks whether path exists on the host via executor.
|
||||
func RemotePathExists(path string) bool {
|
||||
_, err := RemoteStat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// RemoteReadlink resolves a symlink on the host via executor.
|
||||
func RemoteReadlink(path string) (string, error) {
|
||||
out, err := NewRemoteCommandAsFarAsPossible("readlink", "-f", path).Output()
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "readlink -f %s: %s", path, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
Reference in New Issue
Block a user