feat(host): support Hygon DCU HAMI vdev core allocation (#25274)

cmd/climc/shell/llm/llm_router_agent.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm"

pkg/aiproxy/chatlog/chatlog.go

@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+
 	"yunion.io/x/log"
 )
 

pkg/aiproxy/extensions/visual/messages.go

@@ -21,10 +21,10 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/onecloud/pkg/mcclient/auth"
 
 	"yunion.io/x/onecloud/pkg/aiproxy/models"
 	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 var ErrVisualStreamingUnsupported = fmt.Errorf("visual extension does not support streaming")

pkg/aiproxy/extensions/visual/responses.go

@@ -19,10 +19,10 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/onecloud/pkg/mcclient/auth"
 
 	"yunion.io/x/onecloud/pkg/aiproxy/models"
 	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 // ShouldHandle reports whether the Responses visual orchestration path should run.

pkg/aiproxy/ft/interactive.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"golang.org/x/term"
+
 	"yunion.io/x/pkg/errors"
 )
 

pkg/aiproxy/models/ai_providers_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"yunion.io/x/jsonutils"
+
 	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 )

pkg/aiproxy/providers/registry.go

@@ -19,15 +19,13 @@ import (
 	"sync"
 
 	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
-	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
-	// "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" // uncommon
-	"yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic"
-	// "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" // uncommon
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" // "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" // uncommon
 	// "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" // uncommon
 	// "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" // uncommon
 	"yunion.io/x/onecloud/pkg/aiproxy/providers/gemini"
 	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
 	"yunion.io/x/onecloud/pkg/aiproxy/providers/vllm"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy" // "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" // uncommon
 )
 
 var (

pkg/hostman/guestman/pod.go

@@ -57,6 +57,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device"
 	_ "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device/cdi"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
@@ -2253,7 +2254,11 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 			}
 			ctrCfg.Devices = append(ctrCfg.Devices, ctrDevs...)
 		}
-		if err := s.getIsolatedDeviceExtraConfig(spec, ctrCfg); err != nil {
+		if err := func() error {
+			container_device.SetHygonContainerContext(s.GetId(), input.Name)
+			defer container_device.ClearHygonContainerContext()
+			return s.getIsolatedDeviceExtraConfig(spec, ctrCfg)
+		}(); err != nil {
 			return "", err
 		}
 	} else {
@@ -2447,6 +2452,38 @@ func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.Container
 	return nil
 }
 
+func (s *sPodGuestInstance) releaseIsolatedContainerDevices(devs []*hostapi.ContainerDevice) {
+	devsByMan := map[isolated_device.IContainerDeviceManager][]*hostapi.ContainerDevice{}
+	manOrder := []isolated_device.IContainerDeviceManager{}
+	for i := range devs {
+		dev := devs[i]
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		devType := dev.IsolatedDevice.DeviceType
+		if devType != computeapi.GPU_TYPE && devType != computeapi.NPU_TYPE {
+			continue
+		}
+		if dev.IsolatedDevice.IsCDIUsed() || len(dev.IsolatedDevice.OnlyEnv) > 0 {
+			continue
+		}
+		iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+		if iDev == nil {
+			continue
+		}
+		devMan := iDev.GetContainerDeviceManager()
+		if _, ok := devsByMan[devMan]; !ok {
+			manOrder = append(manOrder, devMan)
+		}
+		devsByMan[devMan] = append(devsByMan[devMan], dev)
+	}
+	for _, devMan := range manOrder {
+		if rel, ok := devMan.(isolated_device.IContainerDeviceReleaseManager); ok {
+			rel.ReleaseContainerDevices(devsByMan[devMan])
+		}
+	}
+}
+
 func (s *sPodGuestInstance) getContainerSystemCpusDir(ctrId string) string {
 	rootFsPath, _ := s.GetRootFsMountPath(ctrId)
 	if rootFsPath != "" {
@@ -2607,6 +2644,9 @@ func (s *sPodGuestInstance) ensureContainerSystemCpufreqHostDir(cpuDir, hostCpuP
 }
 
 func (s *sPodGuestInstance) DeleteContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string) (jsonutils.JSONObject, error) {
+	if ctr := s.GetContainerById(ctrId); ctr != nil {
+		s.releaseIsolatedContainerDevices(ctr.Spec.Devices)
+	}
 	criId, err := s.getContainerCRIId(ctrId)
 	if err != nil && errors.Cause(err) != errors.ErrNotFound {
 		return nil, errors.Wrap(err, "getContainerCRIId")

pkg/hostman/hostinfo/hostinfo.go

@@ -2347,7 +2347,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		options.HostOptions.HostType, h.IsContainerHost(), h.IsKvmSupport())
 	log.Infof("==== probeSyncIsolatedDevices hygon config: enableDCU=%v enableHAMI=%v hySmiPath=%s hyhalPath=%s dtkPath=%s",
 		options.HostOptions.EnableContainerHygonDCU,
-		options.HostOptions.EnableContainerHygonDCUHAMI,
+		options.HostOptions.EnableContainerHygonDCUHami,
 		options.HostOptions.HygonHySmiPath,
 		options.HostOptions.HygonHyhalPath,
 		options.HostOptions.HygonDtkPath,
@@ -2361,7 +2361,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		EnableContainerAscendNpu:     options.HostOptions.EnableContainerAscendNPU,
 		EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami,
 		EnableContainerHygonDCU:      options.HostOptions.EnableContainerHygonDCU,
-		EnableContainerHygonDCUHAMI:  options.HostOptions.EnableContainerHygonDCUHAMI,
+		EnableContainerHygonDCUHAMI:  options.HostOptions.EnableContainerHygonDCUHami,
 		EnableWhitelist:              options.HostOptions.EnableIsolatedDeviceWhitelist,
 		SriovNics:                    sriovNics,
 		OvsOffloadNics:               offloadNics,

pkg/hostman/isolated_device/container_device.go

@@ -100,6 +100,11 @@ type IContainerDeviceManager interface {
 	GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount)
 }
 
+// IContainerDeviceReleaseManager is implemented by device managers that need cleanup on container delete.
+type IContainerDeviceReleaseManager interface {
+	ReleaseContainerDevices(devs []*hostapi.ContainerDevice)
+}
+
 type IContainerCDIManager interface {
 	GetKind() apis.ContainerCDIKind
 	GetSpecFilePath() string

pkg/hostman/isolated_device/container_device/cph_amd_gpu.go

@@ -18,10 +18,10 @@ import (
 	"strings"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
-	"yunion.io/x/onecloud/pkg/apis/compute"
 
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 )

pkg/hostman/isolated_device/container_device/cph_aosp_binder.go

@@ -20,10 +20,10 @@ import (
 	"path/filepath"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
-	"yunion.io/x/onecloud/pkg/apis/compute"
 
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"

pkg/hostman/isolated_device/container_device/hygon_context.go

@@ -0,0 +1,45 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import "sync"
+
+var (
+	hygonCtxMu          sync.Mutex
+	hygonPendingGuestId string
+	hygonPendingCtrName string
+)
+
+// SetHygonContainerContext stores guest/container identity for vdev cache directory naming.
+func SetHygonContainerContext(guestId, containerName string) {
+	hygonCtxMu.Lock()
+	defer hygonCtxMu.Unlock()
+	hygonPendingGuestId = guestId
+	hygonPendingCtrName = containerName
+}
+
+// ClearHygonContainerContext clears pending guest/container identity.
+func ClearHygonContainerContext() {
+	hygonCtxMu.Lock()
+	defer hygonCtxMu.Unlock()
+	hygonPendingGuestId = ""
+	hygonPendingCtrName = ""
+}
+
+func getHygonContainerContext() (guestId, containerName string) {
+	hygonCtxMu.Lock()
+	defer hygonCtxMu.Unlock()
+	return hygonPendingGuestId, hygonPendingCtrName
+}

pkg/hostman/isolated_device/container_device/hygon_corealloc.go

@@ -0,0 +1,84 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// initHygonCoreUsage returns a zero-filled hex string for CU bitmap allocation.
+func initHygonCoreUsage(_ int) string {
+	return strings.Repeat("0", 16)
+}
+
+func addHygonCoreUsage(tot string, c string) (string, error) {
+	i := 0
+	res := ""
+	for {
+		if i >= len(tot) || tot[i] == 0 {
+			break
+		}
+		left, _ := strconv.ParseInt(string(tot[i]), 16, 0)
+		right, _ := strconv.ParseInt(string(c[i]), 16, 0)
+		merged := int(left | right)
+		res = fmt.Sprintf("%s%x", res, merged)
+		i++
+	}
+	return res, nil
+}
+
+func hygonByteAlloc(b int, req int) (int, int) {
+	if req == 0 {
+		return 0, 0
+	}
+	remains := req
+	leftstr := fmt.Sprintf("%b", b)
+	for len(leftstr) < 4 {
+		leftstr = "0" + leftstr
+	}
+	res := 0
+	i := 0
+	for i < len(leftstr) {
+		res = res * 2
+		if leftstr[i] == '0' && remains > 0 {
+			remains--
+			res = res + 1
+		}
+		if remains <= 0 {
+			break
+		}
+		i++
+	}
+	return res, remains
+}
+
+func allocHygonCoreUsage(tot string, req int) (string, int, error) {
+	i := len(tot) - 1
+	res := ""
+	remains := req
+	for {
+		if i < 0 {
+			break
+		}
+		left, _ := strconv.ParseInt(string(tot[i]), 16, 0)
+		alloc, newRemains := hygonByteAlloc(int(left), remains)
+		remains = newRemains
+		res = fmt.Sprintf("%x%s", alloc, res)
+		i--
+	}
+	return res, remains, nil
+}

pkg/hostman/isolated_device/container_device/hygon_corealloc_test.go

@@ -0,0 +1,61 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+)
+
+func TestInitHygonCoreUsage(t *testing.T) {
+	assert.Equal(t, "0000000000000000", initHygonCoreUsage(60))
+}
+
+func TestAllocHygonCoreUsage(t *testing.T) {
+	tot := initHygonCoreUsage(60)
+	mask, remains, err := allocHygonCoreUsage(tot, 4)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, remains)
+	assert.NotEmpty(t, mask)
+
+	merged, err := addHygonCoreUsage(tot, mask)
+	assert.NoError(t, err)
+	assert.Equal(t, mask, merged)
+}
+
+func TestHygonPciBusIdFromAddr(t *testing.T) {
+	assert.Equal(t, "0000:09:00.0", hygonPciBusIdFromAddr("09:00.0"))
+	assert.Equal(t, "0000:09:00.0", hygonPciBusIdFromAddr("0000:09:00.0"))
+}
+
+func TestIsHygonVDcuRequest(t *testing.T) {
+	dev := &hostapi.ContainerDevice{
+		IsolatedDevice: &hostapi.ContainerIsolatedDevice{
+			MemoryLimit: 8192,
+		},
+	}
+	assert.True(t, isHygonVDcuRequest([]*hostapi.ContainerDevice{dev}, 65520))
+	assert.False(t, isHygonVDcuRequest([]*hostapi.ContainerDevice{dev, dev}, 65520))
+	dev.IsolatedDevice.MemoryLimit = 65520
+	assert.False(t, isHygonVDcuRequest([]*hostapi.ContainerDevice{dev}, 65520))
+}
+
+func TestHygonVgpuCacheDirName(t *testing.T) {
+	name := hygonVgpuCacheDirName("guest1", "ctr1", 0, 1, 2, "abc", "def")
+	assert.Equal(t, "guest1_ctr1_0_1_2_abc_def", name)
+}

pkg/hostman/isolated_device/container_device/hygon_dcu.go

@@ -70,6 +70,7 @@ type hygonDCU struct {
 	*BaseDevice
 	gpuIndex     int
 	renderPath   string
+	cardPath     string
 	memorySize   int
 	computeUnits int
 }
@@ -86,6 +87,10 @@ func (dev *hygonDCU) GetRenderPath() string {
 	return dev.renderPath
 }
 
+func (dev *hygonDCU) GetCardPath() string {
+	return dev.cardPath
+}
+
 func (dev *hygonDCU) GetComputeUnits() int {
 	return dev.computeUnits
 }
@@ -116,9 +121,8 @@ func (m *hygonDCUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerD
 	return m.buildHygonExtraConfigures(indices)
 }
 
-func buildHygonRuntimeMounts() []*runtimeapi.Mount {
+func buildHygonRuntimeMounts(includeDtk bool) []*runtimeapi.Mount {
 	hyhalPath := options.HostOptions.HygonHyhalPath
-	//dtkPath := options.HostOptions.HygonDtkPath
 	mounts := []*runtimeapi.Mount{}
 	if hygonPathExists(hyhalPath) {
 		mounts = append(mounts, &runtimeapi.Mount{
@@ -127,13 +131,16 @@ func buildHygonRuntimeMounts() []*runtimeapi.Mount {
 			Readonly:      true,
 		})
 	}
-	/*if hygonPathExists(dtkPath) {
-		mounts = append(mounts, &runtimeapi.Mount{
-			ContainerPath: dtkPath,
-			HostPath:      dtkPath,
-			Readonly:      true,
-		})
-	}*/
+	if includeDtk {
+		dtkPath := options.HostOptions.HygonDtkPath
+		if hygonPathExists(dtkPath) {
+			mounts = append(mounts, &runtimeapi.Mount{
+				ContainerPath: "/opt/hygondriver",
+				HostPath:      dtkPath,
+				Readonly:      false,
+			})
+		}
+	}
 	return mounts
 }
 
@@ -166,7 +173,7 @@ func buildHygonRuntimeEnvs(indices []string) []*runtimeapi.KeyValue {
 }
 
 func (m *hygonDCUManager) buildHygonExtraConfigures(indices []string) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
-	return buildHygonRuntimeEnvs(indices), buildHygonRuntimeMounts()
+	return buildHygonRuntimeEnvs(indices), buildHygonRuntimeMounts(false)
 }
 
 func hygonCommonDevices() []*runtimeapi.Device {
@@ -185,6 +192,10 @@ func hygonCommonDevices() []*runtimeapi.Device {
 }
 
 func (m *hygonDCUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	return m.newHygonDrmContainerDevices(dev, true)
+}
+
+func (m *hygonDCUManager) newHygonDrmContainerDevices(dev *hostapi.ContainerDevice, includeCard bool) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
 	if dev.IsolatedDevice == nil {
 		return nil, nil, errors.Errorf("isolated device is nil")
 	}
@@ -201,13 +212,25 @@ func (m *hygonDCUManager) NewContainerDevices(input *hostapi.ContainerCreateInpu
 		return nil, nil, errors.Errorf("hygon dcu %s has empty render path", dev.IsolatedDevice.Id)
 	}
 	perms := "rwm"
-	ctrDevs := []*runtimeapi.Device{
-		{
-			ContainerPath: renderPath,
-			HostPath:      renderPath,
-			Permissions:   perms,
-		},
+	ctrDevs := []*runtimeapi.Device{}
+	if includeCard {
+		cardPath := dcuDev.GetCardPath()
+		if cardPath == "" {
+			cardPath = hygonCardPathForRender(renderPath)
+		}
+		if cardPath != "" && hygonPathExists(cardPath) {
+			ctrDevs = append(ctrDevs, &runtimeapi.Device{
+				ContainerPath: cardPath,
+				HostPath:      cardPath,
+				Permissions:   "rw",
+			})
+		}
 	}
+	ctrDevs = append(ctrDevs, &runtimeapi.Device{
+		ContainerPath: renderPath,
+		HostPath:      renderPath,
+		Permissions:   perms,
+	})
 	return ctrDevs, hygonCommonDevices(), nil
 }
 
@@ -294,6 +317,7 @@ func getHygonDCUs(manager isolated_device.IContainerDeviceManager, sharingMode s
 			BaseDevice:   NewBaseDevice(pciDev, computeapi.GPU_TYPE, strconv.Itoa(idx), sharingMode, 1),
 			gpuIndex:     idx,
 			renderPath:   renderPath,
+			cardPath:     hygonCardPathForRender(renderPath),
 			memorySize:   memSize,
 			computeUnits: computeUnits,
 		}

pkg/hostman/isolated_device/container_device/hygon_dcu_hami.go

@@ -16,10 +16,9 @@ package container_device
 
 import (
 	"fmt"
+	"os"
 	"path"
-	"regexp"
 	"strconv"
-	"strings"
 	"sync"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -32,26 +31,37 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/hostman/options"
-	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func init() {
 	isolated_device.RegisterContainerDeviceManager(newHygonDCUHamiManager())
 }
 
+type hygonVdevAllocation struct {
+	cacheDir string
+	vdevIdx  int
+	pipeID   int
+	devIdx   int
+	coremsk1 string
+	coremsk2 string
+}
+
 type hygonDCUHamiManager struct {
 	*hygonDCUManager
 
-	vdevMu    sync.Mutex
-	allocated map[string]int // containerDeviceId -> vdevIndex
-	vdevInUse map[int]string // vdevIndex -> containerDeviceId
+	mu        sync.Mutex
+	vidx      [hygonMaxVdevIdx]bool
+	pipeid    map[int][hygonMaxPipePerDev]bool
+	coremask  map[int][2]string
+	allocated map[string]*hygonVdevAllocation // containerDeviceId -> allocation
 }
 
 func newHygonDCUHamiManager() *hygonDCUHamiManager {
 	return &hygonDCUHamiManager{
 		hygonDCUManager: newHygonDCUManager(),
-		allocated:       make(map[string]int),
-		vdevInUse:       make(map[int]string),
+		pipeid:          make(map[int][hygonMaxPipePerDev]bool),
+		coremask:        make(map[int][2]string),
+		allocated:       make(map[string]*hygonVdevAllocation),
 	}
 }
 
@@ -63,6 +73,24 @@ func (m *hygonDCUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error)
 	return getHygonDCUs(m, computeapi.DEVICE_SHARING_MODE_HAMI)
 }
 
+func isHygonVDcuRequest(devs []*hostapi.ContainerDevice, fullMemMiB int) bool {
+	if len(devs) == 0 {
+		return false
+	}
+	if len(devs) >= 2 {
+		return false
+	}
+	dev := devs[0]
+	if dev.IsolatedDevice == nil {
+		return false
+	}
+	memLimit := dev.IsolatedDevice.MemoryLimit
+	if memLimit <= 0 {
+		memLimit = fullMemMiB
+	}
+	return memLimit < fullMemMiB
+}
+
 func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
 	if len(devs) == 0 {
 		return nil, nil
@@ -80,9 +108,17 @@ func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.Contai
 		return nil, nil
 	}
 
+	fullMemMiB := dcuDev.GetMemorySize()
+	isVDcu := isHygonVDcuRequest(devs, fullMemMiB)
+	mounts := buildHygonRuntimeMounts(isVDcu)
+
+	if !isVDcu {
+		return buildHygonRuntimeEnvs([]string{strconv.Itoa(dcuDev.GetIndex())}), mounts
+	}
+
 	memLimitMiB := dev.IsolatedDevice.MemoryLimit
 	if memLimitMiB <= 0 {
-		memLimitMiB = dcuDev.GetMemorySize()
+		memLimitMiB = fullMemMiB
 	}
 	smLimit := dev.IsolatedDevice.SmUtilLimit
 	if smLimit <= 0 {
@@ -92,148 +128,233 @@ func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.Contai
 	if computeUnits <= 0 {
 		computeUnits = 60
 	}
-	cuCount := computeUnits * smLimit / 100
-	if cuCount <= 0 {
-		cuCount = 1
+	reqCores := int32(computeUnits * smLimit / 100)
+	if reqCores <= 0 {
+		reqCores = 1
 	}
 
-	vdevIdx, err := m.ensureVdev(dev.IsolatedDevice.Id, dcuDev.GetIndex(), memLimitMiB, cuCount)
+	guestId, containerName := getHygonContainerContext()
+	if guestId == "" {
+		guestId = "unknown"
+	}
+	if containerName == "" {
+		containerName = dev.IsolatedDevice.Id
+	}
+
+	alloc, err := m.ensureVdevAllocation(
+		dev.IsolatedDevice.Id,
+		guestId,
+		containerName,
+		dcuDev,
+		memLimitMiB,
+		reqCores,
+	)
 	if err != nil {
 		log.Errorf("ensure hygon vdev for device %s: %v", dev.IsolatedDevice.Id, err)
-		return nil, nil
+		return nil, mounts
 	}
 
-	envs := buildHygonRuntimeEnvs([]string{strconv.Itoa(vdevIdx)})
-	mounts := buildHygonRuntimeMounts()
+	mounts = append(mounts, &runtimeapi.Mount{
+		ContainerPath: "/etc/vdev/docker/",
+		HostPath:      alloc.cacheDir,
+		Readonly:      false,
+	})
 
-	vdevConfHost := path.Join(options.HostOptions.HygonVdevConfDir, fmt.Sprintf("vdev%d.conf", vdevIdx))
-	vdevConfContainer := path.Join("/etc/vdev/docker", fmt.Sprintf("vdev%d.conf", vdevIdx))
-	if hygonPathExists(vdevConfHost) {
-		mounts = append(mounts, &runtimeapi.Mount{
-			ContainerPath: vdevConfContainer,
-			HostPath:      vdevConfHost,
-			Readonly:      true,
-		})
-	}
-	return envs, mounts
+	return buildHygonRuntimeEnvs([]string{strconv.Itoa(alloc.vdevIdx)}), mounts
 }
 
 func (m *hygonDCUHamiManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
-	return m.hygonDCUManager.NewContainerDevices(input, dev)
+	return m.newHygonDrmContainerDevices(dev, true)
 }
 
-func (m *hygonDCUHamiManager) ensureVdev(containerDevId string, physIdx, memMiB, computeUnits int) (int, error) {
-	m.vdevMu.Lock()
-	defer m.vdevMu.Unlock()
-
-	if vdevIdx, ok := m.allocated[containerDevId]; ok {
-		return vdevIdx, nil
+func (m *hygonDCUHamiManager) ensureCoreMask(devIdx, computeUnits int) {
+	if _, ok := m.coremask[devIdx]; ok {
+		return
 	}
-
-	vdevIdx, err := m.findOrCreateVdev(physIdx, memMiB, computeUnits)
-	if err != nil {
-		return -1, err
-	}
-	m.allocated[containerDevId] = vdevIdx
-	m.vdevInUse[vdevIdx] = containerDevId
-	return vdevIdx, nil
+	init := initHygonCoreUsage(computeUnits)
+	m.coremask[devIdx] = [2]string{init, init}
 }
 
-func (m *hygonDCUHamiManager) findOrCreateVdev(physIdx, memMiB, computeUnits int) (int, error) {
-	existing, err := m.listAvailableVdevIndices(physIdx)
-	if err != nil {
-		log.Warningf("list hygon vdev indices: %v", err)
-	}
-	for _, idx := range existing {
-		if _, used := m.vdevInUse[idx]; !used {
+func (m *hygonDCUHamiManager) allocateVdevIdx() (int, error) {
+	for idx := range m.vidx {
+		if !m.vidx[idx] {
+			m.vidx[idx] = true
 			return idx, nil
 		}
 	}
+	return 0, errors.Error("hygon vdev index out of bound (>200)")
+}
 
-	hySmiPath := options.HostOptions.HygonHySmiPath
-	out, err := procutils.NewRemoteCommandAsFarAsPossible(
-		hySmiPath, "virtual", "-create-vdevices", "1",
-		"-d", strconv.Itoa(physIdx),
-		"-vdevice-compute-units", strconv.Itoa(computeUnits),
-		"-vdevice-memory-size", strconv.Itoa(memMiB),
-	).Output()
-	if err != nil {
-		return -1, errors.Wrapf(err, "create vdev on dcu %d: %s", physIdx, out)
-	}
-
-	created, err := m.listAvailableVdevIndices(physIdx)
-	if err != nil || len(created) == 0 {
-		return physIdx, nil
-	}
-	for _, idx := range created {
-		if _, used := m.vdevInUse[idx]; !used {
+func (m *hygonDCUHamiManager) allocatePipeID(devIdx int) (int, error) {
+	pipes := m.pipeid[devIdx]
+	for idx := range pipes {
+		if !pipes[idx] {
+			pipes[idx] = true
+			m.pipeid[devIdx] = pipes
 			return idx, nil
 		}
 	}
-	return created[len(created)-1], nil
+	return 0, errors.Errorf("hygon pipe index out of bound for device %d", devIdx)
 }
 
-func (m *hygonDCUHamiManager) listAvailableVdevIndices(physIdx int) ([]int, error) {
-	hySmiPath := options.HostOptions.HygonHySmiPath
-	out, err := procutils.NewRemoteCommandAsFarAsPossible(hySmiPath, "virtual", "-show-vdevice-info").Output()
+func (m *hygonDCUHamiManager) ensureVdevAllocation(
+	containerDevId, guestId, containerName string,
+	dcuDev *hygonDCU,
+	memMiB int,
+	reqCores int32,
+) (*hygonVdevAllocation, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if alloc, ok := m.allocated[containerDevId]; ok {
+		return alloc, nil
+	}
+
+	devIdx := dcuDev.GetIndex()
+	computeUnits := dcuDev.GetComputeUnits()
+	if computeUnits <= 0 {
+		computeUnits = 60
+	}
+	m.ensureCoreMask(devIdx, computeUnits)
+
+	coremsk1, reqTmp, err := allocHygonCoreUsage(m.coremask[devIdx][0], int(reqCores))
 	if err != nil {
-		return nil, errors.Wrap(err, "hy-smi virtual -show-vdevice-info")
+		return nil, errors.Wrap(err, "alloc core mask 1")
 	}
-	return parseHySmiVdeviceIndices(string(out), physIdx), nil
+	coremsk2 := initHygonCoreUsage(computeUnits)
+	if reqTmp > 0 {
+		coremsk2, _, err = allocHygonCoreUsage(m.coremask[devIdx][1], reqTmp)
+		if err != nil {
+			return nil, errors.Wrap(err, "alloc core mask 2")
+		}
+	}
+
+	vdevIdx, err := m.allocateVdevIdx()
+	if err != nil {
+		return nil, err
+	}
+	pipeID, err := m.allocatePipeID(devIdx)
+	if err != nil {
+		m.vidx[vdevIdx] = false
+		return nil, err
+	}
+
+	pciBusId := hygonPciBusIdFromAddr(dcuDev.GetAddr())
+	dirName := hygonVgpuCacheDirName(guestId, containerName, devIdx, pipeID, vdevIdx, coremsk1, coremsk2)
+	cacheDir := path.Join(options.HostOptions.HygonVgpuCacheDir, dirName)
+
+	if err := createHygonVdevConfFile(pciBusId, coremsk1, coremsk2, reqCores, int32(memMiB), devIdx, vdevIdx, pipeID, cacheDir, "vdev0.conf"); err != nil {
+		m.vidx[vdevIdx] = false
+		pipes := m.pipeid[devIdx]
+		pipes[pipeID] = false
+		m.pipeid[devIdx] = pipes
+		return nil, err
+	}
+	vdevConfDir := options.HostOptions.HygonVdevConfDir
+	if err := createHygonVdevConfFile(pciBusId, coremsk1, coremsk2, reqCores, int32(memMiB), devIdx, vdevIdx, pipeID, vdevConfDir, fmt.Sprintf("vdev%d.conf", vdevIdx)); err != nil {
+		_ = hygonRemoveAll(cacheDir)
+		m.vidx[vdevIdx] = false
+		pipes := m.pipeid[devIdx]
+		pipes[pipeID] = false
+		m.pipeid[devIdx] = pipes
+		return nil, err
+	}
+
+	coreUsage1, err := addHygonCoreUsage(m.coremask[devIdx][0], coremsk1)
+	if err != nil {
+		return nil, errors.Wrap(err, "add core usage 1")
+	}
+	mask := m.coremask[devIdx]
+	mask[0] = coreUsage1
+	m.coremask[devIdx] = mask
+	coreUsage2, err := addHygonCoreUsage(m.coremask[devIdx][1], coremsk2)
+	if err != nil {
+		return nil, errors.Wrap(err, "add core usage 2")
+	}
+	mask = m.coremask[devIdx]
+	mask[1] = coreUsage2
+	m.coremask[devIdx] = mask
+
+	alloc := &hygonVdevAllocation{
+		cacheDir: cacheDir,
+		vdevIdx:  vdevIdx,
+		pipeID:   pipeID,
+		devIdx:   devIdx,
+		coremsk1: coremsk1,
+		coremsk2: coremsk2,
+	}
+	m.allocated[containerDevId] = alloc
+	return alloc, nil
 }
 
-func parseHySmiVdeviceIndices(output string, physIdx int) []int {
-	indices := []int{}
-	currentVdev := -1
-	currentPhys := -1
-	for _, line := range strings.Split(output, "\n") {
-		line = strings.TrimSpace(line)
-		if strings.HasPrefix(line, "Virtual Device") {
-			if m := regexp.MustCompile(`Virtual Device\s*(\d+)`).FindStringSubmatch(line); len(m) == 2 {
-				currentVdev, _ = strconv.Atoi(m[1])
-			}
+func (m *hygonDCUHamiManager) ReleaseContainerDevices(devs []*hostapi.ContainerDevice) {
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
 			continue
 		}
-		if strings.HasPrefix(line, "Actual Device:") {
-			parts := strings.SplitN(line, ":", 2)
-			if len(parts) == 2 {
-				currentPhys, _ = strconv.Atoi(strings.TrimSpace(parts[1]))
-			}
-			continue
-		}
-		if currentVdev >= 0 && currentPhys == physIdx {
-			indices = append(indices, currentVdev)
-			currentVdev = -1
-			currentPhys = -1
-		}
+		m.releaseVdev(dev.IsolatedDevice.Id)
 	}
-	if len(indices) == 0 {
-		vdevDir := options.HostOptions.HygonVdevConfDir
-		if hygonPathExists(vdevDir) {
-			entries, err := hygonReadDir(vdevDir)
-			if err == nil {
-				for _, e := range entries {
-					name := e.Name()
-					if strings.HasPrefix(name, "vdev") && strings.HasSuffix(name, ".conf") {
-						numStr := strings.TrimSuffix(strings.TrimPrefix(name, "vdev"), ".conf")
-						if idx, err := strconv.Atoi(numStr); err == nil {
-							indices = append(indices, idx)
-						}
-					}
-				}
-			}
-		}
-	}
-	return indices
 }
 
 func (m *hygonDCUHamiManager) releaseVdev(containerDevId string) {
-	m.vdevMu.Lock()
-	defer m.vdevMu.Unlock()
-	vdevIdx, ok := m.allocated[containerDevId]
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	alloc, ok := m.allocated[containerDevId]
 	if !ok {
 		return
 	}
 	delete(m.allocated, containerDevId)
-	delete(m.vdevInUse, vdevIdx)
+
+	m.vidx[alloc.vdevIdx] = false
+	if pipes, ok := m.pipeid[alloc.devIdx]; ok {
+		pipes[alloc.pipeID] = false
+		m.pipeid[alloc.devIdx] = pipes
+	}
+
+	if _, ok := m.coremask[alloc.devIdx]; ok {
+		m.rebuildCoreMaskLocked(alloc.devIdx)
+	}
+
+	if alloc.cacheDir != "" {
+		if err := hygonRemoveAll(alloc.cacheDir); err != nil {
+			log.Warningf("remove hygon vdev cache dir %s: %v", alloc.cacheDir, err)
+		}
+	}
+	vdevConfPath := path.Join(options.HostOptions.HygonVdevConfDir, fmt.Sprintf("vdev%d.conf", alloc.vdevIdx))
+	if err := hygonRemove(vdevConfPath); err != nil && !os.IsNotExist(err) {
+		log.Warningf("remove hygon vdev conf %s: %v", vdevConfPath, err)
+	}
+}
+
+func (m *hygonDCUHamiManager) rebuildCoreMaskLocked(devIdx int) {
+	dcuDev := m.findHygonDCUByIndex(devIdx)
+	computeUnits := 60
+	if dcuDev != nil && dcuDev.GetComputeUnits() > 0 {
+		computeUnits = dcuDev.GetComputeUnits()
+	}
+	init := initHygonCoreUsage(computeUnits)
+	m.coremask[devIdx] = [2]string{init, init}
+	for _, alloc := range m.allocated {
+		if alloc.devIdx != devIdx {
+			continue
+		}
+		mask := m.coremask[devIdx]
+		if usage, err := addHygonCoreUsage(mask[0], alloc.coremsk1); err == nil {
+			mask[0] = usage
+		}
+		if usage, err := addHygonCoreUsage(mask[1], alloc.coremsk2); err == nil {
+			mask[1] = usage
+		}
+		m.coremask[devIdx] = mask
+	}
+}
+
+func (m *hygonDCUHamiManager) findHygonDCUByIndex(devIdx int) *hygonDCU {
+	for _, dev := range hostinfo.Instance().IsolatedDeviceMan.GetDevices() {
+		if dcu, ok := dev.(*hygonDCU); ok && dcu.GetIndex() == devIdx {
+			return dcu
+		}
+	}
+	return nil
 }

pkg/hostman/isolated_device/container_device/hygon_dcu_test.go

@@ -97,19 +97,6 @@ func TestHygonModelNameFromPCIDevice(t *testing.T) {
 	})
 }
 
-func TestParseHySmiVdeviceIndices(t *testing.T) {
-	input := `Virtual Device 0:
- Actual Device: 0
- Compute units: 5
- Global memory: 4294967296 bytes
-Virtual Device 1:
- Actual Device: 0
- Compute units: 15
- Global memory: 8589934592 bytes`
-	indices := parseHySmiVdeviceIndices(input, 0)
-	assert.Equal(t, []int{0, 1}, indices)
-}
-
 func TestHygonRenderPathFromLinkWithRemote(t *testing.T) {
 	assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathFromLinkWithRemote("../renderD128", false))
 	assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathFromLinkWithRemote("/dev/dri/renderD128", true))

pkg/hostman/isolated_device/container_device/hygon_vdev.go

@@ -0,0 +1,204 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+const (
+	hygonDrmVendorID     = "0x1d94"
+	hygonDrmPrefixCard   = "card"
+	hygonDrmPrefixRender = "renderD"
+	hygonMaxVdevIdx      = 200
+	hygonMaxPipePerDev   = 20
+)
+
+type hygonDrmSlice []string
+
+func (s hygonDrmSlice) Len() int           { return len(s) }
+func (s hygonDrmSlice) Less(i, j int) bool { return s[i] < s[j] }
+func (s hygonDrmSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// listHygonDrmDevices lists Hygon DCU card and render DRM device names, sorted.
+func listHygonDrmDevices() (cards []string, renders []string, err error) {
+	const driDir = "/dev/dri"
+	if !hygonPathExists(driDir) {
+		return nil, nil, nil
+	}
+	entries, err := hygonReadDir(driDir)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "read /dev/dri")
+	}
+	names := make([]string, 0)
+	for _, entry := range entries {
+		name := entry.Name()
+		if strings.HasPrefix(name, hygonDrmPrefixCard) || strings.HasPrefix(name, hygonDrmPrefixRender) {
+			names = append(names, name)
+		}
+	}
+	sort.Sort(hygonDrmSlice(names))
+	for _, name := range names {
+		vendorPath := fmt.Sprintf("/sys/class/drm/%s/device/vendor", name)
+		vendorBytes, err := hygonReadFile(vendorPath)
+		if err != nil {
+			log.Warningf("read drm vendor %s: %v", vendorPath, err)
+			continue
+		}
+		vendorID := strings.TrimSpace(string(vendorBytes))
+		if vendorID != hygonDrmVendorID {
+			continue
+		}
+		if strings.HasPrefix(name, hygonDrmPrefixCard) {
+			cards = append(cards, name)
+		}
+		if strings.HasPrefix(name, hygonDrmPrefixRender) {
+			renders = append(renders, name)
+		}
+	}
+	if len(cards) != len(renders) {
+		return cards, renders, errors.Errorf("hygon drm card count %d != render count %d", len(cards), len(renders))
+	}
+	return cards, renders, nil
+}
+
+func hygonCardPathForRender(renderPath string) string {
+	renderBase := path.Base(renderPath)
+	cards, renders, err := listHygonDrmDevices()
+	if err != nil {
+		log.Warningf("list hygon drm devices: %v", err)
+		return ""
+	}
+	for i, render := range renders {
+		if render == renderBase {
+			return path.Join("/dev/dri", cards[i])
+		}
+	}
+	return ""
+}
+
+func hygonPciBusIdFromAddr(pciAddr string) string {
+	pciAddr = strings.TrimSpace(pciAddr)
+	if pciAddr == "" {
+		return ""
+	}
+	if strings.HasPrefix(pciAddr, "0000:") {
+		return pciAddr
+	}
+	return "0000:" + pciAddr
+}
+
+func createHygonVdevConfFile(pciBusId, coremsk1, coremsk2 string, reqcores, memMiB int32, deviceID, vdevIdx, pipeID int, dir, fileName string) error {
+	content := fmt.Sprintf("PciBusId: %s\n", pciBusId)
+	content += fmt.Sprintf("cu_mask: 0x%s\n", coremsk1)
+	content += fmt.Sprintf("cu_mask: 0x%s\n", coremsk2)
+	content += fmt.Sprintf("cu_count: %d\n", reqcores)
+	content += fmt.Sprintf("mem: %d MiB\n", memMiB)
+	content += fmt.Sprintf("device_id: %d\n", deviceID)
+	content += fmt.Sprintf("vdev_id: %d\n", vdevIdx)
+	content += fmt.Sprintf("pipe_id: %d\n", pipeID)
+	content += "enable: 1\n"
+
+	if err := hygonMkdirAll(dir, 0o777); err != nil {
+		return errors.Wrapf(err, "mkdir %s", dir)
+	}
+	filePath := path.Join(dir, fileName)
+	if err := hygonWriteFile(filePath, []byte(content), 0o666); err != nil {
+		return errors.Wrapf(err, "write vdev conf %s", filePath)
+	}
+	log.Infof("created hygon vdev conf: %s", filePath)
+	return nil
+}
+
+func hygonVgpuCacheDirName(guestId, containerName string, devIdx, pipeID, vdevIdx int, coremsk1, coremsk2 string) string {
+	return fmt.Sprintf("%s_%s_%d_%d_%d_%s_%s", guestId, containerName, devIdx, pipeID, vdevIdx, coremsk1, coremsk2)
+}
+
+func hygonReadFile(filePath string) ([]byte, error) {
+	if hygonUseRemoteFS() {
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("cat", filePath).Output()
+		if err != nil {
+			return nil, err
+		}
+		return out, nil
+	}
+	return os.ReadFile(filePath)
+}
+
+func hygonMkdirAll(dir string, perm os.FileMode) error {
+	if hygonUseRemoteFS() {
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output()
+		if err != nil {
+			return errors.Wrapf(err, "remote mkdir %s: %s", dir, out)
+		}
+		if perm != 0 {
+			out, err = procutils.NewRemoteCommandAsFarAsPossible("chmod", fmt.Sprintf("%o", perm), dir).Output()
+			if err != nil {
+				return errors.Wrapf(err, "remote chmod %s: %s", dir, out)
+			}
+		}
+		return nil
+	}
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return err
+	}
+	return os.Chmod(dir, perm)
+}
+
+func hygonWriteFile(filePath string, content []byte, perm os.FileMode) error {
+	if hygonUseRemoteFS() {
+		b64 := base64.StdEncoding.EncodeToString(content)
+		script := fmt.Sprintf("mkdir -p $(dirname %q) && echo %q | base64 -d > %q && chmod %o %q",
+			filePath, b64, filePath, perm, filePath)
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", script).Output()
+		if err != nil {
+			return errors.Wrapf(err, "remote write %s: %s", filePath, out)
+		}
+		return nil
+	}
+	return os.WriteFile(filePath, content, perm)
+}
+
+func hygonRemoveAll(path string) error {
+	if hygonUseRemoteFS() {
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("rm", "-rf", path).Output()
+		if err != nil {
+			return errors.Wrapf(err, "remote rm -rf %s: %s", path, out)
+		}
+		return nil
+	}
+	return os.RemoveAll(path)
+}
+
+func hygonRemove(filePath string) error {
+	if hygonUseRemoteFS() {
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("rm", "-f", filePath).Output()
+		if err != nil {
+			return errors.Wrapf(err, "remote rm %s: %s", filePath, out)
+		}
+		return nil
+	}
+	return os.Remove(filePath)
+}

pkg/hostman/isolated_device/container_device/net_int_device.go

@@ -21,11 +21,11 @@ import (
 	"strings"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
-	"yunion.io/x/onecloud/pkg/apis/compute"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/httperrors"

pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go

@@ -21,11 +21,12 @@ import (
 	"strings"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/pkg/errors"
+
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
-	"yunion.io/x/pkg/errors"
-
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 )
 

pkg/hostman/isolated_device/container_device/vastaitech_gpu.go

@@ -22,10 +22,10 @@ import (
 	"strings"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
-	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 
 	"yunion.io/x/pkg/errors"
 
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	fileutils "yunion.io/x/onecloud/pkg/util/fileutils2"

pkg/hostman/options/options.go

@@ -280,11 +280,12 @@ type SHostOptions struct {
 	AscendNpuHamiLibvnpuPath     string `help:"ascend npu hami libvnpu.so path" default:"/opt/cloud/hami/libvnpu.so"`
 
 	EnableContainerHygonDCU     bool   `help:"enable container hygon dcu" default:"true"`
-	EnableContainerHygonDCUHAMI bool   `help:"enable container hygon dcu hami" default:"false"`
+	EnableContainerHygonDCUHami bool   `help:"enable container hygon dcu hami" default:"false"`
 	HygonHyhalPath              string `help:"hygon hyhal driver path" default:"/opt/hyhal"`
 	HygonDtkPath                string `help:"hygon dtk toolkit path" default:"/opt/dtk"`
 	HygonHySmiPath              string `help:"hygon hy-smi path" default:"/opt/hyhal/bin/hy-smi"`
 	HygonVdevConfDir            string `help:"hygon vdcu config directory" default:"/etc/vdev"`
+	HygonVgpuCacheDir           string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"`
 
 	EnableDirtyRecoverySeconds int  `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"`
 	EnableContainerCniPortmap  bool `help:"Use container cni portmap plugin" default:"false"`

pkg/llm/models/llm_aiproxy_sync.go

@@ -8,6 +8,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/printutils"
 
 	apapi "yunion.io/x/onecloud/pkg/apis/aiproxy"
 	api "yunion.io/x/onecloud/pkg/apis/llm"
@@ -18,7 +19,6 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
-	"yunion.io/x/pkg/util/printutils"
 )
 
 // aiproxyPlaceholderAPIKey is used for local inference backends (vLLM/Ollama/SGLang)

pkg/llm/service/benchmark_handler.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/httperrors"

pkg/mcp-server/adapters/adapter.go

@@ -18,12 +18,14 @@ import (
 	"context"
 
 	"github.com/golang-plus/errors"
+
+	"yunion.io/x/pkg/gotypes"
+
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcp-server/options"
-	"yunion.io/x/pkg/gotypes"
 )
 
 // CloudpodsAdapter 负责 Cloudpods 认证并创建 mcclient.ClientSession，供 climc 工具执行使用。

pkg/mcp-server/climcgen/schema_test.go

@@ -19,10 +19,9 @@ import (
 	"strings"
 	"testing"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/compute"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/image"
-
-	"yunion.io/x/onecloud/cmd/climc/shell"
 )
 
 func TestBuildInputSchemaUsesMcpTags(t *testing.T) {