From c2fbb426f6e6e077dc8dc20ad093f47325610c76 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 4 Aug 2026 10:21:09 +0800 Subject: [PATCH] feat(host): support Hygon DCU HAMI vdev core allocation (#25274) --- cmd/climc/shell/llm/llm_router_agent.go | 1 + pkg/aiproxy/chatlog/chatlog.go | 1 + pkg/aiproxy/extensions/visual/messages.go | 2 +- pkg/aiproxy/extensions/visual/responses.go | 2 +- pkg/aiproxy/ft/interactive.go | 1 + pkg/aiproxy/models/ai_providers_test.go | 1 + pkg/aiproxy/providers/registry.go | 6 +- pkg/hostman/guestman/pod.go | 42 ++- pkg/hostman/hostinfo/hostinfo.go | 4 +- .../isolated_device/container_device.go | 5 + .../container_device/cph_amd_gpu.go | 2 +- .../container_device/cph_aosp_binder.go | 2 +- .../container_device/hygon_context.go | 45 +++ .../container_device/hygon_corealloc.go | 84 +++++ .../container_device/hygon_corealloc_test.go | 61 +++ .../container_device/hygon_dcu.go | 56 ++- .../container_device/hygon_dcu_hami.go | 351 ++++++++++++------ .../container_device/hygon_dcu_test.go | 13 - .../container_device/hygon_vdev.go | 204 ++++++++++ .../container_device/net_int_device.go | 2 +- .../container_device/nvidia_gpu_share.go | 5 +- .../container_device/vastaitech_gpu.go | 2 +- pkg/hostman/options/options.go | 3 +- pkg/llm/models/llm_aiproxy_sync.go | 2 +- pkg/llm/service/benchmark_handler.go | 1 + pkg/mcp-server/adapters/adapter.go | 4 +- pkg/mcp-server/climcgen/schema_test.go | 3 +- 27 files changed, 741 insertions(+), 164 deletions(-) create mode 100644 pkg/hostman/isolated_device/container_device/hygon_context.go create mode 100644 pkg/hostman/isolated_device/container_device/hygon_corealloc.go create mode 100644 pkg/hostman/isolated_device/container_device/hygon_corealloc_test.go create mode 100644 pkg/hostman/isolated_device/container_device/hygon_vdev.go diff --git a/cmd/climc/shell/llm/llm_router_agent.go b/cmd/climc/shell/llm/llm_router_agent.go index 2f6ae3e8f0..46f5122db1 100644 --- a/cmd/climc/shell/llm/llm_router_agent.go +++ b/cmd/climc/shell/llm/llm_router_agent.go @@ -7,6 +7,7 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/cmd/climc/shell" "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" diff --git a/pkg/aiproxy/chatlog/chatlog.go b/pkg/aiproxy/chatlog/chatlog.go index d2c995236a..a0b0f66dab 100644 --- a/pkg/aiproxy/chatlog/chatlog.go +++ b/pkg/aiproxy/chatlog/chatlog.go @@ -30,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "yunion.io/x/log" ) diff --git a/pkg/aiproxy/extensions/visual/messages.go b/pkg/aiproxy/extensions/visual/messages.go index 4ac4baf0bb..f8145dfdfb 100644 --- a/pkg/aiproxy/extensions/visual/messages.go +++ b/pkg/aiproxy/extensions/visual/messages.go @@ -21,10 +21,10 @@ import ( "strings" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/aiproxy/models" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + "yunion.io/x/onecloud/pkg/mcclient/auth" ) var ErrVisualStreamingUnsupported = fmt.Errorf("visual extension does not support streaming") diff --git a/pkg/aiproxy/extensions/visual/responses.go b/pkg/aiproxy/extensions/visual/responses.go index cf9c02fb20..4a6aec3b8c 100644 --- a/pkg/aiproxy/extensions/visual/responses.go +++ b/pkg/aiproxy/extensions/visual/responses.go @@ -19,10 +19,10 @@ import ( "fmt" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/aiproxy/models" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + "yunion.io/x/onecloud/pkg/mcclient/auth" ) // ShouldHandle reports whether the Responses visual orchestration path should run. diff --git a/pkg/aiproxy/ft/interactive.go b/pkg/aiproxy/ft/interactive.go index d8177070db..fbcaf9921e 100644 --- a/pkg/aiproxy/ft/interactive.go +++ b/pkg/aiproxy/ft/interactive.go @@ -22,6 +22,7 @@ import ( "strings" "golang.org/x/term" + "yunion.io/x/pkg/errors" ) diff --git a/pkg/aiproxy/models/ai_providers_test.go b/pkg/aiproxy/models/ai_providers_test.go index 71c8f9a37f..4f89537f69 100644 --- a/pkg/aiproxy/models/ai_providers_test.go +++ b/pkg/aiproxy/models/ai_providers_test.go @@ -6,6 +6,7 @@ import ( "testing" "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" "yunion.io/x/onecloud/pkg/cloudcommon/db" ) diff --git a/pkg/aiproxy/providers/registry.go b/pkg/aiproxy/providers/registry.go index 0927cac728..01549928f3 100644 --- a/pkg/aiproxy/providers/registry.go +++ b/pkg/aiproxy/providers/registry.go @@ -19,15 +19,13 @@ import ( "sync" "yunion.io/x/onecloud/pkg/aiproxy/providerapi" - api "yunion.io/x/onecloud/pkg/apis/aiproxy" - // "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" // uncommon - "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" - // "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" // uncommon + "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" // "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" // uncommon // "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" // uncommon // "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" // uncommon "yunion.io/x/onecloud/pkg/aiproxy/providers/gemini" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" "yunion.io/x/onecloud/pkg/aiproxy/providers/vllm" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" // "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" // uncommon ) var ( diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 2cb97d960e..481096b384 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -57,6 +57,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device" _ "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device/cdi" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" @@ -2253,7 +2254,11 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie } ctrCfg.Devices = append(ctrCfg.Devices, ctrDevs...) } - if err := s.getIsolatedDeviceExtraConfig(spec, ctrCfg); err != nil { + if err := func() error { + container_device.SetHygonContainerContext(s.GetId(), input.Name) + defer container_device.ClearHygonContainerContext() + return s.getIsolatedDeviceExtraConfig(spec, ctrCfg) + }(); err != nil { return "", err } } else { @@ -2447,6 +2452,38 @@ func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.Container return nil } +func (s *sPodGuestInstance) releaseIsolatedContainerDevices(devs []*hostapi.ContainerDevice) { + devsByMan := map[isolated_device.IContainerDeviceManager][]*hostapi.ContainerDevice{} + manOrder := []isolated_device.IContainerDeviceManager{} + for i := range devs { + dev := devs[i] + if dev.IsolatedDevice == nil { + continue + } + devType := dev.IsolatedDevice.DeviceType + if devType != computeapi.GPU_TYPE && devType != computeapi.NPU_TYPE { + continue + } + if dev.IsolatedDevice.IsCDIUsed() || len(dev.IsolatedDevice.OnlyEnv) > 0 { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + continue + } + devMan := iDev.GetContainerDeviceManager() + if _, ok := devsByMan[devMan]; !ok { + manOrder = append(manOrder, devMan) + } + devsByMan[devMan] = append(devsByMan[devMan], dev) + } + for _, devMan := range manOrder { + if rel, ok := devMan.(isolated_device.IContainerDeviceReleaseManager); ok { + rel.ReleaseContainerDevices(devsByMan[devMan]) + } + } +} + func (s *sPodGuestInstance) getContainerSystemCpusDir(ctrId string) string { rootFsPath, _ := s.GetRootFsMountPath(ctrId) if rootFsPath != "" { @@ -2607,6 +2644,9 @@ func (s *sPodGuestInstance) ensureContainerSystemCpufreqHostDir(cpuDir, hostCpuP } func (s *sPodGuestInstance) DeleteContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string) (jsonutils.JSONObject, error) { + if ctr := s.GetContainerById(ctrId); ctr != nil { + s.releaseIsolatedContainerDevices(ctr.Spec.Devices) + } criId, err := s.getContainerCRIId(ctrId) if err != nil && errors.Cause(err) != errors.ErrNotFound { return nil, errors.Wrap(err, "getContainerCRIId") diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 4be35485cb..504067da1f 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2347,7 +2347,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { options.HostOptions.HostType, h.IsContainerHost(), h.IsKvmSupport()) log.Infof("==== probeSyncIsolatedDevices hygon config: enableDCU=%v enableHAMI=%v hySmiPath=%s hyhalPath=%s dtkPath=%s", options.HostOptions.EnableContainerHygonDCU, - options.HostOptions.EnableContainerHygonDCUHAMI, + options.HostOptions.EnableContainerHygonDCUHami, options.HostOptions.HygonHySmiPath, options.HostOptions.HygonHyhalPath, options.HostOptions.HygonDtkPath, @@ -2361,7 +2361,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { EnableContainerAscendNpu: options.HostOptions.EnableContainerAscendNPU, EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami, EnableContainerHygonDCU: options.HostOptions.EnableContainerHygonDCU, - EnableContainerHygonDCUHAMI: options.HostOptions.EnableContainerHygonDCUHAMI, + EnableContainerHygonDCUHAMI: options.HostOptions.EnableContainerHygonDCUHami, EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, SriovNics: sriovNics, OvsOffloadNics: offloadNics, diff --git a/pkg/hostman/isolated_device/container_device.go b/pkg/hostman/isolated_device/container_device.go index b77a5d3c10..85c6d70c76 100644 --- a/pkg/hostman/isolated_device/container_device.go +++ b/pkg/hostman/isolated_device/container_device.go @@ -100,6 +100,11 @@ type IContainerDeviceManager interface { GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) } +// IContainerDeviceReleaseManager is implemented by device managers that need cleanup on container delete. +type IContainerDeviceReleaseManager interface { + ReleaseContainerDevices(devs []*hostapi.ContainerDevice) +} + type IContainerCDIManager interface { GetKind() apis.ContainerCDIKind GetSpecFilePath() string diff --git a/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go b/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go index 0521d9c4f6..0f6e26f157 100644 --- a/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go +++ b/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go @@ -18,10 +18,10 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" ) diff --git a/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go b/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go index f532a81e45..ed48700530 100644 --- a/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go +++ b/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go @@ -20,10 +20,10 @@ import ( "path/filepath" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/util/fileutils2" diff --git a/pkg/hostman/isolated_device/container_device/hygon_context.go b/pkg/hostman/isolated_device/container_device/hygon_context.go new file mode 100644 index 0000000000..1f24e4748a --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/hygon_context.go @@ -0,0 +1,45 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import "sync" + +var ( + hygonCtxMu sync.Mutex + hygonPendingGuestId string + hygonPendingCtrName string +) + +// SetHygonContainerContext stores guest/container identity for vdev cache directory naming. +func SetHygonContainerContext(guestId, containerName string) { + hygonCtxMu.Lock() + defer hygonCtxMu.Unlock() + hygonPendingGuestId = guestId + hygonPendingCtrName = containerName +} + +// ClearHygonContainerContext clears pending guest/container identity. +func ClearHygonContainerContext() { + hygonCtxMu.Lock() + defer hygonCtxMu.Unlock() + hygonPendingGuestId = "" + hygonPendingCtrName = "" +} + +func getHygonContainerContext() (guestId, containerName string) { + hygonCtxMu.Lock() + defer hygonCtxMu.Unlock() + return hygonPendingGuestId, hygonPendingCtrName +} diff --git a/pkg/hostman/isolated_device/container_device/hygon_corealloc.go b/pkg/hostman/isolated_device/container_device/hygon_corealloc.go new file mode 100644 index 0000000000..3d7b4b0a2d --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/hygon_corealloc.go @@ -0,0 +1,84 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "strconv" + "strings" +) + +// initHygonCoreUsage returns a zero-filled hex string for CU bitmap allocation. +func initHygonCoreUsage(_ int) string { + return strings.Repeat("0", 16) +} + +func addHygonCoreUsage(tot string, c string) (string, error) { + i := 0 + res := "" + for { + if i >= len(tot) || tot[i] == 0 { + break + } + left, _ := strconv.ParseInt(string(tot[i]), 16, 0) + right, _ := strconv.ParseInt(string(c[i]), 16, 0) + merged := int(left | right) + res = fmt.Sprintf("%s%x", res, merged) + i++ + } + return res, nil +} + +func hygonByteAlloc(b int, req int) (int, int) { + if req == 0 { + return 0, 0 + } + remains := req + leftstr := fmt.Sprintf("%b", b) + for len(leftstr) < 4 { + leftstr = "0" + leftstr + } + res := 0 + i := 0 + for i < len(leftstr) { + res = res * 2 + if leftstr[i] == '0' && remains > 0 { + remains-- + res = res + 1 + } + if remains <= 0 { + break + } + i++ + } + return res, remains +} + +func allocHygonCoreUsage(tot string, req int) (string, int, error) { + i := len(tot) - 1 + res := "" + remains := req + for { + if i < 0 { + break + } + left, _ := strconv.ParseInt(string(tot[i]), 16, 0) + alloc, newRemains := hygonByteAlloc(int(left), remains) + remains = newRemains + res = fmt.Sprintf("%x%s", alloc, res) + i-- + } + return res, remains, nil +} diff --git a/pkg/hostman/isolated_device/container_device/hygon_corealloc_test.go b/pkg/hostman/isolated_device/container_device/hygon_corealloc_test.go new file mode 100644 index 0000000000..7a2cdb0d17 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/hygon_corealloc_test.go @@ -0,0 +1,61 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + hostapi "yunion.io/x/onecloud/pkg/apis/host" +) + +func TestInitHygonCoreUsage(t *testing.T) { + assert.Equal(t, "0000000000000000", initHygonCoreUsage(60)) +} + +func TestAllocHygonCoreUsage(t *testing.T) { + tot := initHygonCoreUsage(60) + mask, remains, err := allocHygonCoreUsage(tot, 4) + assert.NoError(t, err) + assert.Equal(t, 0, remains) + assert.NotEmpty(t, mask) + + merged, err := addHygonCoreUsage(tot, mask) + assert.NoError(t, err) + assert.Equal(t, mask, merged) +} + +func TestHygonPciBusIdFromAddr(t *testing.T) { + assert.Equal(t, "0000:09:00.0", hygonPciBusIdFromAddr("09:00.0")) + assert.Equal(t, "0000:09:00.0", hygonPciBusIdFromAddr("0000:09:00.0")) +} + +func TestIsHygonVDcuRequest(t *testing.T) { + dev := &hostapi.ContainerDevice{ + IsolatedDevice: &hostapi.ContainerIsolatedDevice{ + MemoryLimit: 8192, + }, + } + assert.True(t, isHygonVDcuRequest([]*hostapi.ContainerDevice{dev}, 65520)) + assert.False(t, isHygonVDcuRequest([]*hostapi.ContainerDevice{dev, dev}, 65520)) + dev.IsolatedDevice.MemoryLimit = 65520 + assert.False(t, isHygonVDcuRequest([]*hostapi.ContainerDevice{dev}, 65520)) +} + +func TestHygonVgpuCacheDirName(t *testing.T) { + name := hygonVgpuCacheDirName("guest1", "ctr1", 0, 1, 2, "abc", "def") + assert.Equal(t, "guest1_ctr1_0_1_2_abc_def", name) +} diff --git a/pkg/hostman/isolated_device/container_device/hygon_dcu.go b/pkg/hostman/isolated_device/container_device/hygon_dcu.go index cf8f92e4eb..366f576f51 100644 --- a/pkg/hostman/isolated_device/container_device/hygon_dcu.go +++ b/pkg/hostman/isolated_device/container_device/hygon_dcu.go @@ -70,6 +70,7 @@ type hygonDCU struct { *BaseDevice gpuIndex int renderPath string + cardPath string memorySize int computeUnits int } @@ -86,6 +87,10 @@ func (dev *hygonDCU) GetRenderPath() string { return dev.renderPath } +func (dev *hygonDCU) GetCardPath() string { + return dev.cardPath +} + func (dev *hygonDCU) GetComputeUnits() int { return dev.computeUnits } @@ -116,9 +121,8 @@ func (m *hygonDCUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerD return m.buildHygonExtraConfigures(indices) } -func buildHygonRuntimeMounts() []*runtimeapi.Mount { +func buildHygonRuntimeMounts(includeDtk bool) []*runtimeapi.Mount { hyhalPath := options.HostOptions.HygonHyhalPath - //dtkPath := options.HostOptions.HygonDtkPath mounts := []*runtimeapi.Mount{} if hygonPathExists(hyhalPath) { mounts = append(mounts, &runtimeapi.Mount{ @@ -127,13 +131,16 @@ func buildHygonRuntimeMounts() []*runtimeapi.Mount { Readonly: true, }) } - /*if hygonPathExists(dtkPath) { - mounts = append(mounts, &runtimeapi.Mount{ - ContainerPath: dtkPath, - HostPath: dtkPath, - Readonly: true, - }) - }*/ + if includeDtk { + dtkPath := options.HostOptions.HygonDtkPath + if hygonPathExists(dtkPath) { + mounts = append(mounts, &runtimeapi.Mount{ + ContainerPath: "/opt/hygondriver", + HostPath: dtkPath, + Readonly: false, + }) + } + } return mounts } @@ -166,7 +173,7 @@ func buildHygonRuntimeEnvs(indices []string) []*runtimeapi.KeyValue { } func (m *hygonDCUManager) buildHygonExtraConfigures(indices []string) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { - return buildHygonRuntimeEnvs(indices), buildHygonRuntimeMounts() + return buildHygonRuntimeEnvs(indices), buildHygonRuntimeMounts(false) } func hygonCommonDevices() []*runtimeapi.Device { @@ -185,6 +192,10 @@ func hygonCommonDevices() []*runtimeapi.Device { } func (m *hygonDCUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + return m.newHygonDrmContainerDevices(dev, true) +} + +func (m *hygonDCUManager) newHygonDrmContainerDevices(dev *hostapi.ContainerDevice, includeCard bool) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { if dev.IsolatedDevice == nil { return nil, nil, errors.Errorf("isolated device is nil") } @@ -201,13 +212,25 @@ func (m *hygonDCUManager) NewContainerDevices(input *hostapi.ContainerCreateInpu return nil, nil, errors.Errorf("hygon dcu %s has empty render path", dev.IsolatedDevice.Id) } perms := "rwm" - ctrDevs := []*runtimeapi.Device{ - { - ContainerPath: renderPath, - HostPath: renderPath, - Permissions: perms, - }, + ctrDevs := []*runtimeapi.Device{} + if includeCard { + cardPath := dcuDev.GetCardPath() + if cardPath == "" { + cardPath = hygonCardPathForRender(renderPath) + } + if cardPath != "" && hygonPathExists(cardPath) { + ctrDevs = append(ctrDevs, &runtimeapi.Device{ + ContainerPath: cardPath, + HostPath: cardPath, + Permissions: "rw", + }) + } } + ctrDevs = append(ctrDevs, &runtimeapi.Device{ + ContainerPath: renderPath, + HostPath: renderPath, + Permissions: perms, + }) return ctrDevs, hygonCommonDevices(), nil } @@ -294,6 +317,7 @@ func getHygonDCUs(manager isolated_device.IContainerDeviceManager, sharingMode s BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, strconv.Itoa(idx), sharingMode, 1), gpuIndex: idx, renderPath: renderPath, + cardPath: hygonCardPathForRender(renderPath), memorySize: memSize, computeUnits: computeUnits, } diff --git a/pkg/hostman/isolated_device/container_device/hygon_dcu_hami.go b/pkg/hostman/isolated_device/container_device/hygon_dcu_hami.go index 8c5d827084..99be40c826 100644 --- a/pkg/hostman/isolated_device/container_device/hygon_dcu_hami.go +++ b/pkg/hostman/isolated_device/container_device/hygon_dcu_hami.go @@ -16,10 +16,9 @@ package container_device import ( "fmt" + "os" "path" - "regexp" "strconv" - "strings" "sync" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -32,26 +31,37 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/options" - "yunion.io/x/onecloud/pkg/util/procutils" ) func init() { isolated_device.RegisterContainerDeviceManager(newHygonDCUHamiManager()) } +type hygonVdevAllocation struct { + cacheDir string + vdevIdx int + pipeID int + devIdx int + coremsk1 string + coremsk2 string +} + type hygonDCUHamiManager struct { *hygonDCUManager - vdevMu sync.Mutex - allocated map[string]int // containerDeviceId -> vdevIndex - vdevInUse map[int]string // vdevIndex -> containerDeviceId + mu sync.Mutex + vidx [hygonMaxVdevIdx]bool + pipeid map[int][hygonMaxPipePerDev]bool + coremask map[int][2]string + allocated map[string]*hygonVdevAllocation // containerDeviceId -> allocation } func newHygonDCUHamiManager() *hygonDCUHamiManager { return &hygonDCUHamiManager{ hygonDCUManager: newHygonDCUManager(), - allocated: make(map[string]int), - vdevInUse: make(map[int]string), + pipeid: make(map[int][hygonMaxPipePerDev]bool), + coremask: make(map[int][2]string), + allocated: make(map[string]*hygonVdevAllocation), } } @@ -63,6 +73,24 @@ func (m *hygonDCUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error) return getHygonDCUs(m, computeapi.DEVICE_SHARING_MODE_HAMI) } +func isHygonVDcuRequest(devs []*hostapi.ContainerDevice, fullMemMiB int) bool { + if len(devs) == 0 { + return false + } + if len(devs) >= 2 { + return false + } + dev := devs[0] + if dev.IsolatedDevice == nil { + return false + } + memLimit := dev.IsolatedDevice.MemoryLimit + if memLimit <= 0 { + memLimit = fullMemMiB + } + return memLimit < fullMemMiB +} + func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { if len(devs) == 0 { return nil, nil @@ -80,9 +108,17 @@ func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.Contai return nil, nil } + fullMemMiB := dcuDev.GetMemorySize() + isVDcu := isHygonVDcuRequest(devs, fullMemMiB) + mounts := buildHygonRuntimeMounts(isVDcu) + + if !isVDcu { + return buildHygonRuntimeEnvs([]string{strconv.Itoa(dcuDev.GetIndex())}), mounts + } + memLimitMiB := dev.IsolatedDevice.MemoryLimit if memLimitMiB <= 0 { - memLimitMiB = dcuDev.GetMemorySize() + memLimitMiB = fullMemMiB } smLimit := dev.IsolatedDevice.SmUtilLimit if smLimit <= 0 { @@ -92,148 +128,233 @@ func (m *hygonDCUHamiManager) GetContainerExtraConfigures(devs []*hostapi.Contai if computeUnits <= 0 { computeUnits = 60 } - cuCount := computeUnits * smLimit / 100 - if cuCount <= 0 { - cuCount = 1 + reqCores := int32(computeUnits * smLimit / 100) + if reqCores <= 0 { + reqCores = 1 } - vdevIdx, err := m.ensureVdev(dev.IsolatedDevice.Id, dcuDev.GetIndex(), memLimitMiB, cuCount) + guestId, containerName := getHygonContainerContext() + if guestId == "" { + guestId = "unknown" + } + if containerName == "" { + containerName = dev.IsolatedDevice.Id + } + + alloc, err := m.ensureVdevAllocation( + dev.IsolatedDevice.Id, + guestId, + containerName, + dcuDev, + memLimitMiB, + reqCores, + ) if err != nil { log.Errorf("ensure hygon vdev for device %s: %v", dev.IsolatedDevice.Id, err) - return nil, nil + return nil, mounts } - envs := buildHygonRuntimeEnvs([]string{strconv.Itoa(vdevIdx)}) - mounts := buildHygonRuntimeMounts() + mounts = append(mounts, &runtimeapi.Mount{ + ContainerPath: "/etc/vdev/docker/", + HostPath: alloc.cacheDir, + Readonly: false, + }) - vdevConfHost := path.Join(options.HostOptions.HygonVdevConfDir, fmt.Sprintf("vdev%d.conf", vdevIdx)) - vdevConfContainer := path.Join("/etc/vdev/docker", fmt.Sprintf("vdev%d.conf", vdevIdx)) - if hygonPathExists(vdevConfHost) { - mounts = append(mounts, &runtimeapi.Mount{ - ContainerPath: vdevConfContainer, - HostPath: vdevConfHost, - Readonly: true, - }) - } - return envs, mounts + return buildHygonRuntimeEnvs([]string{strconv.Itoa(alloc.vdevIdx)}), mounts } func (m *hygonDCUHamiManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { - return m.hygonDCUManager.NewContainerDevices(input, dev) + return m.newHygonDrmContainerDevices(dev, true) } -func (m *hygonDCUHamiManager) ensureVdev(containerDevId string, physIdx, memMiB, computeUnits int) (int, error) { - m.vdevMu.Lock() - defer m.vdevMu.Unlock() - - if vdevIdx, ok := m.allocated[containerDevId]; ok { - return vdevIdx, nil +func (m *hygonDCUHamiManager) ensureCoreMask(devIdx, computeUnits int) { + if _, ok := m.coremask[devIdx]; ok { + return } - - vdevIdx, err := m.findOrCreateVdev(physIdx, memMiB, computeUnits) - if err != nil { - return -1, err - } - m.allocated[containerDevId] = vdevIdx - m.vdevInUse[vdevIdx] = containerDevId - return vdevIdx, nil + init := initHygonCoreUsage(computeUnits) + m.coremask[devIdx] = [2]string{init, init} } -func (m *hygonDCUHamiManager) findOrCreateVdev(physIdx, memMiB, computeUnits int) (int, error) { - existing, err := m.listAvailableVdevIndices(physIdx) - if err != nil { - log.Warningf("list hygon vdev indices: %v", err) - } - for _, idx := range existing { - if _, used := m.vdevInUse[idx]; !used { +func (m *hygonDCUHamiManager) allocateVdevIdx() (int, error) { + for idx := range m.vidx { + if !m.vidx[idx] { + m.vidx[idx] = true return idx, nil } } + return 0, errors.Error("hygon vdev index out of bound (>200)") +} - hySmiPath := options.HostOptions.HygonHySmiPath - out, err := procutils.NewRemoteCommandAsFarAsPossible( - hySmiPath, "virtual", "-create-vdevices", "1", - "-d", strconv.Itoa(physIdx), - "-vdevice-compute-units", strconv.Itoa(computeUnits), - "-vdevice-memory-size", strconv.Itoa(memMiB), - ).Output() - if err != nil { - return -1, errors.Wrapf(err, "create vdev on dcu %d: %s", physIdx, out) - } - - created, err := m.listAvailableVdevIndices(physIdx) - if err != nil || len(created) == 0 { - return physIdx, nil - } - for _, idx := range created { - if _, used := m.vdevInUse[idx]; !used { +func (m *hygonDCUHamiManager) allocatePipeID(devIdx int) (int, error) { + pipes := m.pipeid[devIdx] + for idx := range pipes { + if !pipes[idx] { + pipes[idx] = true + m.pipeid[devIdx] = pipes return idx, nil } } - return created[len(created)-1], nil + return 0, errors.Errorf("hygon pipe index out of bound for device %d", devIdx) } -func (m *hygonDCUHamiManager) listAvailableVdevIndices(physIdx int) ([]int, error) { - hySmiPath := options.HostOptions.HygonHySmiPath - out, err := procutils.NewRemoteCommandAsFarAsPossible(hySmiPath, "virtual", "-show-vdevice-info").Output() +func (m *hygonDCUHamiManager) ensureVdevAllocation( + containerDevId, guestId, containerName string, + dcuDev *hygonDCU, + memMiB int, + reqCores int32, +) (*hygonVdevAllocation, error) { + m.mu.Lock() + defer m.mu.Unlock() + + if alloc, ok := m.allocated[containerDevId]; ok { + return alloc, nil + } + + devIdx := dcuDev.GetIndex() + computeUnits := dcuDev.GetComputeUnits() + if computeUnits <= 0 { + computeUnits = 60 + } + m.ensureCoreMask(devIdx, computeUnits) + + coremsk1, reqTmp, err := allocHygonCoreUsage(m.coremask[devIdx][0], int(reqCores)) if err != nil { - return nil, errors.Wrap(err, "hy-smi virtual -show-vdevice-info") + return nil, errors.Wrap(err, "alloc core mask 1") } - return parseHySmiVdeviceIndices(string(out), physIdx), nil + coremsk2 := initHygonCoreUsage(computeUnits) + if reqTmp > 0 { + coremsk2, _, err = allocHygonCoreUsage(m.coremask[devIdx][1], reqTmp) + if err != nil { + return nil, errors.Wrap(err, "alloc core mask 2") + } + } + + vdevIdx, err := m.allocateVdevIdx() + if err != nil { + return nil, err + } + pipeID, err := m.allocatePipeID(devIdx) + if err != nil { + m.vidx[vdevIdx] = false + return nil, err + } + + pciBusId := hygonPciBusIdFromAddr(dcuDev.GetAddr()) + dirName := hygonVgpuCacheDirName(guestId, containerName, devIdx, pipeID, vdevIdx, coremsk1, coremsk2) + cacheDir := path.Join(options.HostOptions.HygonVgpuCacheDir, dirName) + + if err := createHygonVdevConfFile(pciBusId, coremsk1, coremsk2, reqCores, int32(memMiB), devIdx, vdevIdx, pipeID, cacheDir, "vdev0.conf"); err != nil { + m.vidx[vdevIdx] = false + pipes := m.pipeid[devIdx] + pipes[pipeID] = false + m.pipeid[devIdx] = pipes + return nil, err + } + vdevConfDir := options.HostOptions.HygonVdevConfDir + if err := createHygonVdevConfFile(pciBusId, coremsk1, coremsk2, reqCores, int32(memMiB), devIdx, vdevIdx, pipeID, vdevConfDir, fmt.Sprintf("vdev%d.conf", vdevIdx)); err != nil { + _ = hygonRemoveAll(cacheDir) + m.vidx[vdevIdx] = false + pipes := m.pipeid[devIdx] + pipes[pipeID] = false + m.pipeid[devIdx] = pipes + return nil, err + } + + coreUsage1, err := addHygonCoreUsage(m.coremask[devIdx][0], coremsk1) + if err != nil { + return nil, errors.Wrap(err, "add core usage 1") + } + mask := m.coremask[devIdx] + mask[0] = coreUsage1 + m.coremask[devIdx] = mask + coreUsage2, err := addHygonCoreUsage(m.coremask[devIdx][1], coremsk2) + if err != nil { + return nil, errors.Wrap(err, "add core usage 2") + } + mask = m.coremask[devIdx] + mask[1] = coreUsage2 + m.coremask[devIdx] = mask + + alloc := &hygonVdevAllocation{ + cacheDir: cacheDir, + vdevIdx: vdevIdx, + pipeID: pipeID, + devIdx: devIdx, + coremsk1: coremsk1, + coremsk2: coremsk2, + } + m.allocated[containerDevId] = alloc + return alloc, nil } -func parseHySmiVdeviceIndices(output string, physIdx int) []int { - indices := []int{} - currentVdev := -1 - currentPhys := -1 - for _, line := range strings.Split(output, "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "Virtual Device") { - if m := regexp.MustCompile(`Virtual Device\s*(\d+)`).FindStringSubmatch(line); len(m) == 2 { - currentVdev, _ = strconv.Atoi(m[1]) - } +func (m *hygonDCUHamiManager) ReleaseContainerDevices(devs []*hostapi.ContainerDevice) { + for _, dev := range devs { + if dev.IsolatedDevice == nil { continue } - if strings.HasPrefix(line, "Actual Device:") { - parts := strings.SplitN(line, ":", 2) - if len(parts) == 2 { - currentPhys, _ = strconv.Atoi(strings.TrimSpace(parts[1])) - } - continue - } - if currentVdev >= 0 && currentPhys == physIdx { - indices = append(indices, currentVdev) - currentVdev = -1 - currentPhys = -1 - } + m.releaseVdev(dev.IsolatedDevice.Id) } - if len(indices) == 0 { - vdevDir := options.HostOptions.HygonVdevConfDir - if hygonPathExists(vdevDir) { - entries, err := hygonReadDir(vdevDir) - if err == nil { - for _, e := range entries { - name := e.Name() - if strings.HasPrefix(name, "vdev") && strings.HasSuffix(name, ".conf") { - numStr := strings.TrimSuffix(strings.TrimPrefix(name, "vdev"), ".conf") - if idx, err := strconv.Atoi(numStr); err == nil { - indices = append(indices, idx) - } - } - } - } - } - } - return indices } func (m *hygonDCUHamiManager) releaseVdev(containerDevId string) { - m.vdevMu.Lock() - defer m.vdevMu.Unlock() - vdevIdx, ok := m.allocated[containerDevId] + m.mu.Lock() + defer m.mu.Unlock() + + alloc, ok := m.allocated[containerDevId] if !ok { return } delete(m.allocated, containerDevId) - delete(m.vdevInUse, vdevIdx) + + m.vidx[alloc.vdevIdx] = false + if pipes, ok := m.pipeid[alloc.devIdx]; ok { + pipes[alloc.pipeID] = false + m.pipeid[alloc.devIdx] = pipes + } + + if _, ok := m.coremask[alloc.devIdx]; ok { + m.rebuildCoreMaskLocked(alloc.devIdx) + } + + if alloc.cacheDir != "" { + if err := hygonRemoveAll(alloc.cacheDir); err != nil { + log.Warningf("remove hygon vdev cache dir %s: %v", alloc.cacheDir, err) + } + } + vdevConfPath := path.Join(options.HostOptions.HygonVdevConfDir, fmt.Sprintf("vdev%d.conf", alloc.vdevIdx)) + if err := hygonRemove(vdevConfPath); err != nil && !os.IsNotExist(err) { + log.Warningf("remove hygon vdev conf %s: %v", vdevConfPath, err) + } +} + +func (m *hygonDCUHamiManager) rebuildCoreMaskLocked(devIdx int) { + dcuDev := m.findHygonDCUByIndex(devIdx) + computeUnits := 60 + if dcuDev != nil && dcuDev.GetComputeUnits() > 0 { + computeUnits = dcuDev.GetComputeUnits() + } + init := initHygonCoreUsage(computeUnits) + m.coremask[devIdx] = [2]string{init, init} + for _, alloc := range m.allocated { + if alloc.devIdx != devIdx { + continue + } + mask := m.coremask[devIdx] + if usage, err := addHygonCoreUsage(mask[0], alloc.coremsk1); err == nil { + mask[0] = usage + } + if usage, err := addHygonCoreUsage(mask[1], alloc.coremsk2); err == nil { + mask[1] = usage + } + m.coremask[devIdx] = mask + } +} + +func (m *hygonDCUHamiManager) findHygonDCUByIndex(devIdx int) *hygonDCU { + for _, dev := range hostinfo.Instance().IsolatedDeviceMan.GetDevices() { + if dcu, ok := dev.(*hygonDCU); ok && dcu.GetIndex() == devIdx { + return dcu + } + } + return nil } diff --git a/pkg/hostman/isolated_device/container_device/hygon_dcu_test.go b/pkg/hostman/isolated_device/container_device/hygon_dcu_test.go index f136b72745..be72e20bfd 100644 --- a/pkg/hostman/isolated_device/container_device/hygon_dcu_test.go +++ b/pkg/hostman/isolated_device/container_device/hygon_dcu_test.go @@ -97,19 +97,6 @@ func TestHygonModelNameFromPCIDevice(t *testing.T) { }) } -func TestParseHySmiVdeviceIndices(t *testing.T) { - input := `Virtual Device 0: - Actual Device: 0 - Compute units: 5 - Global memory: 4294967296 bytes -Virtual Device 1: - Actual Device: 0 - Compute units: 15 - Global memory: 8589934592 bytes` - indices := parseHySmiVdeviceIndices(input, 0) - assert.Equal(t, []int{0, 1}, indices) -} - func TestHygonRenderPathFromLinkWithRemote(t *testing.T) { assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathFromLinkWithRemote("../renderD128", false)) assert.Equal(t, "/dev/dri/renderD128", hygonRenderPathFromLinkWithRemote("/dev/dri/renderD128", true)) diff --git a/pkg/hostman/isolated_device/container_device/hygon_vdev.go b/pkg/hostman/isolated_device/container_device/hygon_vdev.go new file mode 100644 index 0000000000..6ceb27f830 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/hygon_vdev.go @@ -0,0 +1,204 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "encoding/base64" + "fmt" + "os" + "path" + "sort" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/util/procutils" +) + +const ( + hygonDrmVendorID = "0x1d94" + hygonDrmPrefixCard = "card" + hygonDrmPrefixRender = "renderD" + hygonMaxVdevIdx = 200 + hygonMaxPipePerDev = 20 +) + +type hygonDrmSlice []string + +func (s hygonDrmSlice) Len() int { return len(s) } +func (s hygonDrmSlice) Less(i, j int) bool { return s[i] < s[j] } +func (s hygonDrmSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// listHygonDrmDevices lists Hygon DCU card and render DRM device names, sorted. +func listHygonDrmDevices() (cards []string, renders []string, err error) { + const driDir = "/dev/dri" + if !hygonPathExists(driDir) { + return nil, nil, nil + } + entries, err := hygonReadDir(driDir) + if err != nil { + return nil, nil, errors.Wrap(err, "read /dev/dri") + } + names := make([]string, 0) + for _, entry := range entries { + name := entry.Name() + if strings.HasPrefix(name, hygonDrmPrefixCard) || strings.HasPrefix(name, hygonDrmPrefixRender) { + names = append(names, name) + } + } + sort.Sort(hygonDrmSlice(names)) + for _, name := range names { + vendorPath := fmt.Sprintf("/sys/class/drm/%s/device/vendor", name) + vendorBytes, err := hygonReadFile(vendorPath) + if err != nil { + log.Warningf("read drm vendor %s: %v", vendorPath, err) + continue + } + vendorID := strings.TrimSpace(string(vendorBytes)) + if vendorID != hygonDrmVendorID { + continue + } + if strings.HasPrefix(name, hygonDrmPrefixCard) { + cards = append(cards, name) + } + if strings.HasPrefix(name, hygonDrmPrefixRender) { + renders = append(renders, name) + } + } + if len(cards) != len(renders) { + return cards, renders, errors.Errorf("hygon drm card count %d != render count %d", len(cards), len(renders)) + } + return cards, renders, nil +} + +func hygonCardPathForRender(renderPath string) string { + renderBase := path.Base(renderPath) + cards, renders, err := listHygonDrmDevices() + if err != nil { + log.Warningf("list hygon drm devices: %v", err) + return "" + } + for i, render := range renders { + if render == renderBase { + return path.Join("/dev/dri", cards[i]) + } + } + return "" +} + +func hygonPciBusIdFromAddr(pciAddr string) string { + pciAddr = strings.TrimSpace(pciAddr) + if pciAddr == "" { + return "" + } + if strings.HasPrefix(pciAddr, "0000:") { + return pciAddr + } + return "0000:" + pciAddr +} + +func createHygonVdevConfFile(pciBusId, coremsk1, coremsk2 string, reqcores, memMiB int32, deviceID, vdevIdx, pipeID int, dir, fileName string) error { + content := fmt.Sprintf("PciBusId: %s\n", pciBusId) + content += fmt.Sprintf("cu_mask: 0x%s\n", coremsk1) + content += fmt.Sprintf("cu_mask: 0x%s\n", coremsk2) + content += fmt.Sprintf("cu_count: %d\n", reqcores) + content += fmt.Sprintf("mem: %d MiB\n", memMiB) + content += fmt.Sprintf("device_id: %d\n", deviceID) + content += fmt.Sprintf("vdev_id: %d\n", vdevIdx) + content += fmt.Sprintf("pipe_id: %d\n", pipeID) + content += "enable: 1\n" + + if err := hygonMkdirAll(dir, 0o777); err != nil { + return errors.Wrapf(err, "mkdir %s", dir) + } + filePath := path.Join(dir, fileName) + if err := hygonWriteFile(filePath, []byte(content), 0o666); err != nil { + return errors.Wrapf(err, "write vdev conf %s", filePath) + } + log.Infof("created hygon vdev conf: %s", filePath) + return nil +} + +func hygonVgpuCacheDirName(guestId, containerName string, devIdx, pipeID, vdevIdx int, coremsk1, coremsk2 string) string { + return fmt.Sprintf("%s_%s_%d_%d_%d_%s_%s", guestId, containerName, devIdx, pipeID, vdevIdx, coremsk1, coremsk2) +} + +func hygonReadFile(filePath string) ([]byte, error) { + if hygonUseRemoteFS() { + out, err := procutils.NewRemoteCommandAsFarAsPossible("cat", filePath).Output() + if err != nil { + return nil, err + } + return out, nil + } + return os.ReadFile(filePath) +} + +func hygonMkdirAll(dir string, perm os.FileMode) error { + if hygonUseRemoteFS() { + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err != nil { + return errors.Wrapf(err, "remote mkdir %s: %s", dir, out) + } + if perm != 0 { + out, err = procutils.NewRemoteCommandAsFarAsPossible("chmod", fmt.Sprintf("%o", perm), dir).Output() + if err != nil { + return errors.Wrapf(err, "remote chmod %s: %s", dir, out) + } + } + return nil + } + if err := os.MkdirAll(dir, perm); err != nil { + return err + } + return os.Chmod(dir, perm) +} + +func hygonWriteFile(filePath string, content []byte, perm os.FileMode) error { + if hygonUseRemoteFS() { + b64 := base64.StdEncoding.EncodeToString(content) + script := fmt.Sprintf("mkdir -p $(dirname %q) && echo %q | base64 -d > %q && chmod %o %q", + filePath, b64, filePath, perm, filePath) + out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", script).Output() + if err != nil { + return errors.Wrapf(err, "remote write %s: %s", filePath, out) + } + return nil + } + return os.WriteFile(filePath, content, perm) +} + +func hygonRemoveAll(path string) error { + if hygonUseRemoteFS() { + out, err := procutils.NewRemoteCommandAsFarAsPossible("rm", "-rf", path).Output() + if err != nil { + return errors.Wrapf(err, "remote rm -rf %s: %s", path, out) + } + return nil + } + return os.RemoveAll(path) +} + +func hygonRemove(filePath string) error { + if hygonUseRemoteFS() { + out, err := procutils.NewRemoteCommandAsFarAsPossible("rm", "-f", filePath).Output() + if err != nil { + return errors.Wrapf(err, "remote rm %s: %s", filePath, out) + } + return nil + } + return os.Remove(filePath) +} diff --git a/pkg/hostman/isolated_device/container_device/net_int_device.go b/pkg/hostman/isolated_device/container_device/net_int_device.go index e021ac9a87..43354d5804 100644 --- a/pkg/hostman/isolated_device/container_device/net_int_device.go +++ b/pkg/hostman/isolated_device/container_device/net_int_device.go @@ -21,11 +21,11 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/httperrors" diff --git a/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go b/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go index 84885baa1d..ffd7f3361d 100644 --- a/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go +++ b/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go @@ -21,11 +21,12 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/pkg/errors" + api "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/hostinfo" - "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/hostman/isolated_device" ) diff --git a/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go b/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go index e4a7d69f09..d4074fcb74 100644 --- a/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go +++ b/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go @@ -22,10 +22,10 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/pkg/errors" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" fileutils "yunion.io/x/onecloud/pkg/util/fileutils2" diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index cf9e22b903..a4b8d11371 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -280,11 +280,12 @@ type SHostOptions struct { AscendNpuHamiLibvnpuPath string `help:"ascend npu hami libvnpu.so path" default:"/opt/cloud/hami/libvnpu.so"` EnableContainerHygonDCU bool `help:"enable container hygon dcu" default:"true"` - EnableContainerHygonDCUHAMI bool `help:"enable container hygon dcu hami" default:"false"` + EnableContainerHygonDCUHami bool `help:"enable container hygon dcu hami" default:"false"` HygonHyhalPath string `help:"hygon hyhal driver path" default:"/opt/hyhal"` HygonDtkPath string `help:"hygon dtk toolkit path" default:"/opt/dtk"` HygonHySmiPath string `help:"hygon hy-smi path" default:"/opt/hyhal/bin/hy-smi"` HygonVdevConfDir string `help:"hygon vdcu config directory" default:"/etc/vdev"` + HygonVgpuCacheDir string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"` EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"` diff --git a/pkg/llm/models/llm_aiproxy_sync.go b/pkg/llm/models/llm_aiproxy_sync.go index 3fbf4db841..1edaa47319 100644 --- a/pkg/llm/models/llm_aiproxy_sync.go +++ b/pkg/llm/models/llm_aiproxy_sync.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/printutils" apapi "yunion.io/x/onecloud/pkg/apis/aiproxy" api "yunion.io/x/onecloud/pkg/apis/llm" @@ -18,7 +19,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" - "yunion.io/x/pkg/util/printutils" ) // aiproxyPlaceholderAPIKey is used for local inference backends (vLLM/Ollama/SGLang) diff --git a/pkg/llm/service/benchmark_handler.go b/pkg/llm/service/benchmark_handler.go index 74d4f988a3..da4b84fd67 100644 --- a/pkg/llm/service/benchmark_handler.go +++ b/pkg/llm/service/benchmark_handler.go @@ -8,6 +8,7 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/httperrors" diff --git a/pkg/mcp-server/adapters/adapter.go b/pkg/mcp-server/adapters/adapter.go index f077406958..01db2a5fbe 100644 --- a/pkg/mcp-server/adapters/adapter.go +++ b/pkg/mcp-server/adapters/adapter.go @@ -18,12 +18,14 @@ import ( "context" "github.com/golang-plus/errors" + + "yunion.io/x/pkg/gotypes" + api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcp-server/options" - "yunion.io/x/pkg/gotypes" ) // CloudpodsAdapter 负责 Cloudpods 认证并创建 mcclient.ClientSession，供 climc 工具执行使用。 diff --git a/pkg/mcp-server/climcgen/schema_test.go b/pkg/mcp-server/climcgen/schema_test.go index 56d007ad43..3faf1a0737 100644 --- a/pkg/mcp-server/climcgen/schema_test.go +++ b/pkg/mcp-server/climcgen/schema_test.go @@ -19,10 +19,9 @@ import ( "strings" "testing" + "yunion.io/x/onecloud/cmd/climc/shell" _ "yunion.io/x/onecloud/cmd/climc/shell/compute" _ "yunion.io/x/onecloud/cmd/climc/shell/image" - - "yunion.io/x/onecloud/cmd/climc/shell" ) func TestBuildInputSchemaUsesMcpTags(t *testing.T) {