From 73ecee35244c7947d59a162d4fa056fd0898695a Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 15 Oct 2026 19:12:31 +0800 Subject: [PATCH] feat(region,host,llm,monitor): support MooreThreads GPU (#25882) --- pkg/apis/compute/container.go | 36 ++- pkg/apis/compute/isolated_device_const.go | 11 +- pkg/apis/compute/mthreads_sgpu.go | 75 +++++ pkg/apis/compute/mthreads_sgpu_test.go | 57 ++++ pkg/baremetal/tasks/baseprepare.go | 4 + pkg/compute/models/guest_actions.go | 39 ++- pkg/compute/models/guests.go | 3 + pkg/compute/models/isolated_devices.go | 56 +++- pkg/hostman/guestman/pod_helper.go | 2 +- pkg/hostman/hostinfo/container.go | 86 ++++++ pkg/hostman/hostinfo/hostinfo.go | 52 ++-- pkg/hostman/hostmetrics/container_metrics.go | 9 + .../container_mthreads_gpu_metrics.go | 274 ++++++++++++++++++ .../container_mthreads_gpu_metrics_test.go | 100 +++++++ .../container_nvidia_gpu_metrics.go | 10 +- pkg/hostman/hostmetrics/hostmetrics.go | 34 ++- .../hostmetrics/testdata/mthreads-pm.txt | 8 + .../testdata/mthreads-processes.txt | 12 + .../isolated_device/container_device.go | 34 ++- .../container_device/mthreads_gmi.go | 193 ++++++++++++ .../container_device/mthreads_gmi_test.go | 110 +++++++ .../container_device/mthreads_gpu.go | 216 ++++++++++++++ .../container_device/mthreads_hami.go | 117 ++++++++ .../container_device/testdata/s5000-8gpu.txt | 54 ++++ .../isolated_device/isolated_device.go | 51 +++- pkg/hostman/options/options.go | 4 + pkg/hostman/system_service/telegraf.go | 8 + pkg/llm/drivers/llm_container/vllm.go | 3 +- .../drivers/llm_container/vllm_hygon_test.go | 43 +++ pkg/llm/models/llm_base.go | 17 ++ pkg/llm/models/llm_devices_test.go | 57 ++++ pkg/llm/models/sku.go | 22 +- pkg/llm/models/sku_hami_test.go | 38 +++ pkg/monitor/dbinit/measurements/all.go | 1 + pkg/monitor/dbinit/measurements/metrics.csv | 138 +++++++-- .../dbinit/measurements/mthreads_gmi.go | 65 +++++ pkg/monitor/dbinit/measurements/pod.go | 5 + 37 files changed, 1941 insertions(+), 103 deletions(-) create mode 100644 pkg/apis/compute/mthreads_sgpu.go create mode 100644 pkg/apis/compute/mthreads_sgpu_test.go create mode 100644 pkg/hostman/hostmetrics/container_mthreads_gpu_metrics.go create mode 100644 pkg/hostman/hostmetrics/container_mthreads_gpu_metrics_test.go create mode 100644 pkg/hostman/hostmetrics/testdata/mthreads-pm.txt create mode 100644 pkg/hostman/hostmetrics/testdata/mthreads-processes.txt create mode 100644 pkg/hostman/isolated_device/container_device/mthreads_gmi.go create mode 100644 pkg/hostman/isolated_device/container_device/mthreads_gmi_test.go create mode 100644 pkg/hostman/isolated_device/container_device/mthreads_gpu.go create mode 100644 pkg/hostman/isolated_device/container_device/mthreads_hami.go create mode 100644 pkg/hostman/isolated_device/container_device/testdata/s5000-8gpu.txt create mode 100644 pkg/monitor/dbinit/measurements/mthreads_gmi.go diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 43a553b3cf..0546477061 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -33,22 +33,24 @@ func init() { } const ( - CONTAINER_DEV_CPH_AMD_GPU = "CPH_AMD_GPU" - CONTAINER_DEV_CPH_AOSP_BINDER = "CPH_AOSP_BINDER" - CONTAINER_DEV_NETINT_CA_ASIC = "NETINT_CA_ASIC" - CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA" - CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU" - CONTAINER_DEV_NVIDIA_MPS = "NVIDIA_MPS" - CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE" - CONTAINER_DEV_NVIDIA_HAMI = "NVIDIA_HAMI" - CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU" - CONTAINER_DEV_ASCEND_NPU_HAMI = "ASCEND_NPU_HAMI" - CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU" - CONTAINER_DEV_HYGON_DCU = "HYGON_DCU" - CONTAINER_DEV_HYGON_DCU_HAMI = "HYGON_DCU_HAMI" - CONTAINER_DEV_ILUVATAR_GPU = "ILUVATAR_GPU" - CONTAINER_DEV_THEAD_PPU = "THEAD_PPU" - CONTAINER_DEV_KUNLUNXIN_XPU = "KUNLUNXIN_XPU" + CONTAINER_DEV_CPH_AMD_GPU = "CPH_AMD_GPU" + CONTAINER_DEV_CPH_AOSP_BINDER = "CPH_AOSP_BINDER" + CONTAINER_DEV_NETINT_CA_ASIC = "NETINT_CA_ASIC" + CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA" + CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU" + CONTAINER_DEV_NVIDIA_MPS = "NVIDIA_MPS" + CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE" + CONTAINER_DEV_NVIDIA_HAMI = "NVIDIA_HAMI" + CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU" + CONTAINER_DEV_ASCEND_NPU_HAMI = "ASCEND_NPU_HAMI" + CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU" + CONTAINER_DEV_HYGON_DCU = "HYGON_DCU" + CONTAINER_DEV_HYGON_DCU_HAMI = "HYGON_DCU_HAMI" + CONTAINER_DEV_ILUVATAR_GPU = "ILUVATAR_GPU" + CONTAINER_DEV_THEAD_PPU = "THEAD_PPU" + CONTAINER_DEV_KUNLUNXIN_XPU = "KUNLUNXIN_XPU" + CONTAINER_DEV_MTHREADS_GPU = "MTHREADS_GPU" + CONTAINER_DEV_MTHREADS_GPU_HAMI = "MTHREADS_GPU_HAMI" ) var ( @@ -64,6 +66,8 @@ var ( CONTAINER_DEV_ILUVATAR_GPU, CONTAINER_DEV_THEAD_PPU, CONTAINER_DEV_KUNLUNXIN_XPU, + CONTAINER_DEV_MTHREADS_GPU, + CONTAINER_DEV_MTHREADS_GPU_HAMI, } ) diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go index 3ce9ed18ed..74710ec38e 100644 --- a/pkg/apis/compute/isolated_device_const.go +++ b/pkg/apis/compute/isolated_device_const.go @@ -53,10 +53,11 @@ const ( ILUVATAR_VENDOR_ID = "1e3e" THEAD_VENDOR_ID = "1ded" KUNLUNXIN_VENDOR_ID = "1d22" + MTHREADS_VENDOR_ID = "1ed5" ) var HeterogeneousVendors = []string{ - VASTAITECH_VENDOR_ID, HYGON_VENDOR_ID, ASCEND_VENDOR_ID, ILUVATAR_VENDOR_ID, THEAD_VENDOR_ID, KUNLUNXIN_VENDOR_ID, + VASTAITECH_VENDOR_ID, HYGON_VENDOR_ID, ASCEND_VENDOR_ID, ILUVATAR_VENDOR_ID, THEAD_VENDOR_ID, KUNLUNXIN_VENDOR_ID, MTHREADS_VENDOR_ID, } var ID_VENDOR_MAP = map[string]string{ @@ -68,6 +69,7 @@ var ID_VENDOR_MAP = map[string]string{ ILUVATAR_VENDOR_ID: "ILUVATAR", THEAD_VENDOR_ID: "THEAD", KUNLUNXIN_VENDOR_ID: "KUNLUNXIN", + MTHREADS_VENDOR_ID: "MTHREADS", } var VENDOR_ID_MAP = map[string]string{ @@ -79,6 +81,7 @@ var VENDOR_ID_MAP = map[string]string{ "ILUVATAR": ILUVATAR_VENDOR_ID, "THEAD": THEAD_VENDOR_ID, "KUNLUNXIN": KUNLUNXIN_VENDOR_ID, + "MTHREADS": MTHREADS_VENDOR_ID, } const ( @@ -107,6 +110,8 @@ var GPU_TYPES = []string{ CONTAINER_DEV_ILUVATAR_GPU, CONTAINER_DEV_THEAD_PPU, CONTAINER_DEV_KUNLUNXIN_XPU, + CONTAINER_DEV_MTHREADS_GPU, + CONTAINER_DEV_MTHREADS_GPU_HAMI, } var NETINT_TYPES = []string{ @@ -121,6 +126,8 @@ var VALID_CONTAINER_DEVICE_TYPES = []string{ CONTAINER_DEV_ILUVATAR_GPU, CONTAINER_DEV_THEAD_PPU, CONTAINER_DEV_KUNLUNXIN_XPU, + CONTAINER_DEV_MTHREADS_GPU, + CONTAINER_DEV_MTHREADS_GPU_HAMI, } var VALID_PASSTHROUGH_TYPES = []string{ @@ -144,6 +151,8 @@ var VITRUAL_DEVICE_TYPES = []string{ CONTAINER_DEV_ILUVATAR_GPU, CONTAINER_DEV_THEAD_PPU, CONTAINER_DEV_KUNLUNXIN_XPU, + CONTAINER_DEV_MTHREADS_GPU, + CONTAINER_DEV_MTHREADS_GPU_HAMI, } func init() { diff --git a/pkg/apis/compute/mthreads_sgpu.go b/pkg/apis/compute/mthreads_sgpu.go new file mode 100644 index 0000000000..29c1a6f73c --- /dev/null +++ b/pkg/apis/compute/mthreads_sgpu.go @@ -0,0 +1,75 @@ +package compute + +import "strings" + +// sGPU memory buckets in MiB. The driver rounds a requested limit up to one of these. +var mthreadsSgpuMemoryBucketsMiB = []int{ + 512, + 1 * 1024, + 2 * 1024, + 4 * 1024, + 8 * 1024, + 16 * 1024, + 32 * 1024, + 48 * 1024, + 64 * 1024, + 80 * 1024, +} + +const ( + MthreadsSgpuMaxInstances = 16 + MthreadsSgpuWeightMin = 1 + MthreadsSgpuWeightMax = 99999 + mthreadsSgpuBytesPerMiB int64 = 1024 * 1024 +) + +// AlignMthreadsSgpuMemoryMiB rounds a requested size up to the next sGPU bucket. +// Zero stays zero. A request above the largest bucket is left unchanged. +func AlignMthreadsSgpuMemoryMiB(request int) int { + if request <= 0 { + return 0 + } + for _, bucket := range mthreadsSgpuMemoryBucketsMiB { + if request <= bucket { + return bucket + } + } + return request +} + +// MthreadsSgpuMemoryLimitBytes is the byte value for MTHREADS_QOS_MEMORY_LIMIT. +func MthreadsSgpuMemoryLimitBytes(memoryMiB int) int64 { + return int64(AlignMthreadsSgpuMemoryMiB(memoryMiB)) * mthreadsSgpuBytesPerMiB +} + +// IsMthreadsHami reports a sliced Moore Threads GPU. Vendor may be the name or the PCI id. +func IsMthreadsHami(vendor, sharingMode string) bool { + if sharingMode != DEVICE_SHARING_MODE_HAMI { + return false + } + vendor = strings.TrimSpace(vendor) + return strings.EqualFold(vendor, "MTHREADS") || strings.EqualFold(vendor, MTHREADS_VENDOR_ID) +} + +// CountMthreadsHamiDevices counts sGPU slices. More than one in a guest is rejected. +func CountMthreadsHamiDevices(devs []*IsolatedDeviceConfig) int { + n := 0 + for _, dev := range devs { + if dev != nil && IsMthreadsHami(dev.Vendor, dev.SharingMode) { + n++ + } + } + return n +} + +// MthreadsSgpuComputingPowerWeight maps an SM percent to the sGPU time-slice weight. +// Zero uses the driver default of 1. +func MthreadsSgpuComputingPowerWeight(smUtilLimit int) int { + if smUtilLimit < MthreadsSgpuWeightMin { + return MthreadsSgpuWeightMin + } + if smUtilLimit > MthreadsSgpuWeightMax { + return MthreadsSgpuWeightMax + } + return smUtilLimit +} diff --git a/pkg/apis/compute/mthreads_sgpu_test.go b/pkg/apis/compute/mthreads_sgpu_test.go new file mode 100644 index 0000000000..b2b0c0dcf7 --- /dev/null +++ b/pkg/apis/compute/mthreads_sgpu_test.go @@ -0,0 +1,57 @@ +package compute + +import "testing" + +func TestAlignMthreadsSgpuMemoryMiB(t *testing.T) { + cases := []struct { + in int + want int + }{ + {0, 0}, + {-1, 0}, + {500, 512}, + {512, 512}, + {1000, 1024}, + {40960, 48 * 1024}, + {81920, 80 * 1024}, + {90000, 90000}, + } + for _, tc := range cases { + if got := AlignMthreadsSgpuMemoryMiB(tc.in); got != tc.want { + t.Fatalf("AlignMthreadsSgpuMemoryMiB(%d)=%d, want %d", tc.in, got, tc.want) + } + } +} + +func TestMthreadsSgpuMemoryLimitBytes(t *testing.T) { + if got := MthreadsSgpuMemoryLimitBytes(2048); got != 2147483648 { + t.Fatalf("bytes=%d, want 2147483648", got) + } + if got := MthreadsSgpuMemoryLimitBytes(0); got != 0 { + t.Fatalf("zero bytes=%d", got) + } +} + +func TestCountMthreadsHamiDevices(t *testing.T) { + devs := []*IsolatedDeviceConfig{ + {Vendor: "MTHREADS", SharingMode: DEVICE_SHARING_MODE_HAMI}, + {Vendor: MTHREADS_VENDOR_ID, SharingMode: DEVICE_SHARING_MODE_HAMI}, + {Vendor: "MTHREADS", SharingMode: DEVICE_SHARING_MODE_EXCLUSIVE}, + {Vendor: "NVIDIA", SharingMode: DEVICE_SHARING_MODE_HAMI}, + } + if got := CountMthreadsHamiDevices(devs); got != 2 { + t.Fatalf("count=%d, want 2", got) + } +} + +func TestMthreadsSgpuComputingPowerWeight(t *testing.T) { + if got := MthreadsSgpuComputingPowerWeight(0); got != 1 { + t.Fatalf("weight=%d, want 1", got) + } + if got := MthreadsSgpuComputingPowerWeight(25); got != 25 { + t.Fatalf("weight=%d, want 25", got) + } + if got := MthreadsSgpuComputingPowerWeight(100000); got != 99999 { + t.Fatalf("weight=%d, want 99999", got) + } +} diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index 6c6c9bbedc..d66db90341 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -809,6 +809,10 @@ func getIsolatedDevicesInfo(cli *ssh.Client, ip net.IP) ([]*isolated_device.PCID log.Infof("skip kunlunxin dev %s with class code %s", dev.Addr, dev.ClassCode) continue } + if dev.VendorId == api.MTHREADS_VENDOR_ID && dev.ClassCode != isolated_device.CLASS_CODE_PA { + log.Infof("skip mthreads dev %s with class code %s", dev.Addr, dev.ClassCode) + continue + } log.Infof("append dev %s", dev.Addr) devs = append(devs, dev) } diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index b4a5947a87..cfc0b22dc2 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -2730,6 +2730,32 @@ func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient } } } + if sharingMode == api.DEVICE_SHARING_MODE_HAMI { + existing := 0 + for i := range attachedGpus { + gdev := attachedGpus[i].GetIsolatedDevice() + if gdev != nil && gdev.isMthreads() && gdev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + existing++ + } + } + adding := 0 + for _, count := range devModelCount { + adding += count + } + if existing > 0 || adding > 1 { + for devModel := range devModelCount { + devs, err := IsolatedDeviceManager.GetAvailableIsolatedDeviceOnHost(host.Id, devModel, sharingMode) + if err != nil { + return httperrors.NewInternalServerError("fetch gpu failed %s", err) + } + for i := range devs { + if devs[i].isMthreads() && existing+adding > 1 { + return httperrors.NewBadRequestError("Moore Threads sGPU does not support multiple cards in one guest; use exclusive mode for multiple cards") + } + } + } + } + } for devModel, count := range devModelCount { devs, err := IsolatedDeviceManager.GetAvailableIsolatedDeviceOnHost(host.Id, devModel, sharingMode) @@ -2849,6 +2875,14 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred } } } + if dev.isMthreads() && dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + for i := range gdevs { + gdev := gdevs[i].GetIsolatedDevice() + if gdev != nil && gdev.isMthreads() && gdev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + return httperrors.NewBadRequestError("Moore Threads sGPU does not support multiple cards in one guest; use exclusive mode for multiple cards") + } + } + } host, _ := self.GetHost() lockman.LockObject(ctx, host) @@ -2888,8 +2922,9 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient. guestIsolatedDevice.DiskIndex = *diskIndex } if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI && memoryRequest != nil { - guestIsolatedDevice.DeviceMemorySize = *memoryRequest - guestIsolatedDevice.SmUtilLimit = int((float64(*memoryRequest) / float64(dev.MemorySize)) * 100.0) + req := dev.hamiMemoryRequest(*memoryRequest) + guestIsolatedDevice.DeviceMemorySize = req + guestIsolatedDevice.SmUtilLimit = int((float64(req) / float64(dev.MemorySize)) * 100.0) } if utils.IsInStringArray(gpuType, []string{api.GPU_HPC, api.GPU_VGA}) { guestIsolatedDevice.GpuType = gpuType diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 5ac1abf908..89b5580dec 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2381,6 +2381,9 @@ func (manager *SGuestManager) validateCreateData( if nvidiaVgpuCnt > 1 { return nil, httperrors.NewBadRequestError("Nvidia vGPU count cannot exceed 1") } + if api.CountMthreadsHamiDevices(input.IsolatedDevices) > 1 { + return nil, httperrors.NewBadRequestError("Moore Threads sGPU does not support multiple cards in one guest; use exclusive mode for multiple cards") + } if nvidiaVgpuCnt > 0 && gpuCnt > 0 { return nil, httperrors.NewBadRequestError("Nvidia vGPU cannot passthrough with other gpus") } diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index f49712d473..9118ba7c6e 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -2020,6 +2020,29 @@ func (manager *SIsolatedDeviceManager) GetHostsGuestIsolatedDevices(hostIds []st return ret } +func (dev *SIsolatedDevice) isMthreads() bool { + return strings.EqualFold(dev.getVendorId(), api.MTHREADS_VENDOR_ID) +} + +func (dev *SIsolatedDevice) hamiMemoryRequest(memoryRequest int) int { + if dev.isMthreads() { + return api.AlignMthreadsSgpuMemoryMiB(memoryRequest) + } + return memoryRequest +} + +func (dev *SIsolatedDevice) mthreadsSgpuInstanceFull() bool { + if !dev.isMthreads() { + return false + } + cnt, err := dev.getAllocatedCount() + if err != nil { + log.Errorf("failed getAllocatedCount %s", err) + return true + } + return cnt >= api.MthreadsSgpuMaxInstances +} + func (dev *SIsolatedDevice) IsEnough(memoryRequest int) bool { if dev.SharingMode != api.DEVICE_SHARING_MODE_HAMI { cnt, err := dev.getAllocatedCount() @@ -2028,14 +2051,17 @@ func (dev *SIsolatedDevice) IsEnough(memoryRequest int) bool { return false } return dev.VirtualNum > cnt - } else { - allocated, err := dev.getAllocatedMemorySize() - if err != nil { - log.Errorf("failed getAllocatedMemorySize %s", err) - return false - } - return (dev.MemorySize - allocated) >= memoryRequest } + if dev.mthreadsSgpuInstanceFull() { + return false + } + memoryRequest = dev.hamiMemoryRequest(memoryRequest) + allocated, err := dev.getAllocatedMemorySize() + if err != nil { + log.Errorf("failed getAllocatedMemorySize %s", err) + return false + } + return (dev.MemorySize - allocated) >= memoryRequest } func (dev *SIsolatedDevice) IsFull() bool { @@ -2046,14 +2072,16 @@ func (dev *SIsolatedDevice) IsFull() bool { return true } return dev.VirtualNum <= cnt - } else { - allocated, err := dev.getAllocatedMemorySize() - if err != nil { - log.Errorf("failed getAllocatedMemorySize %s", err) - return true - } - return dev.MemorySize <= allocated } + if dev.mthreadsSgpuInstanceFull() { + return true + } + allocated, err := dev.getAllocatedMemorySize() + if err != nil { + log.Errorf("failed getAllocatedMemorySize %s", err) + return true + } + return dev.MemorySize <= allocated } func (manager *SIsolatedDeviceManager) InitializeData() error { diff --git a/pkg/hostman/guestman/pod_helper.go b/pkg/hostman/guestman/pod_helper.go index fc4a4aa5f8..0e4f606dbc 100644 --- a/pkg/hostman/guestman/pod_helper.go +++ b/pkg/hostman/guestman/pod_helper.go @@ -130,7 +130,7 @@ func PushContainerdImage(input *hostapi.ContainerPushImageInput) error { // try http protocol errs = append(errs, errors.Wrap(err, "pushImageByCtrCmd: %s")) opt.PlainHttp = true - log.Infof("try push image %s by http", input.Image) + log.Warningf("push image to %s error: %v, try to push it by http", input.Image, err) if err := imgTool.Push(input.Image, opt); err != nil { errs = append(errs, errors.Wrapf(err, "pushImageByCtrCmd by http")) return errors.NewAggregate(errs) diff --git a/pkg/hostman/hostinfo/container.go b/pkg/hostman/hostinfo/container.go index 727abc1c3a..3bc42dbebd 100644 --- a/pkg/hostman/hostinfo/container.go +++ b/pkg/hostman/hostinfo/container.go @@ -17,6 +17,7 @@ package hostinfo import ( "context" "path" + "sort" "strconv" "strings" "time" @@ -184,3 +185,88 @@ func (h *SHostInfo) HasContainerNvidiaGpu() bool { h.containerNvidiaGpus = nvDevs return *h.hasNvidiaGpus } + +func (h *SHostInfo) HasContainerMthreadsGpu() bool { + if h.hasMthreadsGpus != nil { + return *h.hasMthreadsGpus + } + hasMthreadsGpus := false + devs := h.IsolatedDeviceMan.GetDevices() + for i := range devs { + vendorId := strings.ToLower(strings.Split(devs[i].GetVendorDeviceId(), ":")[0]) + if vendorId == apis.MTHREADS_VENDOR_ID { + hasMthreadsGpus = true + break + } + } + h.hasMthreadsGpus = &hasMthreadsGpus + return *h.hasMthreadsGpus +} + +func (h *SHostInfo) GetMthreadsGpuIndexes(ids, addrs []string) []int { + if len(ids) == 0 && len(addrs) == 0 { + return nil + } + wantId := make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id != "" { + wantId[id] = struct{}{} + } + } + wantAddr := make([]string, 0, len(addrs)) + for _, addr := range addrs { + if addr != "" { + wantAddr = append(wantAddr, addr) + } + } + seen := map[int]struct{}{} + out := make([]int, 0) + devs := h.IsolatedDeviceMan.GetDevices() + for i := range devs { + dev := devs[i] + vendorId := strings.ToLower(strings.Split(dev.GetVendorDeviceId(), ":")[0]) + if vendorId != apis.MTHREADS_VENDOR_ID { + continue + } + _, idMatch := wantId[dev.GetCloudId()] + addrMatch := false + if !idMatch { + for _, addr := range wantAddr { + if samePCIAddr(dev.GetAddr(), addr) { + addrMatch = true + break + } + } + } + if !idMatch && !addrMatch { + continue + } + idx := dev.GetIndex() + if idx < 0 { + n, err := strconv.Atoi(dev.GetDevicePath()) + if err != nil { + continue + } + idx = n + } + if _, ok := seen[idx]; ok { + continue + } + seen[idx] = struct{}{} + out = append(out, idx) + } + sort.Ints(out) + return out +} + +func samePCIAddr(a, b string) bool { + a = strings.ToLower(strings.TrimSpace(a)) + b = strings.ToLower(strings.TrimSpace(b)) + if a == "" || b == "" { + return false + } + if a == b { + return true + } + return strings.HasSuffix(a, ":"+b) || strings.HasSuffix(b, ":"+a) +} diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index f7fb820f9a..6ea837d9b8 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -135,6 +135,7 @@ type SHostInfo struct { hasNvidiaGpus *bool hasVastaitechGpus *bool hasCphAmdGpus *bool + hasMthreadsGpus *bool guestManager hostutils.IGuestManager } @@ -2357,24 +2358,26 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { options.HostOptions.HygonDtkPath, ) probeOpts := &isolated_device.SIsolatedDeviceProbeOptions{ - SkipGPUs: options.HostOptions.DisableGPU, - SkipUSBs: options.HostOptions.DisableUSB, - SkipCustomDevs: options.HostOptions.DisableCustomDevice, - EnableCudaHAMI: options.HostOptions.EnableCudaHAMI, - EnableCudaMps: options.HostOptions.EnableCudaMPS, - EnableContainerAscendNpu: options.HostOptions.EnableContainerAscendNPU, - EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami, - EnableContainerHygonDCU: options.HostOptions.EnableContainerHygonDCU, - EnableContainerHygonDCUHAMI: options.HostOptions.EnableContainerHygonDCUHami, - EnableContainerIluvatarGPU: options.HostOptions.EnableContainerIluvatarGPU, - EnableContainerTHeadPPU: options.HostOptions.EnableContainerTHeadPPU, - EnableContainerKunlunxinXPU: options.HostOptions.EnableContainerKunlunxinXPU, - EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, - SriovNics: sriovNics, - OvsOffloadNics: offloadNics, - NvmePciDisks: options.HostOptions.PTNVMEConfigs, - AmdVgpuPFs: options.HostOptions.AMDVgpuPFs, - NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs, + SkipGPUs: options.HostOptions.DisableGPU, + SkipUSBs: options.HostOptions.DisableUSB, + SkipCustomDevs: options.HostOptions.DisableCustomDevice, + EnableCudaHAMI: options.HostOptions.EnableCudaHAMI, + EnableCudaMps: options.HostOptions.EnableCudaMPS, + EnableContainerAscendNpu: options.HostOptions.EnableContainerAscendNPU, + EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami, + EnableContainerHygonDCU: options.HostOptions.EnableContainerHygonDCU, + EnableContainerHygonDCUHAMI: options.HostOptions.EnableContainerHygonDCUHami, + EnableContainerIluvatarGPU: options.HostOptions.EnableContainerIluvatarGPU, + EnableContainerTHeadPPU: options.HostOptions.EnableContainerTHeadPPU, + EnableContainerKunlunxinXPU: options.HostOptions.EnableContainerKunlunxinXPU, + EnableContainerMthreadsGPU: options.HostOptions.EnableContainerMthreadsGPU, + EnableContainerMthreadsGPUHami: options.HostOptions.EnableContainerMthreadsGPUHami, + EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, + SriovNics: sriovNics, + OvsOffloadNics: offloadNics, + NvmePciDisks: options.HostOptions.PTNVMEConfigs, + AmdVgpuPFs: options.HostOptions.AMDVgpuPFs, + NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs, } h.IsolatedDeviceMan.ProbePCIDevices(probeOpts) @@ -2717,6 +2720,7 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { hasIluvatar := false hasTHead := false hasKunlunxin := false + hasMthreads := false hasNvidiasmi := false hasNpusmi := false for _, dev := range devs { @@ -2733,6 +2737,9 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { if vendorId == api.KUNLUNXIN_VENDOR_ID { hasKunlunxin = true } + if vendorId == api.MTHREADS_VENDOR_ID { + hasMthreads = true + } if !utils.IsInStringArray(dev.GetSharingMode(), api.VIRTUAL_SHARING_MODES) { continue } @@ -2813,6 +2820,15 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { system_service.TELEGRAF_INPUT_CONF_LIB_PATH: libPath, } } + if hasMthreads { + smiPath := options.HostOptions.MthreadsGmiPath + if smiPath == "" { + smiPath = "/usr/bin/mthreads-gmi" + } + conf[system_service.TELEGRAF_INPUT_MTHREADSGMI] = map[string]interface{}{ + system_service.TELEGRAF_INPUT_CONF_BIN_PATH: smiPath, + } + } if hasKunlunxin { xreHome := options.HostOptions.KunlunxinXreHome if xreHome == "" { diff --git a/pkg/hostman/hostmetrics/container_metrics.go b/pkg/hostman/hostmetrics/container_metrics.go index c23c16db2b..4c43c5cabb 100644 --- a/pkg/hostman/hostmetrics/container_metrics.go +++ b/pkg/hostman/hostmetrics/container_metrics.go @@ -79,6 +79,10 @@ const ( CPH_AMD_GPU_DEV_ID = "dev_id" CPH_AMD_GPU_MEM = "mem" CPH_AMD_GPU_MEM_UTIL = "mem_util" + + MTHREADS_GPU_MEM = "mem" + MTHREADS_GPU_MEM_UTIL = "mem_util" + MTHREADS_GPU_GPU_UTIL = "gpu_util" ) type CadvisorProcessMetric struct { @@ -113,6 +117,7 @@ type PodMetrics struct { PodNvidiaGpu []*PodNvidiaGpuMetrics `json:"pod_nvidia_gpu"` PodVastaitechGpu []*PodVastaitechGpuMetrics `json:"pod_vastaitech_gpu"` PodCphAmdGpu []*PodCphAmdGpuMetrics `json:"pod_cph_amd_gpu"` + PodMthreadsGpu []*PodMthreadsGpuMetrics `json:"pod_mthreads_gpu"` Containers []*ContainerMetrics `json:"containers"` } @@ -821,6 +826,7 @@ func (m *SGuestMonitor) PodMetrics(prevUsage *GuestMetrics) *PodMetrics { PodNvidiaGpu: m.getPodNvidiaGpuMetrics(), PodVastaitechGpu: m.getPodVastaitechGpuMetrics(), PodCphAmdGpu: m.getPodCphAmdGpuMetrics(), + PodMthreadsGpu: m.getPodMthreadsGpuMetrics(), Containers: containers, } @@ -1031,6 +1037,9 @@ func (d *GuestMetrics) toPodTelegrafData(tagStr string) []string { for i := range m.PodCphAmdGpu { ims = append(ims, m.PodCphAmdGpu[i]) } + for i := range m.PodMthreadsGpu { + ims = append(ims, m.PodMthreadsGpu[i]) + } for _, c := range m.Containers { ims = append(ims, c.ContainerCpu) diff --git a/pkg/hostman/hostmetrics/container_mthreads_gpu_metrics.go b/pkg/hostman/hostmetrics/container_mthreads_gpu_metrics.go new file mode 100644 index 0000000000..1f9c22af81 --- /dev/null +++ b/pkg/hostman/hostmetrics/container_mthreads_gpu_metrics.go @@ -0,0 +1,274 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostmetrics + +import ( + "sort" + "strconv" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +type MthreadsGpuProc struct { + Index int + Pid string + MemMiB float64 + MemUtil float64 // fraction, 0 to 1 + GpuUtil float64 // percent, 0 to 100 +} + +func mthreadsGmiBin() string { + if options.HostOptions.MthreadsGmiPath != "" { + return options.HostOptions.MthreadsGmiPath + } + return "/usr/bin/mthreads-gmi" +} + +func GetMthreadsGpuProcessMetrics() ([]MthreadsGpuProc, error) { + bin := mthreadsGmiBin() + pmOut, pmErr := procutils.NewRemoteCommandAsFarAsPossible(bin, "-pm").Output() + memOut, memErr := procutils.NewRemoteCommandAsFarAsPossible(bin).Output() + if pmErr != nil && memErr != nil { + return nil, errors.Wrapf(pmErr, "mthreads-gmi -pm (%s); default: %v (%s)", pmOut, memErr, memOut) + } + if pmErr != nil { + log.Errorf("mthreads-gmi -pm: %s %s", pmErr, pmOut) + pmOut = nil + } + if memErr != nil { + log.Errorf("mthreads-gmi: %s %s", memErr, memOut) + memOut = nil + } + return mergeMthreadsGpuProcs(parseMthreadsProcessMonitor(string(pmOut)), parseMthreadsProcessMemory(string(memOut))), nil +} + +func parseMthreadsProcessMonitor(output string) []MthreadsGpuProc { + var out []MthreadsGpuProc + for _, line := range strings.Split(output, "\n") { + fields := strings.Fields(strings.TrimSpace(line)) + if len(fields) < 4 { + continue + } + idx, err := strconv.Atoi(fields[0]) + if err != nil { + continue + } + pid := fields[1] + if _, err := strconv.Atoi(pid); err != nil { + continue + } + out = append(out, MthreadsGpuProc{ + Index: idx, + Pid: pid, + MemUtil: parseMthreadsMemPercent(fields[2]), + GpuUtil: parseMthreadsGpuPercent(fields[3]), + }) + } + return out +} + +func parseMthreadsProcessMemory(output string) []MthreadsGpuProc { + var out []MthreadsGpuProc + for _, line := range strings.Split(output, "\n") { + fields := strings.Fields(strings.TrimSpace(line)) + if len(fields) < 3 { + continue + } + memField := fields[len(fields)-1] + if !strings.HasSuffix(memField, "MiB") { + continue + } + idx, err := strconv.Atoi(fields[0]) + if err != nil { + continue + } + pid := fields[1] + if _, err := strconv.Atoi(pid); err != nil { + continue + } + mem, err := strconv.ParseFloat(strings.TrimSuffix(memField, "MiB"), 64) + if err != nil { + continue + } + out = append(out, MthreadsGpuProc{ + Index: idx, + Pid: pid, + MemMiB: mem, + }) + } + return out +} + +func parseMthreadsMemPercent(raw string) float64 { + return parseMthreadsPercent(raw) / 100 +} + +func parseMthreadsGpuPercent(raw string) float64 { + return parseMthreadsPercent(raw) +} + +func parseMthreadsPercent(raw string) float64 { + raw = strings.TrimSpace(raw) + if raw == "" || strings.HasPrefix(raw, "<") { + return 0 + } + raw = strings.TrimSuffix(raw, "%") + v, err := strconv.ParseFloat(raw, 64) + if err != nil { + return 0 + } + return v +} + +func mergeMthreadsGpuProcs(utils, mems []MthreadsGpuProc) []MthreadsGpuProc { + type key struct { + index int + pid string + } + merged := map[key]*MthreadsGpuProc{} + order := make([]key, 0) + touch := func(index int, pid string) *MthreadsGpuProc { + k := key{index, pid} + if p, ok := merged[k]; ok { + return p + } + p := &MthreadsGpuProc{Index: index, Pid: pid} + merged[k] = p + order = append(order, k) + return p + } + for _, p := range utils { + dst := touch(p.Index, p.Pid) + dst.MemUtil = p.MemUtil + dst.GpuUtil = p.GpuUtil + } + for _, p := range mems { + dst := touch(p.Index, p.Pid) + dst.MemMiB = p.MemMiB + } + out := make([]MthreadsGpuProc, 0, len(order)) + for _, k := range order { + out = append(out, *merged[k]) + } + return out +} + +func selectMthreadsGpuProcs(procs []MthreadsGpuProc, podProcs map[string]struct{}, indexes []int) []MthreadsGpuProc { + if len(procs) == 0 || len(podProcs) == 0 || len(indexes) == 0 { + return nil + } + want := make(map[int]struct{}, len(indexes)) + for _, idx := range indexes { + want[idx] = struct{}{} + } + out := make([]MthreadsGpuProc, 0) + for _, p := range procs { + if _, ok := podProcs[p.Pid]; !ok { + continue + } + if _, ok := want[p.Index]; !ok { + continue + } + out = append(out, p) + } + return out +} + +func (m *SGuestMonitor) getPodMthreadsGpuMetrics() []*PodMthreadsGpuMetrics { + if len(m.mthreadsGpuMetrics) == 0 { + return nil + } + type acc struct { + mem float64 + memUtil float64 + gpuUtil float64 + } + by := map[int]*acc{} + for _, p := range m.mthreadsGpuMetrics { + a := by[p.Index] + if a == nil { + a = &acc{} + by[p.Index] = a + } + a.mem += p.MemMiB + a.memUtil += p.MemUtil + if p.GpuUtil > a.gpuUtil { + a.gpuUtil = p.GpuUtil + } + } + indexes := make([]int, 0, len(by)) + for idx := range by { + indexes = append(indexes, idx) + } + sort.Ints(indexes) + res := make([]*PodMthreadsGpuMetrics, 0, len(indexes)) + for _, idx := range indexes { + a := by[idx] + memUtil := a.memUtil + if memUtil > 1 { + memUtil = 1 + } + gpuUtil := a.gpuUtil + if gpuUtil > 100 { + gpuUtil = 100 + } + res = append(res, &PodMthreadsGpuMetrics{ + Index: idx, + Mem: a.mem, + MemUtil: memUtil, + GpuUtil: gpuUtil, + }) + } + return res +} + +type PodMthreadsGpuMetrics struct { + PodMetricMeta + + Index int + Mem float64 + MemUtil float64 + GpuUtil float64 +} + +func (m PodMthreadsGpuMetrics) GetName() string { + return "pod_mthreads_gpu" +} + +func (m PodMthreadsGpuMetrics) GetUniformName() string { + return "pod_gpu" +} + +func (m PodMthreadsGpuMetrics) GetTag() map[string]string { + return map[string]string{ + "index": strconv.Itoa(m.Index), + "dev_type": compute.GPU_TYPE, + "sharing_mode": compute.DEVICE_SHARING_MODE_EXCLUSIVE, + } +} + +func (m PodMthreadsGpuMetrics) ToMap() map[string]interface{} { + return map[string]interface{}{ + MTHREADS_GPU_MEM: m.Mem, + MTHREADS_GPU_MEM_UTIL: m.MemUtil, + MTHREADS_GPU_GPU_UTIL: m.GpuUtil, + } +} diff --git a/pkg/hostman/hostmetrics/container_mthreads_gpu_metrics_test.go b/pkg/hostman/hostmetrics/container_mthreads_gpu_metrics_test.go new file mode 100644 index 0000000000..df2f2b80ee --- /dev/null +++ b/pkg/hostman/hostmetrics/container_mthreads_gpu_metrics_test.go @@ -0,0 +1,100 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostmetrics + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseMthreadsProcessMonitor(t *testing.T) { + buf, err := os.ReadFile(filepath.Join("testdata", "mthreads-pm.txt")) + if err != nil { + t.Fatal(err) + } + got := parseMthreadsProcessMonitor(string(buf)) + if len(got) != 7 { + t.Fatalf("rows=%d, want 7", len(got)) + } + worker := got[2] + if worker.Index != 0 || worker.Pid != "1306725" || worker.MemUtil != 0.10 || worker.GpuUtil != 100 { + t.Fatalf("worker=%+v", worker) + } + if got[0].MemUtil != 0 || got[0].GpuUtil != 0 { + t.Fatalf("<1%% parsed as %+v", got[0]) + } +} + +func TestParseMthreadsProcessMemory(t *testing.T) { + buf, err := os.ReadFile(filepath.Join("testdata", "mthreads-processes.txt")) + if err != nil { + t.Fatal(err) + } + got := parseMthreadsProcessMemory(string(buf)) + if len(got) != 6 { + t.Fatalf("rows=%d, want 6", len(got)) + } + if got[1].Pid != "1305830" || got[1].MemMiB != 36 { + t.Fatalf("engine=%+v", got[1]) + } + if got[2].Pid != "1306725" || got[2].MemMiB != 8758 { + t.Fatalf("worker=%+v", got[2]) + } +} + +func TestAggregateMthreadsPodGpu(t *testing.T) { + pm, err := os.ReadFile(filepath.Join("testdata", "mthreads-pm.txt")) + if err != nil { + t.Fatal(err) + } + mem, err := os.ReadFile(filepath.Join("testdata", "mthreads-processes.txt")) + if err != nil { + t.Fatal(err) + } + procs := mergeMthreadsGpuProcs(parseMthreadsProcessMonitor(string(pm)), parseMthreadsProcessMemory(string(mem))) + podProcs := map[string]struct{}{ + "1305267": {}, + "1305830": {}, + "1306725": {}, + } + selected := selectMthreadsGpuProcs(procs, podProcs, []int{0}) + m := &SGuestMonitor{mthreadsGpuMetrics: selected} + got := m.getPodMthreadsGpuMetrics() + if len(got) != 1 { + t.Fatalf("gpus=%d, want 1", len(got)) + } + if got[0].Index != 0 || got[0].Mem != 8830 || got[0].MemUtil != 0.10 || got[0].GpuUtil != 100 { + t.Fatalf("metric=%+v", got[0]) + } + fields := got[0].ToMap() + if fields[MTHREADS_GPU_MEM] != 8830.0 || fields[MTHREADS_GPU_MEM_UTIL] != 0.10 || fields[MTHREADS_GPU_GPU_UTIL] != 100.0 { + t.Fatalf("fields=%v", fields) + } + if got[0].GetUniformName() != "pod_gpu" { + t.Fatalf("uniform=%s", got[0].GetUniformName()) + } +} + +func TestAggregateMthreadsMemUtilCapAndGpuMax(t *testing.T) { + m := &SGuestMonitor{mthreadsGpuMetrics: []MthreadsGpuProc{ + {Index: 0, Pid: "1", MemMiB: 100, MemUtil: 0.6, GpuUtil: 20}, + {Index: 0, Pid: "2", MemMiB: 50, MemUtil: 0.6, GpuUtil: 90}, + }} + got := m.getPodMthreadsGpuMetrics() + if len(got) != 1 || got[0].Mem != 150 || got[0].MemUtil != 1 || got[0].GpuUtil != 90 { + t.Fatalf("metric=%+v", got) + } +} diff --git a/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go b/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go index d92f9568a8..49082c419d 100644 --- a/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go +++ b/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go @@ -159,10 +159,16 @@ func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string podDesc := pod.GetDesc() hasGpu := false for i := range podDesc.IsolatedDevices { - if podDesc.IsolatedDevices[i].DevType != compute.GPU_TYPE { + dev := podDesc.IsolatedDevices[i] + vendorId := strings.ToLower(strings.Split(dev.VendorDeviceId, ":")[0]) + if vendorId == compute.MTHREADS_VENDOR_ID { + hasGpu = true + break + } + if dev.DevType != compute.GPU_TYPE { continue } - if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) { + if !utils.IsInStringArray(dev.SharingMode, compute.VIRTUAL_SHARING_MODES) { continue } hasGpu = true diff --git a/pkg/hostman/hostmetrics/hostmetrics.go b/pkg/hostman/hostmetrics/hostmetrics.go index 684df1b139..3da0b9eec7 100644 --- a/pkg/hostman/hostmetrics/hostmetrics.go +++ b/pkg/hostman/hostmetrics/hostmetrics.go @@ -64,6 +64,8 @@ type IHostInfo interface { HasContainerNvidiaGpu() bool HasContainerVastaitechGpu() bool HasContainerCphAmdGpu() bool + HasContainerMthreadsGpu() bool + GetMthreadsGpuIndexes(ids, addrs []string) []int GetNvidiaGpuIndexMemoryMap() map[string]int GetNvidiaGpuIndexByDeviceIds(ids []string) map[string]int ReportHostDmesg(data []compute.SKmsgEntry) error @@ -199,6 +201,7 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor { var nvidiaGpuMetrics []NvidiaGpuProcessMetrics = nil var vastaitechGpuMetrics []VastaitechGpuProcessMetrics = nil var cphAmdGpuMetrics []CphAmdGpuProcessMetrics = nil + var mthreadsGpuMetrics []MthreadsGpuProc = nil var gpuPodProcs = s.collectGpuPodsProcesses() guestmanager.Servers.Range(func(k, v interface{}) bool { @@ -274,13 +277,19 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor { log.Errorf("GetCphAmdGpuProcessMetrics %s", err) } } + if s.hostInfo.HasContainerMthreadsGpu() { + mthreadsGpuMetrics, err = GetMthreadsGpuProcessMetrics() + if err != nil { + log.Errorf("GetMthreadsGpuProcessMetrics %s", err) + } + } } podStat, podProcs := GetPodStatsById(podStats, gpuPodProcs, guestId) if podStat != nil { gm, err := NewGuestPodMonitor( instance, guestName, guestId, podStat, - nvidiaGpuMetrics, vastaitechGpuMetrics, cphAmdGpuMetrics, + nvidiaGpuMetrics, vastaitechGpuMetrics, cphAmdGpuMetrics, mthreadsGpuMetrics, s.hostInfo, podProcs, nicsDesc, int(vcpuCount), ) if err != nil { @@ -633,6 +642,7 @@ type SGuestMonitor struct { nvidiaGpuAssigned []nvidiaGpuAssignedQuota vastaitechGpuMetrics []VastaitechGpuProcessMetrics cphAmdGpuMetrics []CphAmdGpuProcessMetrics + mthreadsGpuMetrics []MthreadsGpuProc instance guestman.GuestRuntimeInstance sysFs sysfs.SysFs @@ -650,6 +660,7 @@ func NewGuestMonitor(instance guestman.GuestRuntimeInstance, name, id string, pi func NewGuestPodMonitor( instance guestman.GuestRuntimeInstance, name, id string, stat *stats.PodStats, nvidiaGpuMetrics []NvidiaGpuProcessMetrics, vastaitechGpuMetrics []VastaitechGpuProcessMetrics, cphAmdGpuMetrics []CphAmdGpuProcessMetrics, + mthreadsGpuMetrics []MthreadsGpuProc, hostInstance IHostInfo, podProcs map[string]struct{}, nics []*desc.SGuestNetwork, cpuCount int, ) (*SGuestMonitor, error) { m, err := newGuestMonitor(instance, name, id, nil, nics, cpuCount) @@ -662,12 +673,25 @@ func NewGuestPodMonitor( hasNvGpu := false hasCphAmdGpu := false hasVastaitechGpu := false + hasMthreadsGpu := false + mthreadsIds := make([]string, 0) + mthreadsAddrs := make([]string, 0) nvAssignInputs := make([]nvidiaGpuAssignInput, 0) for i := range podDesc.IsolatedDevices { - if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) { + dev := podDesc.IsolatedDevices[i] + vendorId := strings.ToLower(strings.Split(dev.VendorDeviceId, ":")[0]) + if vendorId == compute.MTHREADS_VENDOR_ID { + hasMthreadsGpu = true + if dev.Id != "" { + mthreadsIds = append(mthreadsIds, dev.Id) + } + if dev.Addr != "" { + mthreadsAddrs = append(mthreadsAddrs, dev.Addr) + } + } + if !utils.IsInStringArray(dev.SharingMode, compute.VIRTUAL_SHARING_MODES) { continue } - vendorId := strings.Split(podDesc.IsolatedDevices[i].VendorDeviceId, ":")[0] switch vendorId { case compute.NVIDIA_VENDOR_ID: hasNvGpu = true @@ -698,6 +722,10 @@ func NewGuestPodMonitor( if hasCphAmdGpu { m.cphAmdGpuMetrics = GetPodCphAmdGpuMetrics(cphAmdGpuMetrics, podProcs) } + if hasMthreadsGpu { + indexes := hostInstance.GetMthreadsGpuIndexes(mthreadsIds, mthreadsAddrs) + m.mthreadsGpuMetrics = selectMthreadsGpuProcs(mthreadsGpuMetrics, podProcs, indexes) + } return m, nil } diff --git a/pkg/hostman/hostmetrics/testdata/mthreads-pm.txt b/pkg/hostman/hostmetrics/testdata/mthreads-pm.txt new file mode 100644 index 0000000000..85a589b145 --- /dev/null +++ b/pkg/hostman/hostmetrics/testdata/mthreads-pm.txt @@ -0,0 +1,8 @@ + gpu pid memUtil gpuUtil + 0 1305267 <1% 0% + 0 1305830 <1% 0% + 0 1306725 10% 100% + 0 1306726 <1% 0% + 1 1305267 0% 0% + 1 1306726 10% 100% + 1 1306725 <1% 0% diff --git a/pkg/hostman/hostmetrics/testdata/mthreads-processes.txt b/pkg/hostman/hostmetrics/testdata/mthreads-processes.txt new file mode 100644 index 0000000000..90736e67cf --- /dev/null +++ b/pkg/hostman/hostmetrics/testdata/mthreads-processes.txt @@ -0,0 +1,12 @@ +Processes: +ID PID Process name GPU Memory + Usage ++-------------------------------------------------------------------+ +0 1305267 /usr/bin/python 36MiB +0 1305830 VLLM::EngineCore 36MiB +0 1306725 VLLM::Worker_TP0 8758MiB +0 1306726 VLLM::Worker_TP1 36MiB +1 1306726 VLLM::Worker_TP1 8758MiB +1 1306725 VLLM::Worker_TP0 36MiB ++-------------------------------------------------------------------+ +| 0% 9081MiB(81920MiB) | diff --git a/pkg/hostman/isolated_device/container_device.go b/pkg/hostman/isolated_device/container_device.go index b66a948c3e..7d6fb95b06 100644 --- a/pkg/hostman/isolated_device/container_device.go +++ b/pkg/hostman/isolated_device/container_device.go @@ -37,22 +37,24 @@ var ( type ContainerDeviceType string const ( - ContainerDeviceTypeCphAMDGPU ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU - ContainerDeviceTypeCphASOPBinder ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER - ContainerNetintCAASIC ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC - ContainerNetintCAQuadra ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA - ContainerDeviceTypeNvidiaGpu ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU - ContainerDeviceTypeNvidiaMps ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS - ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE - ContainerDeviceTypeNvidiaHAMI ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI - ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU - ContainerDeviceTypeAscendNpuHami ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI - ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU - ContainerDeviceTypeHygonDcu ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU - ContainerDeviceTypeHygonDcuHami ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI - ContainerDeviceTypeIluvatarGpu ContainerDeviceType = api.CONTAINER_DEV_ILUVATAR_GPU - ContainerDeviceTypeTHeadPpu ContainerDeviceType = api.CONTAINER_DEV_THEAD_PPU - ContainerDeviceTypeKunlunxinXpu ContainerDeviceType = api.CONTAINER_DEV_KUNLUNXIN_XPU + ContainerDeviceTypeCphAMDGPU ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU + ContainerDeviceTypeCphASOPBinder ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER + ContainerNetintCAASIC ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC + ContainerNetintCAQuadra ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA + ContainerDeviceTypeNvidiaGpu ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU + ContainerDeviceTypeNvidiaMps ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS + ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE + ContainerDeviceTypeNvidiaHAMI ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI + ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU + ContainerDeviceTypeAscendNpuHami ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI + ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU + ContainerDeviceTypeHygonDcu ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU + ContainerDeviceTypeHygonDcuHami ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI + ContainerDeviceTypeIluvatarGpu ContainerDeviceType = api.CONTAINER_DEV_ILUVATAR_GPU + ContainerDeviceTypeTHeadPpu ContainerDeviceType = api.CONTAINER_DEV_THEAD_PPU + ContainerDeviceTypeKunlunxinXpu ContainerDeviceType = api.CONTAINER_DEV_KUNLUNXIN_XPU + ContainerDeviceTypeMthreadsGpu ContainerDeviceType = api.CONTAINER_DEV_MTHREADS_GPU + ContainerDeviceTypeMthreadsGpuHami ContainerDeviceType = api.CONTAINER_DEV_MTHREADS_GPU_HAMI ) func GetContainerDeviceManager(devType ContainerDeviceType) (IContainerDeviceManager, error) { diff --git a/pkg/hostman/isolated_device/container_device/mthreads_gmi.go b/pkg/hostman/isolated_device/container_device/mthreads_gmi.go new file mode 100644 index 0000000000..b768b9da70 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/mthreads_gmi.go @@ -0,0 +1,193 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" +) + +const mthreadsDeviceTypePhysical = "Physical" + +type parsedMthreadsGPU struct { + Index int + Name string + BusId string + DeviceType string + Perf string + GPUUtil int + MemoryUsedMB int + MemorySizeMB int + TempC int + MPCCapable string + ECCMode string +} + +var ( + mthreadsIndexRe = regexp.MustCompile(`^(\d+)\s+(.+)$`) + mthreadsMemRe = regexp.MustCompile(`(\d+)%\s+(\d+)MiB\((\d+)MiB\)`) + mthreadsTempRe = regexp.MustCompile(`(\d+)C`) +) + +func splitMthreadsCols(line string) []string { + parts := strings.Split(line, "|") + cols := make([]string, 0, len(parts)) + for _, p := range parts { + cols = append(cols, strings.TrimSpace(p)) + } + return cols +} + +func parseMthreadsGmiBlock(lines []string) *parsedMthreadsGPU { + if len(lines) == 0 { + return nil + } + cols := splitMthreadsCols(lines[0]) + if len(cols) < 3 { + return nil + } + head := mthreadsIndexRe.FindStringSubmatch(cols[0]) + if len(head) != 3 { + return nil + } + idx, err := strconv.Atoi(head[1]) + if err != nil { + return nil + } + gpu := &parsedMthreadsGPU{ + Index: idx, + Name: strings.TrimSpace(head[2]), + BusId: cols[1], + } + if mem := mthreadsMemRe.FindStringSubmatch(cols[2]); len(mem) == 4 { + gpu.GPUUtil, _ = strconv.Atoi(mem[1]) + gpu.MemoryUsedMB, _ = strconv.Atoi(mem[2]) + gpu.MemorySizeMB, _ = strconv.Atoi(mem[3]) + } + if len(lines) >= 2 { + cols2 := splitMthreadsCols(lines[1]) + if len(cols2) > 0 { + fields := strings.Fields(cols2[0]) + if len(fields) > 0 { + gpu.DeviceType = fields[0] + } + if len(fields) > 1 { + gpu.Perf = fields[1] + } + } + if len(cols2) >= 3 { + if tm := mthreadsTempRe.FindStringSubmatch(cols2[2]); len(tm) == 2 { + gpu.TempC, _ = strconv.Atoi(tm[1]) + } + parts := strings.Fields(cols2[2]) + if len(parts) >= 2 { + gpu.MPCCapable = parts[len(parts)-1] + } + } + } + if len(lines) >= 3 { + cols3 := splitMthreadsCols(lines[2]) + for i := len(cols3) - 1; i >= 0; i-- { + if cols3[i] != "" { + gpu.ECCMode = cols3[i] + break + } + } + } + return gpu +} + +func parseMthreadsGmi(output string) []*parsedMthreadsGPU { + var out []*parsedMthreadsGPU + var buf []string + flush := func() { + if gpu := parseMthreadsGmiBlock(buf); gpu != nil { + out = append(out, gpu) + } + buf = nil + } + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if strings.HasPrefix(line, "+") || strings.HasPrefix(line, "-") { + flush() + continue + } + buf = append(buf, line) + } + flush() + return out +} + +func isMthreadsPhysical(gpu *parsedMthreadsGPU) bool { + return gpu != nil && strings.EqualFold(gpu.DeviceType, mthreadsDeviceTypePhysical) +} + +func mthreadsPCIAddrCandidates(busId string) []string { + return iluvatarPCIAddrCandidates(busId) +} + +func mthreadsDevNode(index int) string { + return fmt.Sprintf("/dev/mtgpu.%d", index) +} + +func mthreadsDeviceSpec(devPath string) *runtimeapi.Device { + return &runtimeapi.Device{ + ContainerPath: devPath, + HostPath: devPath, + Permissions: "rwm", + } +} + +func buildMthreadsRuntimeEnvs(indices []string) []*runtimeapi.KeyValue { + value := strings.Join(indices, ",") + return []*runtimeapi.KeyValue{ + {Key: "MTHREADS_VISIBLE_DEVICES", Value: value}, + {Key: "MTHREADS_DRIVER_CAPABILITIES", Value: "all"}, + // {Key: "MUSA_VISIBLE_DEVICES", Value: value}, + // {Key: "CUDA_VISIBLE_DEVICES", Value: value}, + } +} + +func buildMthreadsExtraConfigures(indices []string) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + if len(indices) == 0 { + return nil, nil + } + return buildMthreadsRuntimeEnvs(indices), nil +} + +func buildMthreadsHamiQosEnvs(memoryLimitMiB, smUtilLimit int) []*runtimeapi.KeyValue { + if memoryLimitMiB <= 0 { + return nil + } + return []*runtimeapi.KeyValue{ + { + Key: "MTHREADS_QOS_MEMORY_LIMIT", + Value: strconv.FormatInt(computeapi.MthreadsSgpuMemoryLimitBytes(memoryLimitMiB), 10), + }, + { + Key: "MTHREADS_QOS_COMPUTING_POWER_WEIGHT", + Value: strconv.Itoa(computeapi.MthreadsSgpuComputingPowerWeight(smUtilLimit)), + }, + } +} diff --git a/pkg/hostman/isolated_device/container_device/mthreads_gmi_test.go b/pkg/hostman/isolated_device/container_device/mthreads_gmi_test.go new file mode 100644 index 0000000000..3659cc2873 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/mthreads_gmi_test.go @@ -0,0 +1,110 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseMthreadsGmiS5000(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "s5000-8gpu.txt")) + require.NoError(t, err) + + gpus := parseMthreadsGmi(string(raw)) + require.Len(t, gpus, 9) + + physical := make([]*parsedMthreadsGPU, 0, 8) + for _, gpu := range gpus { + if isMthreadsPhysical(gpu) { + physical = append(physical, gpu) + } + } + require.Len(t, physical, 8) + + wantBus := []string{ + "00000000:18:00.0", + "00000000:3a:00.0", + "00000000:4b:00.0", + "00000000:5c:00.0", + "00000000:9a:00.0", + "00000000:ba:00.0", + "00000000:ca:00.0", + "00000000:da:00.0", + } + wantTemp := []int{45, 38, 44, 36, 45, 37, 45, 38} + for i, gpu := range physical { + assert.Equal(t, i, gpu.Index) + assert.Equal(t, "MTT S5000", gpu.Name) + assert.Equal(t, wantBus[i], gpu.BusId) + assert.Equal(t, 0, gpu.GPUUtil) + assert.Equal(t, 0, gpu.MemoryUsedMB) + assert.Equal(t, 81920, gpu.MemorySizeMB) + assert.Equal(t, wantTemp[i], gpu.TempC) + assert.Equal(t, "P0", gpu.Perf) + assert.Equal(t, "YES", gpu.MPCCapable) + assert.Equal(t, "On-die, EDC", gpu.ECCMode) + } + + virt := gpus[8] + assert.Equal(t, 8, virt.Index) + assert.Equal(t, "Virt", virt.DeviceType) + assert.False(t, isMthreadsPhysical(virt)) + assert.Equal(t, 10, virt.GPUUtil) + assert.Equal(t, 1024, virt.MemoryUsedMB) +} + +func TestBuildMthreadsExtraConfigures(t *testing.T) { + envs, mounts := buildMthreadsExtraConfigures([]string{"0", "1", "2", "3"}) + assert.Nil(t, mounts) + require.Len(t, envs, 2) + got := map[string]string{} + for _, env := range envs { + got[env.Key] = env.Value + } + assert.Equal(t, "0,1,2,3", got["MTHREADS_VISIBLE_DEVICES"]) + assert.Equal(t, "all", got["MTHREADS_DRIVER_CAPABILITIES"]) + + envs, mounts = buildMthreadsExtraConfigures(nil) + assert.Nil(t, envs) + assert.Nil(t, mounts) +} + +func TestBuildMthreadsHamiQosEnvs(t *testing.T) { + envs := buildMthreadsHamiQosEnvs(2048, 25) + require.Len(t, envs, 2) + got := map[string]string{} + for _, env := range envs { + got[env.Key] = env.Value + } + assert.Equal(t, "2147483648", got["MTHREADS_QOS_MEMORY_LIMIT"]) + assert.Equal(t, "25", got["MTHREADS_QOS_COMPUTING_POWER_WEIGHT"]) + + zero := buildMthreadsHamiQosEnvs(0, 25) + assert.Nil(t, zero) + + defaultWeight := buildMthreadsHamiQosEnvs(512, 0) + require.Len(t, defaultWeight, 2) + assert.Equal(t, "1", defaultWeight[1].Value) +} + +func TestMthreadsDevNode(t *testing.T) { + assert.Equal(t, "/dev/mtgpu.0", mthreadsDevNode(0)) + assert.Equal(t, "/dev/mtgpu.7", mthreadsDevNode(7)) +} diff --git a/pkg/hostman/isolated_device/container_device/mthreads_gpu.go b/pkg/hostman/isolated_device/container_device/mthreads_gpu.go new file mode 100644 index 0000000000..0a0ee395c1 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/mthreads_gpu.go @@ -0,0 +1,216 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + isolated_device.RegisterContainerDeviceManager(newMthreadsGPUManager()) +} + +type mthreadsGPUManager struct{} + +func newMthreadsGPUManager() *mthreadsGPUManager { + return &mthreadsGPUManager{} +} + +func (m *mthreadsGPUManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeMthreadsGpu +} + +func (m *mthreadsGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { + return getMthreadsGPUs(m, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE) +} + +func (m *mthreadsGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *mthreadsGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + return newMthreadsContainerDevices(dev) +} + +func (m *mthreadsGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + return buildMthreadsExtraConfigures(collectMthreadsVisibleIndices(devs)) +} + +type mthreadsGPU struct { + manager isolated_device.IContainerDeviceManager + *BaseDevice + + memSize int + gpuIndex int +} + +func (dev *mthreadsGPU) GetMemorySize() int { + return dev.memSize +} + +func (dev *mthreadsGPU) GetIndex() int { + return dev.gpuIndex +} + +func (dev *mthreadsGPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +const defaultMthreadsGmiPath = "/usr/bin/mthreads-gmi" + +func mthreadsGmiPath() string { + p := options.HostOptions.MthreadsGmiPath + if p != "" { + return p + } + return defaultMthreadsGmiPath +} + +func runMthreadsGmi(args ...string) (string, error) { + bin := mthreadsGmiPath() + cmd := procutils.NewRemoteCommandAsFarAsPossible(bin, args...) + out, err := cmd.Output() + if err != nil { + return string(out), errors.Wrapf(err, "mthreads-gmi %s", strings.Join(args, " ")) + } + return string(out), nil +} + +func lookupMthreadsPCIDevice(busId, modelName string) *isolated_device.PCIDevice { + cands := mthreadsPCIAddrCandidates(busId) + for _, addr := range cands { + pciOutput, err := isolated_device.GetPCIStrByAddr(addr) + if err != nil || len(pciOutput) == 0 { + log.Warningf("mthreads GetPCIStrByAddr %s: %v", addr, err) + continue + } + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + if modelName != "" { + dev.ModelName = modelName + } + return dev + } + fallbackAddr := busId + if len(cands) > 0 { + fallbackAddr = cands[len(cands)-1] + } + return &isolated_device.PCIDevice{ + VendorId: computeapi.MTHREADS_VENDOR_ID, + VendorName: "MTHREADS", + Addr: fallbackAddr, + ModelName: modelName, + DeviceName: modelName, + } +} + +func collectMthreadsVisibleIndices(devs []*hostapi.ContainerDevice) []string { + indices := []string{} + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + continue + } + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*mthreadsGPUManager); !ok { + continue + } + if dev.IsolatedDevice.Path != "" { + indices = append(indices, dev.IsolatedDevice.Path) + continue + } + if dev.IsolatedDevice.Index >= 0 { + indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index)) + } + } + return indices +} + +func newMthreadsContainerDevices(dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + if dev.IsolatedDevice == nil { + return nil, nil, errors.Errorf("isolated device is nil") + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id) + } + gpuDev, ok := iDev.(*mthreadsGPU) + if !ok { + return nil, nil, errors.Errorf("device %s is not mthreads gpu", dev.IsolatedDevice.Id) + } + ctrDevs := []*runtimeapi.Device{} + node := mthreadsDevNode(gpuDev.GetIndex()) + if hygonPathExists(node) { + ctrDevs = append(ctrDevs, mthreadsDeviceSpec(node)) + } else { + log.Warningf("mthreads container device %s not found, skip", node) + } + return ctrDevs, nil, nil +} + +func getMthreadsGPUs(manager isolated_device.IContainerDeviceManager, sharingMode string) ([]isolated_device.IDevice, error) { + bin := mthreadsGmiPath() + if !hygonPathExists(bin) { + log.Infof("mthreads gpu probe skipped: mthreads-gmi not found at %s", bin) + return nil, nil + } + + tableOut, err := runMthreadsGmi() + if err != nil { + return nil, err + } + parsed := parseMthreadsGmi(tableOut) + devs := make([]isolated_device.IDevice, 0, len(parsed)) + for _, gpu := range parsed { + if !isMthreadsPhysical(gpu) { + log.Infof("mthreads gpu %d device type %q, skip", gpu.Index, gpu.DeviceType) + continue + } + pciDev := lookupMthreadsPCIDevice(gpu.BusId, gpu.Name) + indexStr := strconv.Itoa(gpu.Index) + dev := &mthreadsGPU{ + manager: manager, + BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, sharingMode, 1), + memSize: gpu.MemorySizeMB, + gpuIndex: gpu.Index, + } + if gpu.Name != "" { + dev.SetModelName(gpu.Name) + } + log.Infof("mthreads gpu idx=%d model=%s pci=%s memMiB=%d path=%s node=%s", + gpu.Index, gpu.Name, pciDev.Addr, gpu.MemorySizeMB, indexStr, mthreadsDevNode(gpu.Index)) + devs = append(devs, dev) + } + if len(devs) == 0 { + log.Infof("mthreads gpu probe finished: no physical devices found") + return nil, nil + } + return devs, nil +} diff --git a/pkg/hostman/isolated_device/container_device/mthreads_hami.go b/pkg/hostman/isolated_device/container_device/mthreads_hami.go new file mode 100644 index 0000000000..dec4e06dac --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/mthreads_hami.go @@ -0,0 +1,117 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "strconv" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +const mthreadsSgpuPolicyWeak = 1 + +func init() { + isolated_device.RegisterContainerDeviceManager(newMthreadsGPUHamiManager()) +} + +type mthreadsGPUHamiManager struct{} + +func newMthreadsGPUHamiManager() *mthreadsGPUHamiManager { + return &mthreadsGPUHamiManager{} +} + +func (m *mthreadsGPUHamiManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeMthreadsGpuHami +} + +func (m *mthreadsGPUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error) { + devs, err := getMthreadsGPUs(m, computeapi.DEVICE_SHARING_MODE_HAMI) + if err != nil { + return nil, err + } + for _, dev := range devs { + gpu, ok := dev.(*mthreadsGPU) + if !ok { + continue + } + configureMthreadsSgpu(gpu.GetIndex()) + } + return devs, nil +} + +func (m *mthreadsGPUHamiManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *mthreadsGPUHamiManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + return newMthreadsContainerDevices(dev) +} + +func (m *mthreadsGPUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + indices := make([]string, 0) + memoryLimit := 0 + smLimit := 0 + seenLimit := false + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + continue + } + if _, ok := iDev.GetContainerDeviceManager().(*mthreadsGPUHamiManager); !ok { + continue + } + if dev.IsolatedDevice.Path != "" { + indices = append(indices, dev.IsolatedDevice.Path) + } else if dev.IsolatedDevice.Index >= 0 { + indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index)) + } + if !seenLimit { + memoryLimit = dev.IsolatedDevice.MemoryLimit + smLimit = dev.IsolatedDevice.SmUtilLimit + seenLimit = true + } + } + envs, mounts := buildMthreadsExtraConfigures(indices) + if len(envs) == 0 { + return nil, nil + } + envs = append(envs, buildMthreadsHamiQosEnvs(memoryLimit, smLimit)...) + return envs, mounts +} + +func configureMthreadsSgpu(index int) { + writeMthreadsSgpuProc(index, "policy", mthreadsSgpuPolicyWeak) + writeMthreadsSgpuProc(index, "max_inst", computeapi.MthreadsSgpuMaxInstances) +} + +func writeMthreadsSgpuProc(index int, node string, value int) { + procPath := fmt.Sprintf("/proc/sgpu_km/%d/%s", index, node) + out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", fmt.Sprintf("echo %d > %s", value, procPath)).Output() + if err != nil { + log.Warningf("mthreads sgpu set %s=%d: %v %s", procPath, value, err, out) + } +} diff --git a/pkg/hostman/isolated_device/container_device/testdata/s5000-8gpu.txt b/pkg/hostman/isolated_device/container_device/testdata/s5000-8gpu.txt new file mode 100644 index 0000000000..d97e6eaccd --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/testdata/s5000-8gpu.txt @@ -0,0 +1,54 @@ +Tue Oct 13 20:09:50 2026 +--------------------------------------------------------------------- + mthreads-gmi:2.3.3 Driver Version:3.3.8-server +--------------------------------------------------------------------- +ID Name |PCIe |%GPU Mem + Device Type Perf |Pcie Lane Width |Temp MPC Capable + | ECC Mode ++-------------------------------------------------------------------+ +0 MTT S5000 |00000000:18:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |45C YES + | On-die, EDC ++-------------------------------------------------------------------+ +1 MTT S5000 |00000000:3a:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |38C YES + | On-die, EDC ++-------------------------------------------------------------------+ +2 MTT S5000 |00000000:4b:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |44C YES + | On-die, EDC ++-------------------------------------------------------------------+ +3 MTT S5000 |00000000:5c:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |36C YES + | On-die, EDC ++-------------------------------------------------------------------+ +4 MTT S5000 |00000000:9a:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |45C YES + | On-die, EDC ++-------------------------------------------------------------------+ +5 MTT S5000 |00000000:ba:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |37C YES + | On-die, EDC ++-------------------------------------------------------------------+ +6 MTT S5000 |00000000:ca:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |45C YES + | On-die, EDC ++-------------------------------------------------------------------+ +7 MTT S5000 |00000000:da:00.0 |0% 0MiB(81920MiB) + Physical P0 |16x(16x) |38C YES + | On-die, EDC ++-------------------------------------------------------------------+ +8 MTT S5000 MPC |00000000:ea:00.0 |10% 1024MiB(81920MiB) + Virt P0 |16x(16x) |40C YES + | Off ++-------------------------------------------------------------------+ + +--------------------------------------------------------------------- + +--------------------------------------------------------------------- +Processes: +ID PID Process name GPU Memory + Usage ++-------------------------------------------------------------------+ + No running processes found +--------------------------------------------------------------------- diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 253659f5b1..613b3249cb 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -399,6 +399,34 @@ func (man *isolatedDeviceManager) probeContainerKunlunxinXPUs(enable bool) { } } +func (man *isolatedDeviceManager) probeContainerMthreadsGPUs(enable, enableHami bool) { + devType := ContainerDeviceTypeMthreadsGpu + if enableHami { + devType = ContainerDeviceTypeMthreadsGpuHami + } else if !enable { + log.Infof("mthreads gpu probe skipped: enable_container_mthreads_gpu=false and enable_container_mthreads_gpu_hami=false") + return + } + devman, err := GetContainerDeviceManager(devType) + if err != nil { + log.Errorf("no container device manager %s found: %v", devType, err) + return + } + devs, err := devman.ProbeDevices() + if err != nil { + log.Warningf("Probe container mthreads gpu devices: %v", err) + return + } + if len(devs) == 0 { + log.Infof("mthreads gpu probe finished: no devices found") + return + } + for idx, dev := range devs { + man.devices = append(man.devices, dev) + log.Infof("Add Container mthreads GPU device: %d => %#v", idx, dev) + } +} + func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) { if skipGPUs { return @@ -567,16 +595,18 @@ type SIsolatedDeviceProbeOptions struct { SkipUSBs bool SkipCustomDevs bool - EnableCudaHAMI bool - EnableCudaMps bool - EnableContainerAscendNpu bool - EnableContainerAscendNpuHAMI bool - EnableContainerHygonDCU bool - EnableContainerHygonDCUHAMI bool - EnableContainerIluvatarGPU bool - EnableContainerTHeadPPU bool - EnableContainerKunlunxinXPU bool - EnableWhitelist bool + EnableCudaHAMI bool + EnableCudaMps bool + EnableContainerAscendNpu bool + EnableContainerAscendNpuHAMI bool + EnableContainerHygonDCU bool + EnableContainerHygonDCUHAMI bool + EnableContainerIluvatarGPU bool + EnableContainerTHeadPPU bool + EnableContainerKunlunxinXPU bool + EnableContainerMthreadsGPU bool + EnableContainerMthreadsGPUHami bool + EnableWhitelist bool SriovNics, OvsOffloadNics []HostNic @@ -596,6 +626,7 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti man.probeContainerIluvatarGPUs(opts.EnableContainerIluvatarGPU) man.probeContainerTHeadPPUs(opts.EnableContainerTHeadPPU) man.probeContainerKunlunxinXPUs(opts.EnableContainerKunlunxinXPU) + man.probeContainerMthreadsGPUs(opts.EnableContainerMthreadsGPU, opts.EnableContainerMthreadsGPUHami) } else { log.Infof("==== ProbePCIDevices: not container host, hygon container probe will NOT run (use host_type=container for hygon dcu)") devModels, err := man.getCustomIsolatedDeviceModels() diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 01c56fd889..8e77d430bd 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -300,6 +300,10 @@ type SHostOptions struct { KunlunxinXreHome string `help:"kunlunxin xre home" default:"/usr/local/xpu"` KunlunxinXpuSmiPath string `help:"kunlunxin xpu-smi path" default:"/usr/local/bin/xpu-smi"` + EnableContainerMthreadsGPU bool `help:"enable container mthreads gpu" default:"true"` + EnableContainerMthreadsGPUHami bool `help:"enable container mthreads gpu hami" default:"false"` + MthreadsGmiPath string `help:"mthreads-gmi path" default:"/usr/bin/mthreads-gmi"` + EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"` DisableReconcileContainer bool `help:"disable reconcile container" default:"false"` diff --git a/pkg/hostman/system_service/telegraf.go b/pkg/hostman/system_service/telegraf.go index afc09368a9..145b8418e5 100644 --- a/pkg/hostman/system_service/telegraf.go +++ b/pkg/hostman/system_service/telegraf.go @@ -41,6 +41,7 @@ const ( TELEGRAF_INPUT_IXSMI = "ixsmi" TELEGRAF_INPUT_PPUSMI = "ppusmi" TELEGRAF_INPUT_XPUSMI = "xpusmi" + TELEGRAF_INPUT_MTHREADSGMI = "mthreadsgmi" TELEGRAF_INPUT_NVIDIASMI = "nvidia-smi" TELEGRAF_INPUT_NPUSMI = "npu-smi" TELEGRAF_INPUT_CONF_LIB_PATH = "lib_path" @@ -380,6 +381,13 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string { conf += "\n" } + if mtgmi, ok := kwargs[TELEGRAF_INPUT_MTHREADSGMI]; ok { + mtgmiMap, _ := mtgmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_MTHREADSGMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", mtgmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + conf += "\n" + } + if _, ok := kwargs[TELEGRAF_INPUT_NVIDIASMI]; ok { conf += "[[inputs.nvidia_smi]]\n" conf += "\n" diff --git a/pkg/llm/drivers/llm_container/vllm.go b/pkg/llm/drivers/llm_container/vllm.go index 94e1b5bf97..8b26ec6c68 100644 --- a/pkg/llm/drivers/llm_container/vllm.go +++ b/pkg/llm/drivers/llm_container/vllm.go @@ -433,6 +433,7 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo iluvatar := models.HasIluvatarDevices(llm, sku) thead := models.HasTHeadDevices(llm, sku) kunlunxin := models.HasKunlunxinDevices(llm, sku) + mthreads := models.HasMthreadsDevices(llm, sku) startScript := buildVLLMEntrypointScript(modelPath, tensorParallelSize, backendParameters, effSpec, hygon) envs := []*commonapi.ContainerKeyValue{ { @@ -465,7 +466,7 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo Envs: envs, }, } - if hygon || iluvatar || thead || kunlunxin { + if hygon || iluvatar || thead || kunlunxin || mthreads { spec.Command = []string{"/bin/bash", "-c"} } if hasMountedModels { diff --git a/pkg/llm/drivers/llm_container/vllm_hygon_test.go b/pkg/llm/drivers/llm_container/vllm_hygon_test.go index ba0101360f..34c1a1db61 100644 --- a/pkg/llm/drivers/llm_container/vllm_hygon_test.go +++ b/pkg/llm/drivers/llm_container/vllm_hygon_test.go @@ -190,3 +190,46 @@ func TestVLLMGetContainerSpecKunlunxinRuntime(t *testing.T) { t.Fatalf("args should not source dtk env for kunlunxin, got %#v", spec.Args) } } + +func TestVLLMGetContainerSpecMthreadsRuntime(t *testing.T) { + v := newVLLM().(*vllm) + hostPaths := api.HostPaths{ + { + Type: "directory", + Path: "/data/model/Qwen", + Containers: api.ContainerHostPathRelations{ + "0": {MountPath: "/data/model/Qwen"}, + }, + }, + } + sku := &models.SLLMSku{ + SLLMSkuBase: models.SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU}, + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU}, + }, + }, + LLMType: string(api.LLM_CONTAINER_VLLM), + Source: api.LLM_MODEL_SOURCE_LOCAL_PATH, + LocalPath: "/data/model/Qwen", + } + sku.HostPaths = &hostPaths + image := &models.SLLMImage{} + out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "") + if out == nil { + t.Fatal("expected container spec") + } + spec := &out.ContainerSpec + if spec.Capabilities != nil { + t.Fatalf("expected nil capabilities for mthreads, got %#v", spec.Capabilities) + } + if spec.SecurityContext != nil { + t.Fatalf("expected nil security context for mthreads, got %#v", spec.SecurityContext) + } + if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" { + t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command) + } + if len(spec.Args) == 0 || !strings.Contains(spec.Args[0], "--tensor-parallel-size 2") { + t.Fatalf("args = %#v, want tensor-parallel-size 2", spec.Args) + } +} diff --git a/pkg/llm/models/llm_base.go b/pkg/llm/models/llm_base.go index 1de2596a63..c4554ac102 100644 --- a/pkg/llm/models/llm_base.go +++ b/pkg/llm/models/llm_base.go @@ -437,6 +437,23 @@ func HasTHeadDevices(llm *SLLM, sku *SLLMSku) bool { return false } +// HasMthreadsDevices reports whether effective devices include Moore Threads GPU. +func HasMthreadsDevices(llm *SLLM, sku *SLLMSku) bool { + devs := GetEffectiveDevices(llm, sku) + if devs == nil { + return false + } + for _, d := range *devs { + if strings.EqualFold(d.Vendor, "MTHREADS") { + return true + } + if d.DevType == computeapi.CONTAINER_DEV_MTHREADS_GPU || d.DevType == computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI { + return true + } + } + return false +} + // HasKunlunxinDevices reports whether effective devices include Kunlunxin XPU. func HasKunlunxinDevices(llm *SLLM, sku *SLLMSku) bool { devs := GetEffectiveDevices(llm, sku) diff --git a/pkg/llm/models/llm_devices_test.go b/pkg/llm/models/llm_devices_test.go index 133798ce46..7ee479c5c5 100644 --- a/pkg/llm/models/llm_devices_test.go +++ b/pkg/llm/models/llm_devices_test.go @@ -201,3 +201,60 @@ func TestHasKunlunxinDevices(t *testing.T) { t.Fatal("expected normalized GPU+KUNLUNXIN vendor sku to be detected as Kunlunxin") } } + +func TestHasMthreadsDevices(t *testing.T) { + mthreadsSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU}, + }, + }, + } + if !HasMthreadsDevices(nil, mthreadsSku) { + t.Fatal("expected Moore Threads GPU sku to be detected") + } + + hamiSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI}, + }, + }, + } + if !HasMthreadsDevices(nil, hamiSku) { + t.Fatal("expected Moore Threads HAMI sku to be detected") + } + + nvSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU}, + }, + }, + } + if HasMthreadsDevices(nil, nvSku) { + t.Fatal("expected NVIDIA sku not to be detected as Moore Threads") + } + + llm := &SLLM{ + SLLMBase: SLLMBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU}, + }, + }, + } + if !HasMthreadsDevices(llm, nvSku) { + t.Fatal("expected llm device override to win over sku") + } + + normalizedSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.GPU_TYPE, Vendor: "MTHREADS", Model: "MTT S5000"}, + }, + }, + } + if !HasMthreadsDevices(nil, normalizedSku) { + t.Fatal("expected normalized GPU+MTHREADS vendor sku to be detected as Moore Threads") + } +} diff --git a/pkg/llm/models/sku.go b/pkg/llm/models/sku.go index 58ca568714..f1b1ac6cef 100644 --- a/pkg/llm/models/sku.go +++ b/pkg/llm/models/sku.go @@ -86,14 +86,22 @@ func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred return input, nil } -// normalizeLLMSkuDevices maps NVIDIA_* / HYGON_* / ASCEND_* / ILUVATAR_* / THEAD_* / KUNLUNXIN_* DevTypes onto +// normalizeLLMSkuDevices maps NVIDIA_* / HYGON_* / ASCEND_* / ILUVATAR_* / THEAD_* / KUNLUNXIN_* / MTHREADS_* DevTypes onto // GPU|NPU + SharingMode, and defaults empty DevType/SharingMode appropriately. func normalizeLLMSkuDevices(devices *api.Devices) error { if devices == nil || len(*devices) == 0 { return nil } + mthreadsHami := 0 for i := range *devices { normalizeLLMSkuDevice(&(*devices)[i]) + dev := (*devices)[i] + if computeapi.IsMthreadsHami(dev.Vendor, dev.SharingMode) { + mthreadsHami++ + } + } + if mthreadsHami > 1 { + return httperrors.NewInputParameterError("Moore Threads sGPU does not support multiple cards in one task; use exclusive mode for multiple cards") } return nil } @@ -175,6 +183,16 @@ func normalizeLLMSkuDevice(dev *api.Device) { if dev.SharingMode == "" { dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE } + case computeapi.CONTAINER_DEV_MTHREADS_GPU: + dev.DevType = computeapi.GPU_TYPE + if dev.SharingMode == "" { + dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE + } + case computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI: + dev.DevType = computeapi.GPU_TYPE + if dev.SharingMode == "" { + dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI + } } if dev.SharingMode == "" { dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI @@ -194,6 +212,8 @@ func normalizeLLMSkuDevice(dev *api.Device) { dev.Vendor = "THEAD" case computeapi.CONTAINER_DEV_KUNLUNXIN_XPU: dev.Vendor = "KUNLUNXIN" + case computeapi.CONTAINER_DEV_MTHREADS_GPU, computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI: + dev.Vendor = "MTHREADS" } } dev.Vendor = canonicalizeLLMDeviceVendor(dev.Vendor) diff --git a/pkg/llm/models/sku_hami_test.go b/pkg/llm/models/sku_hami_test.go index 491b6703f3..077804dd5f 100644 --- a/pkg/llm/models/sku_hami_test.go +++ b/pkg/llm/models/sku_hami_test.go @@ -104,6 +104,20 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) { wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, wantVendor: "KUNLUNXIN", }, + { + name: "MTHREADS_GPU", + in: api.Device{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU}, + wantDevType: computeapi.GPU_TYPE, + wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, + wantVendor: "MTHREADS", + }, + { + name: "MTHREADS_GPU_HAMI", + in: api.Device{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI}, + wantDevType: computeapi.GPU_TYPE, + wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI, + wantVendor: "MTHREADS", + }, { name: "ILUVATAR vendor preserved and canonicalized", in: api.Device{DevType: computeapi.GPU_TYPE, Vendor: "iluvatar", Model: "BI-V150S"}, @@ -157,6 +171,30 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) { } } +func TestNormalizeLLMSkuDevicesRejectsMultipleMthreadsHami(t *testing.T) { + sliced := api.Devices{ + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI, Model: "MTT S5000"}, + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI, Model: "MTT S5000"}, + } + if err := normalizeLLMSkuDevices(&sliced); err == nil { + t.Fatal("expected multiple Moore Threads sGPU cards to be rejected") + } + exclusive := api.Devices{ + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU, Model: "MTT S5000"}, + {DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU, Model: "MTT S5000"}, + } + if err := normalizeLLMSkuDevices(&exclusive); err != nil { + t.Fatalf("exclusive multi-card should be allowed: %v", err) + } + nvidia := api.Devices{ + {DevType: computeapi.CONTAINER_DEV_NVIDIA_HAMI, Model: "A100"}, + {DevType: computeapi.CONTAINER_DEV_NVIDIA_HAMI, Model: "A100"}, + } + if err := normalizeLLMSkuDevices(&nvidia); err != nil { + t.Fatalf("NVIDIA HAMI multi-card should be allowed: %v", err) + } +} + func TestNormalizeLLMSkuDevicesAllowsHAMIWithoutStoredVram(t *testing.T) { devs := api.Devices{{Model: "A100"}} if err := normalizeLLMSkuDevices(&devs); err != nil { diff --git a/pkg/monitor/dbinit/measurements/all.go b/pkg/monitor/dbinit/measurements/all.go index b6b232a787..622c432f49 100644 --- a/pkg/monitor/dbinit/measurements/all.go +++ b/pkg/monitor/dbinit/measurements/all.go @@ -41,6 +41,7 @@ var All = []SMeasurement{ ixsmi, ppusmi, xpusmi, + mthreadsGmi, worker, serviceHttpCode, diff --git a/pkg/monitor/dbinit/measurements/metrics.csv b/pkg/monitor/dbinit/measurements/metrics.csv index 29f42a54ce..a72ee1eba0 100644 --- a/pkg/monitor/dbinit/measurements/metrics.csv +++ b/pkg/monitor/dbinit/measurements/metrics.csv @@ -165,14 +165,14 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "nvidia_smi","Nvidia GPU metrics","host","telegraf","memory_used","GPU memory used size","" "nvidia_smi","Nvidia GPU metrics","host","telegraf","utilization_gpu","GPU utilization","%" "nvidia_smi","Nvidia GPU metrics","host","telegraf","utilization_memory","GPU memory utilization","%" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","clocks_current_graphics","GPU current clocks, MHz","" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","temperature_gpu","GPU temperature","" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","memory_total","GPU memory total size","" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","memory_free","GPU memory free size","" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","memory_used","GPU memory used size","" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","utilization_gpu","GPU utilization","%" -"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","utilization_memory","GPU memory utilization","%" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","clocks_current_graphics","GPU current clocks, MHz","" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","temperature_gpu","GPU temperature","" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","memory_total","GPU memory total size","" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","memory_free","GPU memory free size","" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","memory_used","GPU memory used size","" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","utilization_gpu","GPU utilization","%" +"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","utilization_memory","GPU memory utilization","%" "processes","Processes status","host","telegraf","total","Total processes count","count" "processes","Processes status","host","telegraf","total_threads","Total threads count","count" "processes","Processes status","host","telegraf","zombies","Z state, zombie processes count","count" @@ -247,16 +247,26 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "vasmi","Vasmi GPU metrics","host","telegraf","eclk","eclk, MHz","" "vasmi","Vasmi GPU metrics","host","telegraf","gclk","gclk, MHz","" "vasmi","Vasmi GPU metrics","host","telegraf","aic_power","AIC power","" -"npu_smi","NPU metrics","host","telegraf","temperature","NPU temperature","" -"npu_smi","NPU metrics","host","telegraf","npu_real_time_power","NPU real time power","" -"npu_smi","NPU metrics","host","telegraf","npu_utilization","NPU utilization","%" -"npu_smi","NPU metrics","host","telegraf","hbm_usage_rate","NPU hbm usage rate","%" -"npu_smi","NPU metrics","host","telegraf","hbm_capacity","NPU hbm capacity","" -"npu_smi","NPU metrics","host","telegraf","aicore_usage_rate","NPU aicore usage rate","%" -"npu_smi","NPU metrics","host","telegraf","aivector_usage_rate","NPU aivector usage rate","%" -"npu_smi","NPU metrics","host","telegraf","aicube_usage_rate","NPU aicube usage rate","%" -"npu_smi","NPU metrics","host","telegraf","aicpu_usage_rate","NPU aicpu usage rate","%" -"npu_smi","NPU metrics","host","telegraf","hbm_bandwidth_usage_rate","NPU hbm bandwidth usage rate","" +"npu_smi","Ascend NPU metrics","host","telegraf","temperature","NPU temperature","" +"npu_smi","Ascend NPU metrics","host","telegraf","npu_real_time_power","NPU real time power","" +"npu_smi","Ascend NPU metrics","host","telegraf","npu_utilization","NPU utilization","%" +"npu_smi","Ascend NPU metrics","host","telegraf","hbm_usage_rate","NPU hbm usage rate","%" +"npu_smi","Ascend NPU metrics","host","telegraf","hbm_capacity","NPU hbm capacity","" +"npu_smi","Ascend NPU metrics","host","telegraf","aicore_usage_rate","NPU aicore usage rate","%" +"npu_smi","Ascend NPU metrics","host","telegraf","aivector_usage_rate","NPU aivector usage rate","%" +"npu_smi","Ascend NPU metrics","host","telegraf","aicube_usage_rate","NPU aicube usage rate","%" +"npu_smi","Ascend NPU metrics","host","telegraf","aicpu_usage_rate","NPU aicpu usage rate","%" +"npu_smi","Ascend NPU metrics","host","telegraf","hbm_bandwidth_usage_rate","NPU hbm bandwidth usage rate","" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","temperature","NPU temperature","" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","npu_real_time_power","NPU real time power","" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","npu_utilization","NPU utilization","%" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","hbm_usage_rate","NPU hbm usage rate","%" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","hbm_capacity","NPU hbm capacity","" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aicore_usage_rate","NPU aicore usage rate","%" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aivector_usage_rate","NPU aivector usage rate","%" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aicube_usage_rate","NPU aicube usage rate","%" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aicpu_usage_rate","NPU aicpu usage rate","%" +"agent_npu_smi","Ascend NPU metrics","agent","telegraf","hbm_bandwidth_usage_rate","NPU hbm bandwidth usage rate","" "hysmi","Hygon DCU metrics","host","telegraf","temperature_gpu","DCU temperature","" "hysmi","Hygon DCU metrics","host","telegraf","power_draw","DCU power draw","" "hysmi","Hygon DCU metrics","host","telegraf","power_cap","DCU power cap","" @@ -264,6 +274,31 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "hysmi","Hygon DCU metrics","host","telegraf","utilization_memory","DCU memory utilization","%" "hysmi","Hygon DCU metrics","host","telegraf","utilization_encoder","DCU encoder utilization","%" "hysmi","Hygon DCU metrics","host","telegraf","utilization_decoder","DCU decoder utilization","%" +"hysmi","Hygon DCU metrics","host","telegraf","memory_total","GPU memory total size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_free","GPU memory free size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_used","GPU memory used size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_gtt_total","GPU memory gtt size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_gtt_free","GPU memory gtt free size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_gtt_used","GPU memory gtt used size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_vis_vram_total","GPU memory vis vram size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_vis_vram_free","GPU memory vis vram free size","" +"hysmi","Hygon DCU metrics","host","telegraf","memory_vis_vram_used","GPU memory vis vram used size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","temperature_gpu","DCU temperature","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","power_draw","DCU power draw","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","power_cap","DCU power cap","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_gpu","DCU utilization","%" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_memory","DCU memory utilization","%" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_encoder","DCU encoder utilization","%" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_decoder","DCU decoder utilization","%" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_total","GPU memory total size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_free","GPU memory free size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_used","GPU memory used size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_gtt_total","GPU memory gtt size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_gtt_free","GPU memory gtt free size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_gtt_used","GPU memory gtt used size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_vis_vram_total","GPU memory vis vram size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_vis_vram_free","GPU memory vis vram free size","" +"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_vis_vram_used","GPU memory vis vram used size","" "ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_gpu","GPU temperature","" "ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_memory","GPU memory temperature","" "ixsmi","Iluvatar GPU metrics","host","telegraf","memory_total","GPU memory total size","" @@ -274,6 +309,16 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "ixsmi","Iluvatar GPU metrics","host","telegraf","power_draw","GPU power draw","" "ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_sm","GPU current SM clocks, MHz","" "ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","temperature_gpu","GPU temperature","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","temperature_memory","GPU memory temperature","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","memory_total","GPU memory total size","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","memory_free","GPU memory free size","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","memory_used","GPU memory used size","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","utilization_gpu","GPU utilization","%" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","utilization_memory","GPU memory utilization","%" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","power_draw","GPU power draw","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","clocks_current_sm","GPU current SM clocks, MHz","" +"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" "ppusmi","T-Head PPU metrics","host","telegraf","temperature_gpu","PPU temperature","" "ppusmi","T-Head PPU metrics","host","telegraf","temperature_memory","PPU memory temperature","" "ppusmi","T-Head PPU metrics","host","telegraf","memory_total","PPU memory total size","" @@ -284,6 +329,16 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "ppusmi","T-Head PPU metrics","host","telegraf","power_draw","PPU power draw","" "ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_sm","PPU current CU clocks, MHz","" "ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_memory","PPU current memory clocks, MHz","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","temperature_gpu","PPU temperature","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","temperature_memory","PPU memory temperature","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","memory_total","PPU memory total size","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","memory_free","PPU memory free size","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","memory_used","PPU memory used size","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","utilization_gpu","PPU utilization","%" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","utilization_memory","PPU memory utilization","%" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","power_draw","PPU power draw","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","clocks_current_sm","PPU current CU clocks, MHz","" +"agent_ppusmi","T-Head PPU metrics","agent","telegraf","clocks_current_memory","PPU current memory clocks, MHz","" "xpusmi","Kunlunxin XPU metrics","host","telegraf","temperature_gpu","XPU temperature","" "xpusmi","Kunlunxin XPU metrics","host","telegraf","memory_total","XPU memory total size","" "xpusmi","Kunlunxin XPU metrics","host","telegraf","memory_free","XPU memory free size","" @@ -302,6 +357,52 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "xpusmi","Kunlunxin XPU metrics","host","telegraf","ecc_errors_dram_uncorrectable","XPU uncorrectable DRAM ECC errors","" "xpusmi","Kunlunxin XPU metrics","host","telegraf","ecc_errors_dram_correctable_aggregate","XPU correctable DRAM ECC errors in total","" "xpusmi","Kunlunxin XPU metrics","host","telegraf","ecc_errors_dram_uncorrectable_aggregate","XPU uncorrectable DRAM ECC errors in total","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","temperature_gpu","XPU temperature","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","memory_total","XPU memory total size","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","memory_free","XPU memory free size","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","memory_used","XPU memory used size","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","l3_memory_total","XPU L3 memory total size","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","l3_memory_free","XPU L3 memory free size","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","l3_memory_used","XPU L3 memory used size","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","utilization_gpu","XPU utilization","%" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","power_draw","XPU power draw","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","power_limit","XPU power limit","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","clocks_current_cluster","XPU current cluster clocks, MHz","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","clocks_current_cdnn","XPU current CDNN clocks, MHz","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","pcie_link_gen_current","XPU current PCIe link generation","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","pcie_link_width_current","XPU current PCIe link width","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_correctable","XPU correctable DRAM ECC errors","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_uncorrectable","XPU uncorrectable DRAM ECC errors","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_correctable_aggregate","XPU correctable DRAM ECC errors in total","" +"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_uncorrectable_aggregate","XPU uncorrectable DRAM ECC errors in total","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","temperature_gpu","GPU temperature","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","memory_total","GPU memory total size","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","memory_free","GPU memory free size","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","memory_used","GPU memory used size","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_gpu","GPU utilization","%" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_memory","GPU memory utilization","%" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_encoder","GPU encoder utilization","%" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_decoder","GPU decoder utilization","%" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","power_draw","GPU power draw","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","power_limit","GPU power limit","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","clocks_current_graphics","GPU current clocks, MHz","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","pcie_link_gen_current","GPU PCIe link generation","" +"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","pcie_link_width_current","GPU PCIe link width","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","temperature_gpu","GPU temperature","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","memory_total","GPU memory total size","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","memory_free","GPU memory free size","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","memory_used","GPU memory used size","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_gpu","GPU utilization","%" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_memory","GPU memory utilization","%" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_encoder","GPU encoder utilization","%" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_decoder","GPU decoder utilization","%" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","power_draw","GPU power draw","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","power_limit","GPU power limit","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","clocks_current_graphics","GPU current clocks, MHz","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","pcie_link_gen_current","GPU PCIe link generation","" +"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","pcie_link_width_current","GPU PCIe link width","" "worker","Worker queue","system","system","active_worker_cnt","Active Worker Count","NULL" "worker","Worker queue","system","system","max_worker_count","Max Worker Count","NULL" "worker","Worker queue","system","system","detach_worker_cnt","Detach worker Count","NULL" @@ -613,6 +714,7 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "pod_cpu","Pod cpu","container","telegraf","usage_rate","Pod cpu usage rate","%" "pod_gpu","Pod gpu","container","telegraf","mem","Pod gpu mem used","Mb" "pod_gpu","Pod gpu","container","telegraf","mem_util","Pod gpu mem usage","%" +"pod_gpu","Pod gpu","container","telegraf","gpu_util","Pod gpu utilization","%" "pod_mem","Pod memory","container","telegraf","usage_rate","Pod memory usage rate","%" "pod_mem","Pod memory","container","telegraf","working_set_rate","Container memory working set usage rate","%" "pod_mem","Pod memory","container","telegraf","usage_bytes","Container memory usage bytes","byte" diff --git a/pkg/monitor/dbinit/measurements/mthreads_gmi.go b/pkg/monitor/dbinit/measurements/mthreads_gmi.go new file mode 100644 index 0000000000..71501b6469 --- /dev/null +++ b/pkg/monitor/dbinit/measurements/mthreads_gmi.go @@ -0,0 +1,65 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var mthreadsGmi = SMeasurement{ + Context: newHostAgentMonitorContext("mthreads_gmi", "Moore Threads GPU metrics"), + Metrics: []SMetric{ + { + "temperature_gpu", "GPU temperature", "", + }, + { + "memory_total", "GPU memory total size", "", + }, + { + "memory_free", "GPU memory free size", "", + }, + { + "memory_used", "GPU memory used size", "", + }, + { + "utilization_gpu", "GPU utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "utilization_memory", "GPU memory utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "utilization_encoder", "GPU encoder utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "utilization_decoder", "GPU decoder utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "power_draw", "GPU power draw", "", + }, + { + "power_limit", "GPU power limit", "", + }, + { + "clocks_current_graphics", "GPU current clocks, MHz", "", + }, + { + "clocks_current_memory", "GPU current memory clocks, MHz", "", + }, + { + "pcie_link_gen_current", "GPU PCIe link generation", "", + }, + { + "pcie_link_width_current", "GPU PCIe link width", "", + }, + }, +} diff --git a/pkg/monitor/dbinit/measurements/pod.go b/pkg/monitor/dbinit/measurements/pod.go index a6940be271..02d765201c 100644 --- a/pkg/monitor/dbinit/measurements/pod.go +++ b/pkg/monitor/dbinit/measurements/pod.go @@ -59,6 +59,11 @@ var podGpu = SMeasurement{ DisplayName: "Pod gpu mem usage", Unit: monitor.METRIC_UNIT_PERCENT, }, + { + Name: "gpu_util", + DisplayName: "Pod gpu utilization", + Unit: monitor.METRIC_UNIT_PERCENT, + }, }, }