feat(region,host,llm,monitor): support MooreThreads GPU (#25882)

2026-10-16 15:07:59 +08:00 · 2026-10-15 19:12:31 +08:00
parent acd776ffff
commit 73ecee3524
37 changed files with 1941 additions and 103 deletions
@@ -33,22 +33,24 @@ func init() {
 }

 const (
-	CONTAINER_DEV_CPH_AMD_GPU      = "CPH_AMD_GPU"
-	CONTAINER_DEV_CPH_AOSP_BINDER  = "CPH_AOSP_BINDER"
-	CONTAINER_DEV_NETINT_CA_ASIC   = "NETINT_CA_ASIC"
-	CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA"
-	CONTAINER_DEV_NVIDIA_GPU       = "NVIDIA_GPU"
-	CONTAINER_DEV_NVIDIA_MPS       = "NVIDIA_MPS"
-	CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE"
-	CONTAINER_DEV_NVIDIA_HAMI      = "NVIDIA_HAMI"
-	CONTAINER_DEV_ASCEND_NPU       = "ASCEND_NPU"
-	CONTAINER_DEV_ASCEND_NPU_HAMI  = "ASCEND_NPU_HAMI"
-	CONTAINER_DEV_VASTAITECH_GPU   = "VASTAITECH_GPU"
-	CONTAINER_DEV_HYGON_DCU        = "HYGON_DCU"
-	CONTAINER_DEV_HYGON_DCU_HAMI   = "HYGON_DCU_HAMI"
-	CONTAINER_DEV_ILUVATAR_GPU     = "ILUVATAR_GPU"
-	CONTAINER_DEV_THEAD_PPU        = "THEAD_PPU"
-	CONTAINER_DEV_KUNLUNXIN_XPU    = "KUNLUNXIN_XPU"
+	CONTAINER_DEV_CPH_AMD_GPU       = "CPH_AMD_GPU"
+	CONTAINER_DEV_CPH_AOSP_BINDER   = "CPH_AOSP_BINDER"
+	CONTAINER_DEV_NETINT_CA_ASIC    = "NETINT_CA_ASIC"
+	CONTAINER_DEV_NETINT_CA_QUADRA  = "NETINT_CA_QUADRA"
+	CONTAINER_DEV_NVIDIA_GPU        = "NVIDIA_GPU"
+	CONTAINER_DEV_NVIDIA_MPS        = "NVIDIA_MPS"
+	CONTAINER_DEV_NVIDIA_GPU_SHARE  = "NVIDIA_GPU_SHARE"
+	CONTAINER_DEV_NVIDIA_HAMI       = "NVIDIA_HAMI"
+	CONTAINER_DEV_ASCEND_NPU        = "ASCEND_NPU"
+	CONTAINER_DEV_ASCEND_NPU_HAMI   = "ASCEND_NPU_HAMI"
+	CONTAINER_DEV_VASTAITECH_GPU    = "VASTAITECH_GPU"
+	CONTAINER_DEV_HYGON_DCU         = "HYGON_DCU"
+	CONTAINER_DEV_HYGON_DCU_HAMI    = "HYGON_DCU_HAMI"
+	CONTAINER_DEV_ILUVATAR_GPU      = "ILUVATAR_GPU"
+	CONTAINER_DEV_THEAD_PPU         = "THEAD_PPU"
+	CONTAINER_DEV_KUNLUNXIN_XPU     = "KUNLUNXIN_XPU"
+	CONTAINER_DEV_MTHREADS_GPU      = "MTHREADS_GPU"
+	CONTAINER_DEV_MTHREADS_GPU_HAMI = "MTHREADS_GPU_HAMI"
 )

 var (
@@ -64,6 +66,8 @@ var (
 		CONTAINER_DEV_ILUVATAR_GPU,
 		CONTAINER_DEV_THEAD_PPU,
 		CONTAINER_DEV_KUNLUNXIN_XPU,
+		CONTAINER_DEV_MTHREADS_GPU,
+		CONTAINER_DEV_MTHREADS_GPU_HAMI,
 	}
 )

@@ -53,10 +53,11 @@ const (
 	ILUVATAR_VENDOR_ID   = "1e3e"
 	THEAD_VENDOR_ID      = "1ded"
 	KUNLUNXIN_VENDOR_ID  = "1d22"
+	MTHREADS_VENDOR_ID   = "1ed5"
 )

 var HeterogeneousVendors = []string{
-	VASTAITECH_VENDOR_ID, HYGON_VENDOR_ID, ASCEND_VENDOR_ID, ILUVATAR_VENDOR_ID, THEAD_VENDOR_ID, KUNLUNXIN_VENDOR_ID,
+	VASTAITECH_VENDOR_ID, HYGON_VENDOR_ID, ASCEND_VENDOR_ID, ILUVATAR_VENDOR_ID, THEAD_VENDOR_ID, KUNLUNXIN_VENDOR_ID, MTHREADS_VENDOR_ID,
 }

 var ID_VENDOR_MAP = map[string]string{
@@ -68,6 +69,7 @@ var ID_VENDOR_MAP = map[string]string{
 	ILUVATAR_VENDOR_ID:   "ILUVATAR",
 	THEAD_VENDOR_ID:      "THEAD",
 	KUNLUNXIN_VENDOR_ID:  "KUNLUNXIN",
+	MTHREADS_VENDOR_ID:   "MTHREADS",
 }

 var VENDOR_ID_MAP = map[string]string{
@@ -79,6 +81,7 @@ var VENDOR_ID_MAP = map[string]string{
 	"ILUVATAR":   ILUVATAR_VENDOR_ID,
 	"THEAD":      THEAD_VENDOR_ID,
 	"KUNLUNXIN":  KUNLUNXIN_VENDOR_ID,
+	"MTHREADS":   MTHREADS_VENDOR_ID,
 }

 const (
@@ -107,6 +110,8 @@ var GPU_TYPES = []string{
 	CONTAINER_DEV_ILUVATAR_GPU,
 	CONTAINER_DEV_THEAD_PPU,
 	CONTAINER_DEV_KUNLUNXIN_XPU,
+	CONTAINER_DEV_MTHREADS_GPU,
+	CONTAINER_DEV_MTHREADS_GPU_HAMI,
 }

 var NETINT_TYPES = []string{
@@ -121,6 +126,8 @@ var VALID_CONTAINER_DEVICE_TYPES = []string{
 	CONTAINER_DEV_ILUVATAR_GPU,
 	CONTAINER_DEV_THEAD_PPU,
 	CONTAINER_DEV_KUNLUNXIN_XPU,
+	CONTAINER_DEV_MTHREADS_GPU,
+	CONTAINER_DEV_MTHREADS_GPU_HAMI,
 }

 var VALID_PASSTHROUGH_TYPES = []string{
@@ -144,6 +151,8 @@ var VITRUAL_DEVICE_TYPES = []string{
 	CONTAINER_DEV_ILUVATAR_GPU,
 	CONTAINER_DEV_THEAD_PPU,
 	CONTAINER_DEV_KUNLUNXIN_XPU,
+	CONTAINER_DEV_MTHREADS_GPU,
+	CONTAINER_DEV_MTHREADS_GPU_HAMI,
 }

 func init() {
@@ -0,0 +1,75 @@
+package compute
+
+import "strings"
+
+// sGPU memory buckets in MiB. The driver rounds a requested limit up to one of these.
+var mthreadsSgpuMemoryBucketsMiB = []int{
+	512,
+	1 * 1024,
+	2 * 1024,
+	4 * 1024,
+	8 * 1024,
+	16 * 1024,
+	32 * 1024,
+	48 * 1024,
+	64 * 1024,
+	80 * 1024,
+}
+
+const (
+	MthreadsSgpuMaxInstances       = 16
+	MthreadsSgpuWeightMin          = 1
+	MthreadsSgpuWeightMax          = 99999
+	mthreadsSgpuBytesPerMiB  int64 = 1024 * 1024
+)
+
+// AlignMthreadsSgpuMemoryMiB rounds a requested size up to the next sGPU bucket.
+// Zero stays zero. A request above the largest bucket is left unchanged.
+func AlignMthreadsSgpuMemoryMiB(request int) int {
+	if request <= 0 {
+		return 0
+	}
+	for _, bucket := range mthreadsSgpuMemoryBucketsMiB {
+		if request <= bucket {
+			return bucket
+		}
+	}
+	return request
+}
+
+// MthreadsSgpuMemoryLimitBytes is the byte value for MTHREADS_QOS_MEMORY_LIMIT.
+func MthreadsSgpuMemoryLimitBytes(memoryMiB int) int64 {
+	return int64(AlignMthreadsSgpuMemoryMiB(memoryMiB)) * mthreadsSgpuBytesPerMiB
+}
+
+// IsMthreadsHami reports a sliced Moore Threads GPU. Vendor may be the name or the PCI id.
+func IsMthreadsHami(vendor, sharingMode string) bool {
+	if sharingMode != DEVICE_SHARING_MODE_HAMI {
+		return false
+	}
+	vendor = strings.TrimSpace(vendor)
+	return strings.EqualFold(vendor, "MTHREADS") || strings.EqualFold(vendor, MTHREADS_VENDOR_ID)
+}
+
+// CountMthreadsHamiDevices counts sGPU slices. More than one in a guest is rejected.
+func CountMthreadsHamiDevices(devs []*IsolatedDeviceConfig) int {
+	n := 0
+	for _, dev := range devs {
+		if dev != nil && IsMthreadsHami(dev.Vendor, dev.SharingMode) {
+			n++
+		}
+	}
+	return n
+}
+
+// MthreadsSgpuComputingPowerWeight maps an SM percent to the sGPU time-slice weight.
+// Zero uses the driver default of 1.
+func MthreadsSgpuComputingPowerWeight(smUtilLimit int) int {
+	if smUtilLimit < MthreadsSgpuWeightMin {
+		return MthreadsSgpuWeightMin
+	}
+	if smUtilLimit > MthreadsSgpuWeightMax {
+		return MthreadsSgpuWeightMax
+	}
+	return smUtilLimit
+}
@@ -0,0 +1,57 @@
+package compute
+
+import "testing"
+
+func TestAlignMthreadsSgpuMemoryMiB(t *testing.T) {
+	cases := []struct {
+		in   int
+		want int
+	}{
+		{0, 0},
+		{-1, 0},
+		{500, 512},
+		{512, 512},
+		{1000, 1024},
+		{40960, 48 * 1024},
+		{81920, 80 * 1024},
+		{90000, 90000},
+	}
+	for _, tc := range cases {
+		if got := AlignMthreadsSgpuMemoryMiB(tc.in); got != tc.want {
+			t.Fatalf("AlignMthreadsSgpuMemoryMiB(%d)=%d, want %d", tc.in, got, tc.want)
+		}
+	}
+}
+
+func TestMthreadsSgpuMemoryLimitBytes(t *testing.T) {
+	if got := MthreadsSgpuMemoryLimitBytes(2048); got != 2147483648 {
+		t.Fatalf("bytes=%d, want 2147483648", got)
+	}
+	if got := MthreadsSgpuMemoryLimitBytes(0); got != 0 {
+		t.Fatalf("zero bytes=%d", got)
+	}
+}
+
+func TestCountMthreadsHamiDevices(t *testing.T) {
+	devs := []*IsolatedDeviceConfig{
+		{Vendor: "MTHREADS", SharingMode: DEVICE_SHARING_MODE_HAMI},
+		{Vendor: MTHREADS_VENDOR_ID, SharingMode: DEVICE_SHARING_MODE_HAMI},
+		{Vendor: "MTHREADS", SharingMode: DEVICE_SHARING_MODE_EXCLUSIVE},
+		{Vendor: "NVIDIA", SharingMode: DEVICE_SHARING_MODE_HAMI},
+	}
+	if got := CountMthreadsHamiDevices(devs); got != 2 {
+		t.Fatalf("count=%d, want 2", got)
+	}
+}
+
+func TestMthreadsSgpuComputingPowerWeight(t *testing.T) {
+	if got := MthreadsSgpuComputingPowerWeight(0); got != 1 {
+		t.Fatalf("weight=%d, want 1", got)
+	}
+	if got := MthreadsSgpuComputingPowerWeight(25); got != 25 {
+		t.Fatalf("weight=%d, want 25", got)
+	}
+	if got := MthreadsSgpuComputingPowerWeight(100000); got != 99999 {
+		t.Fatalf("weight=%d, want 99999", got)
+	}
+}
@@ -809,6 +809,10 @@ func getIsolatedDevicesInfo(cli *ssh.Client, ip net.IP) ([]*isolated_device.PCID
 					log.Infof("skip kunlunxin dev %s with class code %s", dev.Addr, dev.ClassCode)
 					continue
 				}
+				if dev.VendorId == api.MTHREADS_VENDOR_ID && dev.ClassCode != isolated_device.CLASS_CODE_PA {
+					log.Infof("skip mthreads dev %s with class code %s", dev.Addr, dev.ClassCode)
+					continue
+				}
 				log.Infof("append dev %s", dev.Addr)
 				devs = append(devs, dev)
 			}
@@ -2730,6 +2730,32 @@ func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient
 			}
 		}
 	}
+	if sharingMode == api.DEVICE_SHARING_MODE_HAMI {
+		existing := 0
+		for i := range attachedGpus {
+			gdev := attachedGpus[i].GetIsolatedDevice()
+			if gdev != nil && gdev.isMthreads() && gdev.SharingMode == api.DEVICE_SHARING_MODE_HAMI {
+				existing++
+			}
+		}
+		adding := 0
+		for _, count := range devModelCount {
+			adding += count
+		}
+		if existing > 0 || adding > 1 {
+			for devModel := range devModelCount {
+				devs, err := IsolatedDeviceManager.GetAvailableIsolatedDeviceOnHost(host.Id, devModel, sharingMode)
+				if err != nil {
+					return httperrors.NewInternalServerError("fetch gpu failed %s", err)
+				}
+				for i := range devs {
+					if devs[i].isMthreads() && existing+adding > 1 {
+						return httperrors.NewBadRequestError("Moore Threads sGPU does not support multiple cards in one guest; use exclusive mode for multiple cards")
+					}
+				}
+			}
+		}
+	}

 	for devModel, count := range devModelCount {
 		devs, err := IsolatedDeviceManager.GetAvailableIsolatedDeviceOnHost(host.Id, devModel, sharingMode)
@@ -2849,6 +2875,14 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred
 			}
 		}
 	}
+	if dev.isMthreads() && dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI {
+		for i := range gdevs {
+			gdev := gdevs[i].GetIsolatedDevice()
+			if gdev != nil && gdev.isMthreads() && gdev.SharingMode == api.DEVICE_SHARING_MODE_HAMI {
+				return httperrors.NewBadRequestError("Moore Threads sGPU does not support multiple cards in one guest; use exclusive mode for multiple cards")
+			}
+		}
+	}

 	host, _ := self.GetHost()
 	lockman.LockObject(ctx, host)
@@ -2888,8 +2922,9 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.
 		guestIsolatedDevice.DiskIndex = *diskIndex
 	}
 	if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI && memoryRequest != nil {
-		guestIsolatedDevice.DeviceMemorySize = *memoryRequest
-		guestIsolatedDevice.SmUtilLimit = int((float64(*memoryRequest) / float64(dev.MemorySize)) * 100.0)
+		req := dev.hamiMemoryRequest(*memoryRequest)
+		guestIsolatedDevice.DeviceMemorySize = req
+		guestIsolatedDevice.SmUtilLimit = int((float64(req) / float64(dev.MemorySize)) * 100.0)
 	}
 	if utils.IsInStringArray(gpuType, []string{api.GPU_HPC, api.GPU_VGA}) {
 		guestIsolatedDevice.GpuType = gpuType
@@ -2381,6 +2381,9 @@ func (manager *SGuestManager) validateCreateData(
 	if nvidiaVgpuCnt > 1 {
 		return nil, httperrors.NewBadRequestError("Nvidia vGPU count cannot exceed 1")
 	}
+	if api.CountMthreadsHamiDevices(input.IsolatedDevices) > 1 {
+		return nil, httperrors.NewBadRequestError("Moore Threads sGPU does not support multiple cards in one guest; use exclusive mode for multiple cards")
+	}
 	if nvidiaVgpuCnt > 0 && gpuCnt > 0 {
 		return nil, httperrors.NewBadRequestError("Nvidia vGPU cannot passthrough with other gpus")
 	}
@@ -2020,6 +2020,29 @@ func (manager *SIsolatedDeviceManager) GetHostsGuestIsolatedDevices(hostIds []st
 	return ret
 }

+func (dev *SIsolatedDevice) isMthreads() bool {
+	return strings.EqualFold(dev.getVendorId(), api.MTHREADS_VENDOR_ID)
+}
+
+func (dev *SIsolatedDevice) hamiMemoryRequest(memoryRequest int) int {
+	if dev.isMthreads() {
+		return api.AlignMthreadsSgpuMemoryMiB(memoryRequest)
+	}
+	return memoryRequest
+}
+
+func (dev *SIsolatedDevice) mthreadsSgpuInstanceFull() bool {
+	if !dev.isMthreads() {
+		return false
+	}
+	cnt, err := dev.getAllocatedCount()
+	if err != nil {
+		log.Errorf("failed getAllocatedCount %s", err)
+		return true
+	}
+	return cnt >= api.MthreadsSgpuMaxInstances
+}
+
 func (dev *SIsolatedDevice) IsEnough(memoryRequest int) bool {
 	if dev.SharingMode != api.DEVICE_SHARING_MODE_HAMI {
 		cnt, err := dev.getAllocatedCount()
@@ -2028,14 +2051,17 @@ func (dev *SIsolatedDevice) IsEnough(memoryRequest int) bool {
 			return false
 		}
 		return dev.VirtualNum > cnt
-	} else {
-		allocated, err := dev.getAllocatedMemorySize()
-		if err != nil {
-			log.Errorf("failed getAllocatedMemorySize %s", err)
-			return false
-		}
-		return (dev.MemorySize - allocated) >= memoryRequest
 	}
+	if dev.mthreadsSgpuInstanceFull() {
+		return false
+	}
+	memoryRequest = dev.hamiMemoryRequest(memoryRequest)
+	allocated, err := dev.getAllocatedMemorySize()
+	if err != nil {
+		log.Errorf("failed getAllocatedMemorySize %s", err)
+		return false
+	}
+	return (dev.MemorySize - allocated) >= memoryRequest
 }

 func (dev *SIsolatedDevice) IsFull() bool {
@@ -2046,14 +2072,16 @@ func (dev *SIsolatedDevice) IsFull() bool {
 			return true
 		}
 		return dev.VirtualNum <= cnt
-	} else {
-		allocated, err := dev.getAllocatedMemorySize()
-		if err != nil {
-			log.Errorf("failed getAllocatedMemorySize %s", err)
-			return true
-		}
-		return dev.MemorySize <= allocated
 	}
+	if dev.mthreadsSgpuInstanceFull() {
+		return true
+	}
+	allocated, err := dev.getAllocatedMemorySize()
+	if err != nil {
+		log.Errorf("failed getAllocatedMemorySize %s", err)
+		return true
+	}
+	return dev.MemorySize <= allocated
 }

 func (manager *SIsolatedDeviceManager) InitializeData() error {
@@ -130,7 +130,7 @@ func PushContainerdImage(input *hostapi.ContainerPushImageInput) error {
 		// try http protocol
 		errs = append(errs, errors.Wrap(err, "pushImageByCtrCmd: %s"))
 		opt.PlainHttp = true
-		log.Infof("try push image %s by http", input.Image)
+		log.Warningf("push image to %s error: %v, try to push it by http", input.Image, err)
 		if err := imgTool.Push(input.Image, opt); err != nil {
 			errs = append(errs, errors.Wrapf(err, "pushImageByCtrCmd by http"))
 			return errors.NewAggregate(errs)
@@ -17,6 +17,7 @@ package hostinfo
 import (
 	"context"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -184,3 +185,88 @@ func (h *SHostInfo) HasContainerNvidiaGpu() bool {
 	h.containerNvidiaGpus = nvDevs
 	return *h.hasNvidiaGpus
 }
+
+func (h *SHostInfo) HasContainerMthreadsGpu() bool {
+	if h.hasMthreadsGpus != nil {
+		return *h.hasMthreadsGpus
+	}
+	hasMthreadsGpus := false
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		vendorId := strings.ToLower(strings.Split(devs[i].GetVendorDeviceId(), ":")[0])
+		if vendorId == apis.MTHREADS_VENDOR_ID {
+			hasMthreadsGpus = true
+			break
+		}
+	}
+	h.hasMthreadsGpus = &hasMthreadsGpus
+	return *h.hasMthreadsGpus
+}
+
+func (h *SHostInfo) GetMthreadsGpuIndexes(ids, addrs []string) []int {
+	if len(ids) == 0 && len(addrs) == 0 {
+		return nil
+	}
+	wantId := make(map[string]struct{}, len(ids))
+	for _, id := range ids {
+		if id != "" {
+			wantId[id] = struct{}{}
+		}
+	}
+	wantAddr := make([]string, 0, len(addrs))
+	for _, addr := range addrs {
+		if addr != "" {
+			wantAddr = append(wantAddr, addr)
+		}
+	}
+	seen := map[int]struct{}{}
+	out := make([]int, 0)
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		dev := devs[i]
+		vendorId := strings.ToLower(strings.Split(dev.GetVendorDeviceId(), ":")[0])
+		if vendorId != apis.MTHREADS_VENDOR_ID {
+			continue
+		}
+		_, idMatch := wantId[dev.GetCloudId()]
+		addrMatch := false
+		if !idMatch {
+			for _, addr := range wantAddr {
+				if samePCIAddr(dev.GetAddr(), addr) {
+					addrMatch = true
+					break
+				}
+			}
+		}
+		if !idMatch && !addrMatch {
+			continue
+		}
+		idx := dev.GetIndex()
+		if idx < 0 {
+			n, err := strconv.Atoi(dev.GetDevicePath())
+			if err != nil {
+				continue
+			}
+			idx = n
+		}
+		if _, ok := seen[idx]; ok {
+			continue
+		}
+		seen[idx] = struct{}{}
+		out = append(out, idx)
+	}
+	sort.Ints(out)
+	return out
+}
+
+func samePCIAddr(a, b string) bool {
+	a = strings.ToLower(strings.TrimSpace(a))
+	b = strings.ToLower(strings.TrimSpace(b))
+	if a == "" || b == "" {
+		return false
+	}
+	if a == b {
+		return true
+	}
+	return strings.HasSuffix(a, ":"+b) || strings.HasSuffix(b, ":"+a)
+}
@@ -135,6 +135,7 @@ type SHostInfo struct {
 	hasNvidiaGpus                  *bool
 	hasVastaitechGpus              *bool
 	hasCphAmdGpus                  *bool
+	hasMthreadsGpus                *bool

 	guestManager hostutils.IGuestManager
 }
@@ -2357,24 +2358,26 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		options.HostOptions.HygonDtkPath,
 	)
 	probeOpts := &isolated_device.SIsolatedDeviceProbeOptions{
-		SkipGPUs:                     options.HostOptions.DisableGPU,
-		SkipUSBs:                     options.HostOptions.DisableUSB,
-		SkipCustomDevs:               options.HostOptions.DisableCustomDevice,
-		EnableCudaHAMI:               options.HostOptions.EnableCudaHAMI,
-		EnableCudaMps:                options.HostOptions.EnableCudaMPS,
-		EnableContainerAscendNpu:     options.HostOptions.EnableContainerAscendNPU,
-		EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami,
-		EnableContainerHygonDCU:      options.HostOptions.EnableContainerHygonDCU,
-		EnableContainerHygonDCUHAMI:  options.HostOptions.EnableContainerHygonDCUHami,
-		EnableContainerIluvatarGPU:   options.HostOptions.EnableContainerIluvatarGPU,
-		EnableContainerTHeadPPU:      options.HostOptions.EnableContainerTHeadPPU,
-		EnableContainerKunlunxinXPU:  options.HostOptions.EnableContainerKunlunxinXPU,
-		EnableWhitelist:              options.HostOptions.EnableIsolatedDeviceWhitelist,
-		SriovNics:                    sriovNics,
-		OvsOffloadNics:               offloadNics,
-		NvmePciDisks:                 options.HostOptions.PTNVMEConfigs,
-		AmdVgpuPFs:                   options.HostOptions.AMDVgpuPFs,
-		NvidiaVgpuPFs:                options.HostOptions.NVIDIAVgpuPFs,
+		SkipGPUs:                       options.HostOptions.DisableGPU,
+		SkipUSBs:                       options.HostOptions.DisableUSB,
+		SkipCustomDevs:                 options.HostOptions.DisableCustomDevice,
+		EnableCudaHAMI:                 options.HostOptions.EnableCudaHAMI,
+		EnableCudaMps:                  options.HostOptions.EnableCudaMPS,
+		EnableContainerAscendNpu:       options.HostOptions.EnableContainerAscendNPU,
+		EnableContainerAscendNpuHAMI:   options.HostOptions.EnableContainerAscendNPUHami,
+		EnableContainerHygonDCU:        options.HostOptions.EnableContainerHygonDCU,
+		EnableContainerHygonDCUHAMI:    options.HostOptions.EnableContainerHygonDCUHami,
+		EnableContainerIluvatarGPU:     options.HostOptions.EnableContainerIluvatarGPU,
+		EnableContainerTHeadPPU:        options.HostOptions.EnableContainerTHeadPPU,
+		EnableContainerKunlunxinXPU:    options.HostOptions.EnableContainerKunlunxinXPU,
+		EnableContainerMthreadsGPU:     options.HostOptions.EnableContainerMthreadsGPU,
+		EnableContainerMthreadsGPUHami: options.HostOptions.EnableContainerMthreadsGPUHami,
+		EnableWhitelist:                options.HostOptions.EnableIsolatedDeviceWhitelist,
+		SriovNics:                      sriovNics,
+		OvsOffloadNics:                 offloadNics,
+		NvmePciDisks:                   options.HostOptions.PTNVMEConfigs,
+		AmdVgpuPFs:                     options.HostOptions.AMDVgpuPFs,
+		NvidiaVgpuPFs:                  options.HostOptions.NVIDIAVgpuPFs,
 	}
 	h.IsolatedDeviceMan.ProbePCIDevices(probeOpts)

@@ -2717,6 +2720,7 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 	hasIluvatar := false
 	hasTHead := false
 	hasKunlunxin := false
+	hasMthreads := false
 	hasNvidiasmi := false
 	hasNpusmi := false
 	for _, dev := range devs {
@@ -2733,6 +2737,9 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 		if vendorId == api.KUNLUNXIN_VENDOR_ID {
 			hasKunlunxin = true
 		}
+		if vendorId == api.MTHREADS_VENDOR_ID {
+			hasMthreads = true
+		}
 		if !utils.IsInStringArray(dev.GetSharingMode(), api.VIRTUAL_SHARING_MODES) {
 			continue
 		}
@@ -2813,6 +2820,15 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 			system_service.TELEGRAF_INPUT_CONF_LIB_PATH: libPath,
 		}
 	}
+	if hasMthreads {
+		smiPath := options.HostOptions.MthreadsGmiPath
+		if smiPath == "" {
+			smiPath = "/usr/bin/mthreads-gmi"
+		}
+		conf[system_service.TELEGRAF_INPUT_MTHREADSGMI] = map[string]interface{}{
+			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: smiPath,
+		}
+	}
 	if hasKunlunxin {
 		xreHome := options.HostOptions.KunlunxinXreHome
 		if xreHome == "" {
@@ -79,6 +79,10 @@ const (
 	CPH_AMD_GPU_DEV_ID   = "dev_id"
 	CPH_AMD_GPU_MEM      = "mem"
 	CPH_AMD_GPU_MEM_UTIL = "mem_util"
+
+	MTHREADS_GPU_MEM      = "mem"
+	MTHREADS_GPU_MEM_UTIL = "mem_util"
+	MTHREADS_GPU_GPU_UTIL = "gpu_util"
 )

 type CadvisorProcessMetric struct {
@@ -113,6 +117,7 @@ type PodMetrics struct {
 	PodNvidiaGpu     []*PodNvidiaGpuMetrics     `json:"pod_nvidia_gpu"`
 	PodVastaitechGpu []*PodVastaitechGpuMetrics `json:"pod_vastaitech_gpu"`
 	PodCphAmdGpu     []*PodCphAmdGpuMetrics     `json:"pod_cph_amd_gpu"`
+	PodMthreadsGpu   []*PodMthreadsGpuMetrics   `json:"pod_mthreads_gpu"`
 	Containers       []*ContainerMetrics        `json:"containers"`
 }

@@ -821,6 +826,7 @@ func (m *SGuestMonitor) PodMetrics(prevUsage *GuestMetrics) *PodMetrics {
 		PodNvidiaGpu:     m.getPodNvidiaGpuMetrics(),
 		PodVastaitechGpu: m.getPodVastaitechGpuMetrics(),
 		PodCphAmdGpu:     m.getPodCphAmdGpuMetrics(),
+		PodMthreadsGpu:   m.getPodMthreadsGpuMetrics(),
 		Containers:       containers,
 	}

@@ -1031,6 +1037,9 @@ func (d *GuestMetrics) toPodTelegrafData(tagStr string) []string {
 	for i := range m.PodCphAmdGpu {
 		ims = append(ims, m.PodCphAmdGpu[i])
 	}
+	for i := range m.PodMthreadsGpu {
+		ims = append(ims, m.PodMthreadsGpu[i])
+	}

 	for _, c := range m.Containers {
 		ims = append(ims, c.ContainerCpu)
@@ -0,0 +1,274 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type MthreadsGpuProc struct {
+	Index   int
+	Pid     string
+	MemMiB  float64
+	MemUtil float64 // fraction, 0 to 1
+	GpuUtil float64 // percent, 0 to 100
+}
+
+func mthreadsGmiBin() string {
+	if options.HostOptions.MthreadsGmiPath != "" {
+		return options.HostOptions.MthreadsGmiPath
+	}
+	return "/usr/bin/mthreads-gmi"
+}
+
+func GetMthreadsGpuProcessMetrics() ([]MthreadsGpuProc, error) {
+	bin := mthreadsGmiBin()
+	pmOut, pmErr := procutils.NewRemoteCommandAsFarAsPossible(bin, "-pm").Output()
+	memOut, memErr := procutils.NewRemoteCommandAsFarAsPossible(bin).Output()
+	if pmErr != nil && memErr != nil {
+		return nil, errors.Wrapf(pmErr, "mthreads-gmi -pm (%s); default: %v (%s)", pmOut, memErr, memOut)
+	}
+	if pmErr != nil {
+		log.Errorf("mthreads-gmi -pm: %s %s", pmErr, pmOut)
+		pmOut = nil
+	}
+	if memErr != nil {
+		log.Errorf("mthreads-gmi: %s %s", memErr, memOut)
+		memOut = nil
+	}
+	return mergeMthreadsGpuProcs(parseMthreadsProcessMonitor(string(pmOut)), parseMthreadsProcessMemory(string(memOut))), nil
+}
+
+func parseMthreadsProcessMonitor(output string) []MthreadsGpuProc {
+	var out []MthreadsGpuProc
+	for _, line := range strings.Split(output, "\n") {
+		fields := strings.Fields(strings.TrimSpace(line))
+		if len(fields) < 4 {
+			continue
+		}
+		idx, err := strconv.Atoi(fields[0])
+		if err != nil {
+			continue
+		}
+		pid := fields[1]
+		if _, err := strconv.Atoi(pid); err != nil {
+			continue
+		}
+		out = append(out, MthreadsGpuProc{
+			Index:   idx,
+			Pid:     pid,
+			MemUtil: parseMthreadsMemPercent(fields[2]),
+			GpuUtil: parseMthreadsGpuPercent(fields[3]),
+		})
+	}
+	return out
+}
+
+func parseMthreadsProcessMemory(output string) []MthreadsGpuProc {
+	var out []MthreadsGpuProc
+	for _, line := range strings.Split(output, "\n") {
+		fields := strings.Fields(strings.TrimSpace(line))
+		if len(fields) < 3 {
+			continue
+		}
+		memField := fields[len(fields)-1]
+		if !strings.HasSuffix(memField, "MiB") {
+			continue
+		}
+		idx, err := strconv.Atoi(fields[0])
+		if err != nil {
+			continue
+		}
+		pid := fields[1]
+		if _, err := strconv.Atoi(pid); err != nil {
+			continue
+		}
+		mem, err := strconv.ParseFloat(strings.TrimSuffix(memField, "MiB"), 64)
+		if err != nil {
+			continue
+		}
+		out = append(out, MthreadsGpuProc{
+			Index:  idx,
+			Pid:    pid,
+			MemMiB: mem,
+		})
+	}
+	return out
+}
+
+func parseMthreadsMemPercent(raw string) float64 {
+	return parseMthreadsPercent(raw) / 100
+}
+
+func parseMthreadsGpuPercent(raw string) float64 {
+	return parseMthreadsPercent(raw)
+}
+
+func parseMthreadsPercent(raw string) float64 {
+	raw = strings.TrimSpace(raw)
+	if raw == "" || strings.HasPrefix(raw, "<") {
+		return 0
+	}
+	raw = strings.TrimSuffix(raw, "%")
+	v, err := strconv.ParseFloat(raw, 64)
+	if err != nil {
+		return 0
+	}
+	return v
+}
+
+func mergeMthreadsGpuProcs(utils, mems []MthreadsGpuProc) []MthreadsGpuProc {
+	type key struct {
+		index int
+		pid   string
+	}
+	merged := map[key]*MthreadsGpuProc{}
+	order := make([]key, 0)
+	touch := func(index int, pid string) *MthreadsGpuProc {
+		k := key{index, pid}
+		if p, ok := merged[k]; ok {
+			return p
+		}
+		p := &MthreadsGpuProc{Index: index, Pid: pid}
+		merged[k] = p
+		order = append(order, k)
+		return p
+	}
+	for _, p := range utils {
+		dst := touch(p.Index, p.Pid)
+		dst.MemUtil = p.MemUtil
+		dst.GpuUtil = p.GpuUtil
+	}
+	for _, p := range mems {
+		dst := touch(p.Index, p.Pid)
+		dst.MemMiB = p.MemMiB
+	}
+	out := make([]MthreadsGpuProc, 0, len(order))
+	for _, k := range order {
+		out = append(out, *merged[k])
+	}
+	return out
+}
+
+func selectMthreadsGpuProcs(procs []MthreadsGpuProc, podProcs map[string]struct{}, indexes []int) []MthreadsGpuProc {
+	if len(procs) == 0 || len(podProcs) == 0 || len(indexes) == 0 {
+		return nil
+	}
+	want := make(map[int]struct{}, len(indexes))
+	for _, idx := range indexes {
+		want[idx] = struct{}{}
+	}
+	out := make([]MthreadsGpuProc, 0)
+	for _, p := range procs {
+		if _, ok := podProcs[p.Pid]; !ok {
+			continue
+		}
+		if _, ok := want[p.Index]; !ok {
+			continue
+		}
+		out = append(out, p)
+	}
+	return out
+}
+
+func (m *SGuestMonitor) getPodMthreadsGpuMetrics() []*PodMthreadsGpuMetrics {
+	if len(m.mthreadsGpuMetrics) == 0 {
+		return nil
+	}
+	type acc struct {
+		mem     float64
+		memUtil float64
+		gpuUtil float64
+	}
+	by := map[int]*acc{}
+	for _, p := range m.mthreadsGpuMetrics {
+		a := by[p.Index]
+		if a == nil {
+			a = &acc{}
+			by[p.Index] = a
+		}
+		a.mem += p.MemMiB
+		a.memUtil += p.MemUtil
+		if p.GpuUtil > a.gpuUtil {
+			a.gpuUtil = p.GpuUtil
+		}
+	}
+	indexes := make([]int, 0, len(by))
+	for idx := range by {
+		indexes = append(indexes, idx)
+	}
+	sort.Ints(indexes)
+	res := make([]*PodMthreadsGpuMetrics, 0, len(indexes))
+	for _, idx := range indexes {
+		a := by[idx]
+		memUtil := a.memUtil
+		if memUtil > 1 {
+			memUtil = 1
+		}
+		gpuUtil := a.gpuUtil
+		if gpuUtil > 100 {
+			gpuUtil = 100
+		}
+		res = append(res, &PodMthreadsGpuMetrics{
+			Index:   idx,
+			Mem:     a.mem,
+			MemUtil: memUtil,
+			GpuUtil: gpuUtil,
+		})
+	}
+	return res
+}
+
+type PodMthreadsGpuMetrics struct {
+	PodMetricMeta
+
+	Index   int
+	Mem     float64
+	MemUtil float64
+	GpuUtil float64
+}
+
+func (m PodMthreadsGpuMetrics) GetName() string {
+	return "pod_mthreads_gpu"
+}
+
+func (m PodMthreadsGpuMetrics) GetUniformName() string {
+	return "pod_gpu"
+}
+
+func (m PodMthreadsGpuMetrics) GetTag() map[string]string {
+	return map[string]string{
+		"index":        strconv.Itoa(m.Index),
+		"dev_type":     compute.GPU_TYPE,
+		"sharing_mode": compute.DEVICE_SHARING_MODE_EXCLUSIVE,
+	}
+}
+
+func (m PodMthreadsGpuMetrics) ToMap() map[string]interface{} {
+	return map[string]interface{}{
+		MTHREADS_GPU_MEM:      m.Mem,
+		MTHREADS_GPU_MEM_UTIL: m.MemUtil,
+		MTHREADS_GPU_GPU_UTIL: m.GpuUtil,
+	}
+}
@@ -0,0 +1,100 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestParseMthreadsProcessMonitor(t *testing.T) {
+	buf, err := os.ReadFile(filepath.Join("testdata", "mthreads-pm.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := parseMthreadsProcessMonitor(string(buf))
+	if len(got) != 7 {
+		t.Fatalf("rows=%d, want 7", len(got))
+	}
+	worker := got[2]
+	if worker.Index != 0 || worker.Pid != "1306725" || worker.MemUtil != 0.10 || worker.GpuUtil != 100 {
+		t.Fatalf("worker=%+v", worker)
+	}
+	if got[0].MemUtil != 0 || got[0].GpuUtil != 0 {
+		t.Fatalf("<1%% parsed as %+v", got[0])
+	}
+}
+
+func TestParseMthreadsProcessMemory(t *testing.T) {
+	buf, err := os.ReadFile(filepath.Join("testdata", "mthreads-processes.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := parseMthreadsProcessMemory(string(buf))
+	if len(got) != 6 {
+		t.Fatalf("rows=%d, want 6", len(got))
+	}
+	if got[1].Pid != "1305830" || got[1].MemMiB != 36 {
+		t.Fatalf("engine=%+v", got[1])
+	}
+	if got[2].Pid != "1306725" || got[2].MemMiB != 8758 {
+		t.Fatalf("worker=%+v", got[2])
+	}
+}
+
+func TestAggregateMthreadsPodGpu(t *testing.T) {
+	pm, err := os.ReadFile(filepath.Join("testdata", "mthreads-pm.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	mem, err := os.ReadFile(filepath.Join("testdata", "mthreads-processes.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	procs := mergeMthreadsGpuProcs(parseMthreadsProcessMonitor(string(pm)), parseMthreadsProcessMemory(string(mem)))
+	podProcs := map[string]struct{}{
+		"1305267": {},
+		"1305830": {},
+		"1306725": {},
+	}
+	selected := selectMthreadsGpuProcs(procs, podProcs, []int{0})
+	m := &SGuestMonitor{mthreadsGpuMetrics: selected}
+	got := m.getPodMthreadsGpuMetrics()
+	if len(got) != 1 {
+		t.Fatalf("gpus=%d, want 1", len(got))
+	}
+	if got[0].Index != 0 || got[0].Mem != 8830 || got[0].MemUtil != 0.10 || got[0].GpuUtil != 100 {
+		t.Fatalf("metric=%+v", got[0])
+	}
+	fields := got[0].ToMap()
+	if fields[MTHREADS_GPU_MEM] != 8830.0 || fields[MTHREADS_GPU_MEM_UTIL] != 0.10 || fields[MTHREADS_GPU_GPU_UTIL] != 100.0 {
+		t.Fatalf("fields=%v", fields)
+	}
+	if got[0].GetUniformName() != "pod_gpu" {
+		t.Fatalf("uniform=%s", got[0].GetUniformName())
+	}
+}
+
+func TestAggregateMthreadsMemUtilCapAndGpuMax(t *testing.T) {
+	m := &SGuestMonitor{mthreadsGpuMetrics: []MthreadsGpuProc{
+		{Index: 0, Pid: "1", MemMiB: 100, MemUtil: 0.6, GpuUtil: 20},
+		{Index: 0, Pid: "2", MemMiB: 50, MemUtil: 0.6, GpuUtil: 90},
+	}}
+	got := m.getPodMthreadsGpuMetrics()
+	if len(got) != 1 || got[0].Mem != 150 || got[0].MemUtil != 1 || got[0].GpuUtil != 90 {
+		t.Fatalf("metric=%+v", got)
+	}
+}
@@ -159,10 +159,16 @@ func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string
 		podDesc := pod.GetDesc()
 		hasGpu := false
 		for i := range podDesc.IsolatedDevices {
-			if podDesc.IsolatedDevices[i].DevType != compute.GPU_TYPE {
+			dev := podDesc.IsolatedDevices[i]
+			vendorId := strings.ToLower(strings.Split(dev.VendorDeviceId, ":")[0])
+			if vendorId == compute.MTHREADS_VENDOR_ID {
+				hasGpu = true
+				break
+			}
+			if dev.DevType != compute.GPU_TYPE {
 				continue
 			}
-			if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) {
+			if !utils.IsInStringArray(dev.SharingMode, compute.VIRTUAL_SHARING_MODES) {
 				continue
 			}
 			hasGpu = true
@@ -64,6 +64,8 @@ type IHostInfo interface {
 	HasContainerNvidiaGpu() bool
 	HasContainerVastaitechGpu() bool
 	HasContainerCphAmdGpu() bool
+	HasContainerMthreadsGpu() bool
+	GetMthreadsGpuIndexes(ids, addrs []string) []int
 	GetNvidiaGpuIndexMemoryMap() map[string]int
 	GetNvidiaGpuIndexByDeviceIds(ids []string) map[string]int
 	ReportHostDmesg(data []compute.SKmsgEntry) error
@@ -199,6 +201,7 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor {
 	var nvidiaGpuMetrics []NvidiaGpuProcessMetrics = nil
 	var vastaitechGpuMetrics []VastaitechGpuProcessMetrics = nil
 	var cphAmdGpuMetrics []CphAmdGpuProcessMetrics = nil
+	var mthreadsGpuMetrics []MthreadsGpuProc = nil
 	var gpuPodProcs = s.collectGpuPodsProcesses()

 	guestmanager.Servers.Range(func(k, v interface{}) bool {
@@ -274,13 +277,19 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor {
 						log.Errorf("GetCphAmdGpuProcessMetrics %s", err)
 					}
 				}
+				if s.hostInfo.HasContainerMthreadsGpu() {
+					mthreadsGpuMetrics, err = GetMthreadsGpuProcessMetrics()
+					if err != nil {
+						log.Errorf("GetMthreadsGpuProcessMetrics %s", err)
+					}
+				}
 			}

 			podStat, podProcs := GetPodStatsById(podStats, gpuPodProcs, guestId)
 			if podStat != nil {
 				gm, err := NewGuestPodMonitor(
 					instance, guestName, guestId, podStat,
-					nvidiaGpuMetrics, vastaitechGpuMetrics, cphAmdGpuMetrics,
+					nvidiaGpuMetrics, vastaitechGpuMetrics, cphAmdGpuMetrics, mthreadsGpuMetrics,
 					s.hostInfo, podProcs, nicsDesc, int(vcpuCount),
 				)
 				if err != nil {
@@ -633,6 +642,7 @@ type SGuestMonitor struct {
 	nvidiaGpuAssigned       []nvidiaGpuAssignedQuota
 	vastaitechGpuMetrics    []VastaitechGpuProcessMetrics
 	cphAmdGpuMetrics        []CphAmdGpuProcessMetrics
+	mthreadsGpuMetrics      []MthreadsGpuProc
 	instance                guestman.GuestRuntimeInstance
 	sysFs                   sysfs.SysFs

@@ -650,6 +660,7 @@ func NewGuestMonitor(instance guestman.GuestRuntimeInstance, name, id string, pi
 func NewGuestPodMonitor(
 	instance guestman.GuestRuntimeInstance, name, id string, stat *stats.PodStats,
 	nvidiaGpuMetrics []NvidiaGpuProcessMetrics, vastaitechGpuMetrics []VastaitechGpuProcessMetrics, cphAmdGpuMetrics []CphAmdGpuProcessMetrics,
+	mthreadsGpuMetrics []MthreadsGpuProc,
 	hostInstance IHostInfo, podProcs map[string]struct{}, nics []*desc.SGuestNetwork, cpuCount int,
 ) (*SGuestMonitor, error) {
 	m, err := newGuestMonitor(instance, name, id, nil, nics, cpuCount)
@@ -662,12 +673,25 @@ func NewGuestPodMonitor(
 	hasNvGpu := false
 	hasCphAmdGpu := false
 	hasVastaitechGpu := false
+	hasMthreadsGpu := false
+	mthreadsIds := make([]string, 0)
+	mthreadsAddrs := make([]string, 0)
 	nvAssignInputs := make([]nvidiaGpuAssignInput, 0)
 	for i := range podDesc.IsolatedDevices {
-		if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) {
+		dev := podDesc.IsolatedDevices[i]
+		vendorId := strings.ToLower(strings.Split(dev.VendorDeviceId, ":")[0])
+		if vendorId == compute.MTHREADS_VENDOR_ID {
+			hasMthreadsGpu = true
+			if dev.Id != "" {
+				mthreadsIds = append(mthreadsIds, dev.Id)
+			}
+			if dev.Addr != "" {
+				mthreadsAddrs = append(mthreadsAddrs, dev.Addr)
+			}
+		}
+		if !utils.IsInStringArray(dev.SharingMode, compute.VIRTUAL_SHARING_MODES) {
 			continue
 		}
-		vendorId := strings.Split(podDesc.IsolatedDevices[i].VendorDeviceId, ":")[0]
 		switch vendorId {
 		case compute.NVIDIA_VENDOR_ID:
 			hasNvGpu = true
@@ -698,6 +722,10 @@ func NewGuestPodMonitor(
 	if hasCphAmdGpu {
 		m.cphAmdGpuMetrics = GetPodCphAmdGpuMetrics(cphAmdGpuMetrics, podProcs)
 	}
+	if hasMthreadsGpu {
+		indexes := hostInstance.GetMthreadsGpuIndexes(mthreadsIds, mthreadsAddrs)
+		m.mthreadsGpuMetrics = selectMthreadsGpuProcs(mthreadsGpuMetrics, podProcs, indexes)
+	}
 	return m, nil
 }

@@ -0,0 +1,8 @@
+  gpu       pid   memUtil   gpuUtil
+    0   1305267       <1%        0%
+    0   1305830       <1%        0%
+    0   1306725       10%      100%
+    0   1306726       <1%        0%
+    1   1305267        0%        0%
+    1   1306726       10%      100%
+    1   1306725       <1%        0%
@@ -0,0 +1,12 @@
+Processes:
+ID   PID           Process name                           GPU Memory
+                                                               Usage
+-------------------------------------------------------------------+
+0    1305267       /usr/bin/python                             36MiB
+0    1305830       VLLM::EngineCore                            36MiB
+0    1306725       VLLM::Worker_TP0                          8758MiB
+0    1306726       VLLM::Worker_TP1                            36MiB
+1    1306726       VLLM::Worker_TP1                          8758MiB
+1    1306725       VLLM::Worker_TP0                            36MiB
+-------------------------------------------------------------------+
+|  0%   9081MiB(81920MiB) |
@@ -37,22 +37,24 @@ var (
 type ContainerDeviceType string

 const (
-	ContainerDeviceTypeCphAMDGPU      ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU
-	ContainerDeviceTypeCphASOPBinder  ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER
-	ContainerNetintCAASIC             ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC
-	ContainerNetintCAQuadra           ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA
-	ContainerDeviceTypeNvidiaGpu      ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
-	ContainerDeviceTypeNvidiaMps      ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS
-	ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE
-	ContainerDeviceTypeNvidiaHAMI     ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI
-	ContainerDeviceTypeAscendNpu      ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
-	ContainerDeviceTypeAscendNpuHami  ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI
-	ContainerDeviceTypeVastaitechGpu  ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
-	ContainerDeviceTypeHygonDcu       ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU
-	ContainerDeviceTypeHygonDcuHami   ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI
-	ContainerDeviceTypeIluvatarGpu    ContainerDeviceType = api.CONTAINER_DEV_ILUVATAR_GPU
-	ContainerDeviceTypeTHeadPpu       ContainerDeviceType = api.CONTAINER_DEV_THEAD_PPU
-	ContainerDeviceTypeKunlunxinXpu   ContainerDeviceType = api.CONTAINER_DEV_KUNLUNXIN_XPU
+	ContainerDeviceTypeCphAMDGPU       ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU
+	ContainerDeviceTypeCphASOPBinder   ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER
+	ContainerNetintCAASIC              ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC
+	ContainerNetintCAQuadra            ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA
+	ContainerDeviceTypeNvidiaGpu       ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
+	ContainerDeviceTypeNvidiaMps       ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS
+	ContainerDeviceTypeNvidiaGpuShare  ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE
+	ContainerDeviceTypeNvidiaHAMI      ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI
+	ContainerDeviceTypeAscendNpu       ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU
+	ContainerDeviceTypeAscendNpuHami   ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU_HAMI
+	ContainerDeviceTypeVastaitechGpu   ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
+	ContainerDeviceTypeHygonDcu        ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU
+	ContainerDeviceTypeHygonDcuHami    ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI
+	ContainerDeviceTypeIluvatarGpu     ContainerDeviceType = api.CONTAINER_DEV_ILUVATAR_GPU
+	ContainerDeviceTypeTHeadPpu        ContainerDeviceType = api.CONTAINER_DEV_THEAD_PPU
+	ContainerDeviceTypeKunlunxinXpu    ContainerDeviceType = api.CONTAINER_DEV_KUNLUNXIN_XPU
+	ContainerDeviceTypeMthreadsGpu     ContainerDeviceType = api.CONTAINER_DEV_MTHREADS_GPU
+	ContainerDeviceTypeMthreadsGpuHami ContainerDeviceType = api.CONTAINER_DEV_MTHREADS_GPU_HAMI
 )

 func GetContainerDeviceManager(devType ContainerDeviceType) (IContainerDeviceManager, error) {
@@ -0,0 +1,193 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+)
+
+const mthreadsDeviceTypePhysical = "Physical"
+
+type parsedMthreadsGPU struct {
+	Index        int
+	Name         string
+	BusId        string
+	DeviceType   string
+	Perf         string
+	GPUUtil      int
+	MemoryUsedMB int
+	MemorySizeMB int
+	TempC        int
+	MPCCapable   string
+	ECCMode      string
+}
+
+var (
+	mthreadsIndexRe = regexp.MustCompile(`^(\d+)\s+(.+)$`)
+	mthreadsMemRe   = regexp.MustCompile(`(\d+)%\s+(\d+)MiB\((\d+)MiB\)`)
+	mthreadsTempRe  = regexp.MustCompile(`(\d+)C`)
+)
+
+func splitMthreadsCols(line string) []string {
+	parts := strings.Split(line, "|")
+	cols := make([]string, 0, len(parts))
+	for _, p := range parts {
+		cols = append(cols, strings.TrimSpace(p))
+	}
+	return cols
+}
+
+func parseMthreadsGmiBlock(lines []string) *parsedMthreadsGPU {
+	if len(lines) == 0 {
+		return nil
+	}
+	cols := splitMthreadsCols(lines[0])
+	if len(cols) < 3 {
+		return nil
+	}
+	head := mthreadsIndexRe.FindStringSubmatch(cols[0])
+	if len(head) != 3 {
+		return nil
+	}
+	idx, err := strconv.Atoi(head[1])
+	if err != nil {
+		return nil
+	}
+	gpu := &parsedMthreadsGPU{
+		Index: idx,
+		Name:  strings.TrimSpace(head[2]),
+		BusId: cols[1],
+	}
+	if mem := mthreadsMemRe.FindStringSubmatch(cols[2]); len(mem) == 4 {
+		gpu.GPUUtil, _ = strconv.Atoi(mem[1])
+		gpu.MemoryUsedMB, _ = strconv.Atoi(mem[2])
+		gpu.MemorySizeMB, _ = strconv.Atoi(mem[3])
+	}
+	if len(lines) >= 2 {
+		cols2 := splitMthreadsCols(lines[1])
+		if len(cols2) > 0 {
+			fields := strings.Fields(cols2[0])
+			if len(fields) > 0 {
+				gpu.DeviceType = fields[0]
+			}
+			if len(fields) > 1 {
+				gpu.Perf = fields[1]
+			}
+		}
+		if len(cols2) >= 3 {
+			if tm := mthreadsTempRe.FindStringSubmatch(cols2[2]); len(tm) == 2 {
+				gpu.TempC, _ = strconv.Atoi(tm[1])
+			}
+			parts := strings.Fields(cols2[2])
+			if len(parts) >= 2 {
+				gpu.MPCCapable = parts[len(parts)-1]
+			}
+		}
+	}
+	if len(lines) >= 3 {
+		cols3 := splitMthreadsCols(lines[2])
+		for i := len(cols3) - 1; i >= 0; i-- {
+			if cols3[i] != "" {
+				gpu.ECCMode = cols3[i]
+				break
+			}
+		}
+	}
+	return gpu
+}
+
+func parseMthreadsGmi(output string) []*parsedMthreadsGPU {
+	var out []*parsedMthreadsGPU
+	var buf []string
+	flush := func() {
+		if gpu := parseMthreadsGmiBlock(buf); gpu != nil {
+			out = append(out, gpu)
+		}
+		buf = nil
+	}
+	for _, raw := range strings.Split(output, "\n") {
+		line := strings.TrimSpace(raw)
+		if line == "" {
+			continue
+		}
+		if strings.HasPrefix(line, "+") || strings.HasPrefix(line, "-") {
+			flush()
+			continue
+		}
+		buf = append(buf, line)
+	}
+	flush()
+	return out
+}
+
+func isMthreadsPhysical(gpu *parsedMthreadsGPU) bool {
+	return gpu != nil && strings.EqualFold(gpu.DeviceType, mthreadsDeviceTypePhysical)
+}
+
+func mthreadsPCIAddrCandidates(busId string) []string {
+	return iluvatarPCIAddrCandidates(busId)
+}
+
+func mthreadsDevNode(index int) string {
+	return fmt.Sprintf("/dev/mtgpu.%d", index)
+}
+
+func mthreadsDeviceSpec(devPath string) *runtimeapi.Device {
+	return &runtimeapi.Device{
+		ContainerPath: devPath,
+		HostPath:      devPath,
+		Permissions:   "rwm",
+	}
+}
+
+func buildMthreadsRuntimeEnvs(indices []string) []*runtimeapi.KeyValue {
+	value := strings.Join(indices, ",")
+	return []*runtimeapi.KeyValue{
+		{Key: "MTHREADS_VISIBLE_DEVICES", Value: value},
+		{Key: "MTHREADS_DRIVER_CAPABILITIES", Value: "all"},
+		// {Key: "MUSA_VISIBLE_DEVICES", Value: value},
+		// {Key: "CUDA_VISIBLE_DEVICES", Value: value},
+	}
+}
+
+func buildMthreadsExtraConfigures(indices []string) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	if len(indices) == 0 {
+		return nil, nil
+	}
+	return buildMthreadsRuntimeEnvs(indices), nil
+}
+
+func buildMthreadsHamiQosEnvs(memoryLimitMiB, smUtilLimit int) []*runtimeapi.KeyValue {
+	if memoryLimitMiB <= 0 {
+		return nil
+	}
+	return []*runtimeapi.KeyValue{
+		{
+			Key:   "MTHREADS_QOS_MEMORY_LIMIT",
+			Value: strconv.FormatInt(computeapi.MthreadsSgpuMemoryLimitBytes(memoryLimitMiB), 10),
+		},
+		{
+			Key:   "MTHREADS_QOS_COMPUTING_POWER_WEIGHT",
+			Value: strconv.Itoa(computeapi.MthreadsSgpuComputingPowerWeight(smUtilLimit)),
+		},
+	}
+}
@@ -0,0 +1,110 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseMthreadsGmiS5000(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("testdata", "s5000-8gpu.txt"))
+	require.NoError(t, err)
+
+	gpus := parseMthreadsGmi(string(raw))
+	require.Len(t, gpus, 9)
+
+	physical := make([]*parsedMthreadsGPU, 0, 8)
+	for _, gpu := range gpus {
+		if isMthreadsPhysical(gpu) {
+			physical = append(physical, gpu)
+		}
+	}
+	require.Len(t, physical, 8)
+
+	wantBus := []string{
+		"00000000:18:00.0",
+		"00000000:3a:00.0",
+		"00000000:4b:00.0",
+		"00000000:5c:00.0",
+		"00000000:9a:00.0",
+		"00000000:ba:00.0",
+		"00000000:ca:00.0",
+		"00000000:da:00.0",
+	}
+	wantTemp := []int{45, 38, 44, 36, 45, 37, 45, 38}
+	for i, gpu := range physical {
+		assert.Equal(t, i, gpu.Index)
+		assert.Equal(t, "MTT S5000", gpu.Name)
+		assert.Equal(t, wantBus[i], gpu.BusId)
+		assert.Equal(t, 0, gpu.GPUUtil)
+		assert.Equal(t, 0, gpu.MemoryUsedMB)
+		assert.Equal(t, 81920, gpu.MemorySizeMB)
+		assert.Equal(t, wantTemp[i], gpu.TempC)
+		assert.Equal(t, "P0", gpu.Perf)
+		assert.Equal(t, "YES", gpu.MPCCapable)
+		assert.Equal(t, "On-die, EDC", gpu.ECCMode)
+	}
+
+	virt := gpus[8]
+	assert.Equal(t, 8, virt.Index)
+	assert.Equal(t, "Virt", virt.DeviceType)
+	assert.False(t, isMthreadsPhysical(virt))
+	assert.Equal(t, 10, virt.GPUUtil)
+	assert.Equal(t, 1024, virt.MemoryUsedMB)
+}
+
+func TestBuildMthreadsExtraConfigures(t *testing.T) {
+	envs, mounts := buildMthreadsExtraConfigures([]string{"0", "1", "2", "3"})
+	assert.Nil(t, mounts)
+	require.Len(t, envs, 2)
+	got := map[string]string{}
+	for _, env := range envs {
+		got[env.Key] = env.Value
+	}
+	assert.Equal(t, "0,1,2,3", got["MTHREADS_VISIBLE_DEVICES"])
+	assert.Equal(t, "all", got["MTHREADS_DRIVER_CAPABILITIES"])
+
+	envs, mounts = buildMthreadsExtraConfigures(nil)
+	assert.Nil(t, envs)
+	assert.Nil(t, mounts)
+}
+
+func TestBuildMthreadsHamiQosEnvs(t *testing.T) {
+	envs := buildMthreadsHamiQosEnvs(2048, 25)
+	require.Len(t, envs, 2)
+	got := map[string]string{}
+	for _, env := range envs {
+		got[env.Key] = env.Value
+	}
+	assert.Equal(t, "2147483648", got["MTHREADS_QOS_MEMORY_LIMIT"])
+	assert.Equal(t, "25", got["MTHREADS_QOS_COMPUTING_POWER_WEIGHT"])
+
+	zero := buildMthreadsHamiQosEnvs(0, 25)
+	assert.Nil(t, zero)
+
+	defaultWeight := buildMthreadsHamiQosEnvs(512, 0)
+	require.Len(t, defaultWeight, 2)
+	assert.Equal(t, "1", defaultWeight[1].Value)
+}
+
+func TestMthreadsDevNode(t *testing.T) {
+	assert.Equal(t, "/dev/mtgpu.0", mthreadsDevNode(0))
+	assert.Equal(t, "/dev/mtgpu.7", mthreadsDevNode(7))
+}
@@ -0,0 +1,216 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newMthreadsGPUManager())
+}
+
+type mthreadsGPUManager struct{}
+
+func newMthreadsGPUManager() *mthreadsGPUManager {
+	return &mthreadsGPUManager{}
+}
+
+func (m *mthreadsGPUManager) GetRegisterType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeMthreadsGpu
+}
+
+func (m *mthreadsGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	return getMthreadsGPUs(m, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE)
+}
+
+func (m *mthreadsGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *mthreadsGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	return newMthreadsContainerDevices(dev)
+}
+
+func (m *mthreadsGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	return buildMthreadsExtraConfigures(collectMthreadsVisibleIndices(devs))
+}
+
+type mthreadsGPU struct {
+	manager isolated_device.IContainerDeviceManager
+	*BaseDevice
+
+	memSize  int
+	gpuIndex int
+}
+
+func (dev *mthreadsGPU) GetMemorySize() int {
+	return dev.memSize
+}
+
+func (dev *mthreadsGPU) GetIndex() int {
+	return dev.gpuIndex
+}
+
+func (dev *mthreadsGPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager {
+	return dev.manager
+}
+
+const defaultMthreadsGmiPath = "/usr/bin/mthreads-gmi"
+
+func mthreadsGmiPath() string {
+	p := options.HostOptions.MthreadsGmiPath
+	if p != "" {
+		return p
+	}
+	return defaultMthreadsGmiPath
+}
+
+func runMthreadsGmi(args ...string) (string, error) {
+	bin := mthreadsGmiPath()
+	cmd := procutils.NewRemoteCommandAsFarAsPossible(bin, args...)
+	out, err := cmd.Output()
+	if err != nil {
+		return string(out), errors.Wrapf(err, "mthreads-gmi %s", strings.Join(args, " "))
+	}
+	return string(out), nil
+}
+
+func lookupMthreadsPCIDevice(busId, modelName string) *isolated_device.PCIDevice {
+	cands := mthreadsPCIAddrCandidates(busId)
+	for _, addr := range cands {
+		pciOutput, err := isolated_device.GetPCIStrByAddr(addr)
+		if err != nil || len(pciOutput) == 0 {
+			log.Warningf("mthreads GetPCIStrByAddr %s: %v", addr, err)
+			continue
+		}
+		dev := isolated_device.NewPCIDevice2(pciOutput[0])
+		if modelName != "" {
+			dev.ModelName = modelName
+		}
+		return dev
+	}
+	fallbackAddr := busId
+	if len(cands) > 0 {
+		fallbackAddr = cands[len(cands)-1]
+	}
+	return &isolated_device.PCIDevice{
+		VendorId:   computeapi.MTHREADS_VENDOR_ID,
+		VendorName: "MTHREADS",
+		Addr:       fallbackAddr,
+		ModelName:  modelName,
+		DeviceName: modelName,
+	}
+}
+
+func collectMthreadsVisibleIndices(devs []*hostapi.ContainerDevice) []string {
+	indices := []string{}
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+		if iDev == nil {
+			continue
+		}
+		devMan := iDev.GetContainerDeviceManager()
+		if _, ok := devMan.(*mthreadsGPUManager); !ok {
+			continue
+		}
+		if dev.IsolatedDevice.Path != "" {
+			indices = append(indices, dev.IsolatedDevice.Path)
+			continue
+		}
+		if dev.IsolatedDevice.Index >= 0 {
+			indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index))
+		}
+	}
+	return indices
+}
+
+func newMthreadsContainerDevices(dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	if dev.IsolatedDevice == nil {
+		return nil, nil, errors.Errorf("isolated device is nil")
+	}
+	iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+	if iDev == nil {
+		return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id)
+	}
+	gpuDev, ok := iDev.(*mthreadsGPU)
+	if !ok {
+		return nil, nil, errors.Errorf("device %s is not mthreads gpu", dev.IsolatedDevice.Id)
+	}
+	ctrDevs := []*runtimeapi.Device{}
+	node := mthreadsDevNode(gpuDev.GetIndex())
+	if hygonPathExists(node) {
+		ctrDevs = append(ctrDevs, mthreadsDeviceSpec(node))
+	} else {
+		log.Warningf("mthreads container device %s not found, skip", node)
+	}
+	return ctrDevs, nil, nil
+}
+
+func getMthreadsGPUs(manager isolated_device.IContainerDeviceManager, sharingMode string) ([]isolated_device.IDevice, error) {
+	bin := mthreadsGmiPath()
+	if !hygonPathExists(bin) {
+		log.Infof("mthreads gpu probe skipped: mthreads-gmi not found at %s", bin)
+		return nil, nil
+	}
+
+	tableOut, err := runMthreadsGmi()
+	if err != nil {
+		return nil, err
+	}
+	parsed := parseMthreadsGmi(tableOut)
+	devs := make([]isolated_device.IDevice, 0, len(parsed))
+	for _, gpu := range parsed {
+		if !isMthreadsPhysical(gpu) {
+			log.Infof("mthreads gpu %d device type %q, skip", gpu.Index, gpu.DeviceType)
+			continue
+		}
+		pciDev := lookupMthreadsPCIDevice(gpu.BusId, gpu.Name)
+		indexStr := strconv.Itoa(gpu.Index)
+		dev := &mthreadsGPU{
+			manager:    manager,
+			BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, sharingMode, 1),
+			memSize:    gpu.MemorySizeMB,
+			gpuIndex:   gpu.Index,
+		}
+		if gpu.Name != "" {
+			dev.SetModelName(gpu.Name)
+		}
+		log.Infof("mthreads gpu idx=%d model=%s pci=%s memMiB=%d path=%s node=%s",
+			gpu.Index, gpu.Name, pciDev.Addr, gpu.MemorySizeMB, indexStr, mthreadsDevNode(gpu.Index))
+		devs = append(devs, dev)
+	}
+	if len(devs) == 0 {
+		log.Infof("mthreads gpu probe finished: no physical devices found")
+		return nil, nil
+	}
+	return devs, nil
+}
@@ -0,0 +1,117 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"strconv"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+const mthreadsSgpuPolicyWeak = 1
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newMthreadsGPUHamiManager())
+}
+
+type mthreadsGPUHamiManager struct{}
+
+func newMthreadsGPUHamiManager() *mthreadsGPUHamiManager {
+	return &mthreadsGPUHamiManager{}
+}
+
+func (m *mthreadsGPUHamiManager) GetRegisterType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeMthreadsGpuHami
+}
+
+func (m *mthreadsGPUHamiManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	devs, err := getMthreadsGPUs(m, computeapi.DEVICE_SHARING_MODE_HAMI)
+	if err != nil {
+		return nil, err
+	}
+	for _, dev := range devs {
+		gpu, ok := dev.(*mthreadsGPU)
+		if !ok {
+			continue
+		}
+		configureMthreadsSgpu(gpu.GetIndex())
+	}
+	return devs, nil
+}
+
+func (m *mthreadsGPUHamiManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *mthreadsGPUHamiManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	return newMthreadsContainerDevices(dev)
+}
+
+func (m *mthreadsGPUHamiManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	indices := make([]string, 0)
+	memoryLimit := 0
+	smLimit := 0
+	seenLimit := false
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+		if iDev == nil {
+			continue
+		}
+		if _, ok := iDev.GetContainerDeviceManager().(*mthreadsGPUHamiManager); !ok {
+			continue
+		}
+		if dev.IsolatedDevice.Path != "" {
+			indices = append(indices, dev.IsolatedDevice.Path)
+		} else if dev.IsolatedDevice.Index >= 0 {
+			indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index))
+		}
+		if !seenLimit {
+			memoryLimit = dev.IsolatedDevice.MemoryLimit
+			smLimit = dev.IsolatedDevice.SmUtilLimit
+			seenLimit = true
+		}
+	}
+	envs, mounts := buildMthreadsExtraConfigures(indices)
+	if len(envs) == 0 {
+		return nil, nil
+	}
+	envs = append(envs, buildMthreadsHamiQosEnvs(memoryLimit, smLimit)...)
+	return envs, mounts
+}
+
+func configureMthreadsSgpu(index int) {
+	writeMthreadsSgpuProc(index, "policy", mthreadsSgpuPolicyWeak)
+	writeMthreadsSgpuProc(index, "max_inst", computeapi.MthreadsSgpuMaxInstances)
+}
+
+func writeMthreadsSgpuProc(index int, node string, value int) {
+	procPath := fmt.Sprintf("/proc/sgpu_km/%d/%s", index, node)
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", fmt.Sprintf("echo %d > %s", value, procPath)).Output()
+	if err != nil {
+		log.Warningf("mthreads sgpu set %s=%d: %v %s", procPath, value, err, out)
+	}
+}
@@ -0,0 +1,54 @@
+Tue Oct 13 20:09:50 2026
+---------------------------------------------------------------------
+    mthreads-gmi:2.3.3           Driver Version:3.3.8-server
+---------------------------------------------------------------------
+ID   Name                 |PCIe                |%GPU  Mem
+     Device Type   Perf   |Pcie Lane Width     |Temp  MPC Capable
+                                               |      ECC Mode
+-------------------------------------------------------------------+
+0    MTT S5000            |00000000:18:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |45C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+1    MTT S5000            |00000000:3a:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |38C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+2    MTT S5000            |00000000:4b:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |44C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+3    MTT S5000            |00000000:5c:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |36C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+4    MTT S5000            |00000000:9a:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |45C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+5    MTT S5000            |00000000:ba:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |37C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+6    MTT S5000            |00000000:ca:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |45C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+7    MTT S5000            |00000000:da:00.0    |0%    0MiB(81920MiB)
+     Physical      P0     |16x(16x)            |38C   YES
+                                               |      On-die, EDC
+-------------------------------------------------------------------+
+8    MTT S5000 MPC        |00000000:ea:00.0    |10%   1024MiB(81920MiB)
+     Virt          P0     |16x(16x)            |40C   YES
+                                               |      Off
+-------------------------------------------------------------------+
+
+---------------------------------------------------------------------
+
+---------------------------------------------------------------------
+Processes:
+ID   PID           Process name                           GPU Memory
+                                                               Usage
+-------------------------------------------------------------------+
+   No running processes found
+---------------------------------------------------------------------
@@ -399,6 +399,34 @@ func (man *isolatedDeviceManager) probeContainerKunlunxinXPUs(enable bool) {
 	}
 }

+func (man *isolatedDeviceManager) probeContainerMthreadsGPUs(enable, enableHami bool) {
+	devType := ContainerDeviceTypeMthreadsGpu
+	if enableHami {
+		devType = ContainerDeviceTypeMthreadsGpuHami
+	} else if !enable {
+		log.Infof("mthreads gpu probe skipped: enable_container_mthreads_gpu=false and enable_container_mthreads_gpu_hami=false")
+		return
+	}
+	devman, err := GetContainerDeviceManager(devType)
+	if err != nil {
+		log.Errorf("no container device manager %s found: %v", devType, err)
+		return
+	}
+	devs, err := devman.ProbeDevices()
+	if err != nil {
+		log.Warningf("Probe container mthreads gpu devices: %v", err)
+		return
+	}
+	if len(devs) == 0 {
+		log.Infof("mthreads gpu probe finished: no devices found")
+		return
+	}
+	for idx, dev := range devs {
+		man.devices = append(man.devices, dev)
+		log.Infof("Add Container mthreads GPU device: %d => %#v", idx, dev)
+	}
+}
+
 func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) {
 	if skipGPUs {
 		return
@@ -567,16 +595,18 @@ type SIsolatedDeviceProbeOptions struct {
 	SkipUSBs       bool
 	SkipCustomDevs bool

-	EnableCudaHAMI               bool
-	EnableCudaMps                bool
-	EnableContainerAscendNpu     bool
-	EnableContainerAscendNpuHAMI bool
-	EnableContainerHygonDCU      bool
-	EnableContainerHygonDCUHAMI  bool
-	EnableContainerIluvatarGPU   bool
-	EnableContainerTHeadPPU      bool
-	EnableContainerKunlunxinXPU  bool
-	EnableWhitelist              bool
+	EnableCudaHAMI                 bool
+	EnableCudaMps                  bool
+	EnableContainerAscendNpu       bool
+	EnableContainerAscendNpuHAMI   bool
+	EnableContainerHygonDCU        bool
+	EnableContainerHygonDCUHAMI    bool
+	EnableContainerIluvatarGPU     bool
+	EnableContainerTHeadPPU        bool
+	EnableContainerKunlunxinXPU    bool
+	EnableContainerMthreadsGPU     bool
+	EnableContainerMthreadsGPUHami bool
+	EnableWhitelist                bool

 	SriovNics, OvsOffloadNics []HostNic

@@ -596,6 +626,7 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti
 		man.probeContainerIluvatarGPUs(opts.EnableContainerIluvatarGPU)
 		man.probeContainerTHeadPPUs(opts.EnableContainerTHeadPPU)
 		man.probeContainerKunlunxinXPUs(opts.EnableContainerKunlunxinXPU)
+		man.probeContainerMthreadsGPUs(opts.EnableContainerMthreadsGPU, opts.EnableContainerMthreadsGPUHami)
 	} else {
 		log.Infof("==== ProbePCIDevices: not container host, hygon container probe will NOT run (use host_type=container for hygon dcu)")
 		devModels, err := man.getCustomIsolatedDeviceModels()
@@ -300,6 +300,10 @@ type SHostOptions struct {
 	KunlunxinXreHome            string `help:"kunlunxin xre home" default:"/usr/local/xpu"`
 	KunlunxinXpuSmiPath         string `help:"kunlunxin xpu-smi path" default:"/usr/local/bin/xpu-smi"`

+	EnableContainerMthreadsGPU     bool   `help:"enable container mthreads gpu" default:"true"`
+	EnableContainerMthreadsGPUHami bool   `help:"enable container mthreads gpu hami" default:"false"`
+	MthreadsGmiPath                string `help:"mthreads-gmi path" default:"/usr/bin/mthreads-gmi"`
+
 	EnableDirtyRecoverySeconds int  `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"`
 	EnableContainerCniPortmap  bool `help:"Use container cni portmap plugin" default:"false"`
 	DisableReconcileContainer  bool `help:"disable reconcile container" default:"false"`
@@ -41,6 +41,7 @@ const (
 	TELEGRAF_INPUT_IXSMI               = "ixsmi"
 	TELEGRAF_INPUT_PPUSMI              = "ppusmi"
 	TELEGRAF_INPUT_XPUSMI              = "xpusmi"
+	TELEGRAF_INPUT_MTHREADSGMI         = "mthreadsgmi"
 	TELEGRAF_INPUT_NVIDIASMI           = "nvidia-smi"
 	TELEGRAF_INPUT_NPUSMI              = "npu-smi"
 	TELEGRAF_INPUT_CONF_LIB_PATH       = "lib_path"
@@ -380,6 +381,13 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string {
 		conf += "\n"
 	}

+	if mtgmi, ok := kwargs[TELEGRAF_INPUT_MTHREADSGMI]; ok {
+		mtgmiMap, _ := mtgmi.(map[string]interface{})
+		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_MTHREADSGMI)
+		conf += fmt.Sprintf("  bin_path = \"%s\"\n", mtgmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string))
+		conf += "\n"
+	}
+
 	if _, ok := kwargs[TELEGRAF_INPUT_NVIDIASMI]; ok {
 		conf += "[[inputs.nvidia_smi]]\n"
 		conf += "\n"
@@ -433,6 +433,7 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo
 	iluvatar := models.HasIluvatarDevices(llm, sku)
 	thead := models.HasTHeadDevices(llm, sku)
 	kunlunxin := models.HasKunlunxinDevices(llm, sku)
+	mthreads := models.HasMthreadsDevices(llm, sku)
 	startScript := buildVLLMEntrypointScript(modelPath, tensorParallelSize, backendParameters, effSpec, hygon)
 	envs := []*commonapi.ContainerKeyValue{
 		{
@@ -465,7 +466,7 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo
 			Envs:              envs,
 		},
 	}
-	if hygon || iluvatar || thead || kunlunxin {
+	if hygon || iluvatar || thead || kunlunxin || mthreads {
 		spec.Command = []string{"/bin/bash", "-c"}
 	}
 	if hasMountedModels {
@@ -190,3 +190,46 @@ func TestVLLMGetContainerSpecKunlunxinRuntime(t *testing.T) {
 		t.Fatalf("args should not source dtk env for kunlunxin, got %#v", spec.Args)
 	}
 }
+
+func TestVLLMGetContainerSpecMthreadsRuntime(t *testing.T) {
+	v := newVLLM().(*vllm)
+	hostPaths := api.HostPaths{
+		{
+			Type: "directory",
+			Path: "/data/model/Qwen",
+			Containers: api.ContainerHostPathRelations{
+				"0": {MountPath: "/data/model/Qwen"},
+			},
+		},
+	}
+	sku := &models.SLLMSku{
+		SLLMSkuBase: models.SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU},
+				{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU},
+			},
+		},
+		LLMType:   string(api.LLM_CONTAINER_VLLM),
+		Source:    api.LLM_MODEL_SOURCE_LOCAL_PATH,
+		LocalPath: "/data/model/Qwen",
+	}
+	sku.HostPaths = &hostPaths
+	image := &models.SLLMImage{}
+	out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "")
+	if out == nil {
+		t.Fatal("expected container spec")
+	}
+	spec := &out.ContainerSpec
+	if spec.Capabilities != nil {
+		t.Fatalf("expected nil capabilities for mthreads, got %#v", spec.Capabilities)
+	}
+	if spec.SecurityContext != nil {
+		t.Fatalf("expected nil security context for mthreads, got %#v", spec.SecurityContext)
+	}
+	if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" {
+		t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command)
+	}
+	if len(spec.Args) == 0 || !strings.Contains(spec.Args[0], "--tensor-parallel-size 2") {
+		t.Fatalf("args = %#v, want tensor-parallel-size 2", spec.Args)
+	}
+}
@@ -437,6 +437,23 @@ func HasTHeadDevices(llm *SLLM, sku *SLLMSku) bool {
 	return false
 }

+// HasMthreadsDevices reports whether effective devices include Moore Threads GPU.
+func HasMthreadsDevices(llm *SLLM, sku *SLLMSku) bool {
+	devs := GetEffectiveDevices(llm, sku)
+	if devs == nil {
+		return false
+	}
+	for _, d := range *devs {
+		if strings.EqualFold(d.Vendor, "MTHREADS") {
+			return true
+		}
+		if d.DevType == computeapi.CONTAINER_DEV_MTHREADS_GPU || d.DevType == computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI {
+			return true
+		}
+	}
+	return false
+}
+
 // HasKunlunxinDevices reports whether effective devices include Kunlunxin XPU.
 func HasKunlunxinDevices(llm *SLLM, sku *SLLMSku) bool {
 	devs := GetEffectiveDevices(llm, sku)
@@ -201,3 +201,60 @@ func TestHasKunlunxinDevices(t *testing.T) {
 		t.Fatal("expected normalized GPU+KUNLUNXIN vendor sku to be detected as Kunlunxin")
 	}
 }
+
+func TestHasMthreadsDevices(t *testing.T) {
+	mthreadsSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU},
+			},
+		},
+	}
+	if !HasMthreadsDevices(nil, mthreadsSku) {
+		t.Fatal("expected Moore Threads GPU sku to be detected")
+	}
+
+	hamiSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI},
+			},
+		},
+	}
+	if !HasMthreadsDevices(nil, hamiSku) {
+		t.Fatal("expected Moore Threads HAMI sku to be detected")
+	}
+
+	nvSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU},
+			},
+		},
+	}
+	if HasMthreadsDevices(nil, nvSku) {
+		t.Fatal("expected NVIDIA sku not to be detected as Moore Threads")
+	}
+
+	llm := &SLLM{
+		SLLMBase: SLLMBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU},
+			},
+		},
+	}
+	if !HasMthreadsDevices(llm, nvSku) {
+		t.Fatal("expected llm device override to win over sku")
+	}
+
+	normalizedSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.GPU_TYPE, Vendor: "MTHREADS", Model: "MTT S5000"},
+			},
+		},
+	}
+	if !HasMthreadsDevices(nil, normalizedSku) {
+		t.Fatal("expected normalized GPU+MTHREADS vendor sku to be detected as Moore Threads")
+	}
+}
@@ -86,14 +86,22 @@ func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred
 	return input, nil
 }

-// normalizeLLMSkuDevices maps NVIDIA_* / HYGON_* / ASCEND_* / ILUVATAR_* / THEAD_* / KUNLUNXIN_* DevTypes onto
+// normalizeLLMSkuDevices maps NVIDIA_* / HYGON_* / ASCEND_* / ILUVATAR_* / THEAD_* / KUNLUNXIN_* / MTHREADS_* DevTypes onto
 // GPU|NPU + SharingMode, and defaults empty DevType/SharingMode appropriately.
 func normalizeLLMSkuDevices(devices *api.Devices) error {
 	if devices == nil || len(*devices) == 0 {
 		return nil
 	}
+	mthreadsHami := 0
 	for i := range *devices {
 		normalizeLLMSkuDevice(&(*devices)[i])
+		dev := (*devices)[i]
+		if computeapi.IsMthreadsHami(dev.Vendor, dev.SharingMode) {
+			mthreadsHami++
+		}
+	}
+	if mthreadsHami > 1 {
+		return httperrors.NewInputParameterError("Moore Threads sGPU does not support multiple cards in one task; use exclusive mode for multiple cards")
 	}
 	return nil
 }
@@ -175,6 +183,16 @@ func normalizeLLMSkuDevice(dev *api.Device) {
 		if dev.SharingMode == "" {
 			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
 		}
+	case computeapi.CONTAINER_DEV_MTHREADS_GPU:
+		dev.DevType = computeapi.GPU_TYPE
+		if dev.SharingMode == "" {
+			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
+		}
+	case computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI:
+		dev.DevType = computeapi.GPU_TYPE
+		if dev.SharingMode == "" {
+			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
+		}
 	}
 	if dev.SharingMode == "" {
 		dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
@@ -194,6 +212,8 @@ func normalizeLLMSkuDevice(dev *api.Device) {
 			dev.Vendor = "THEAD"
 		case computeapi.CONTAINER_DEV_KUNLUNXIN_XPU:
 			dev.Vendor = "KUNLUNXIN"
+		case computeapi.CONTAINER_DEV_MTHREADS_GPU, computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI:
+			dev.Vendor = "MTHREADS"
 		}
 	}
 	dev.Vendor = canonicalizeLLMDeviceVendor(dev.Vendor)
@@ -104,6 +104,20 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) {
 			wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE,
 			wantVendor:      "KUNLUNXIN",
 		},
+		{
+			name:            "MTHREADS_GPU",
+			in:              api.Device{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU},
+			wantDevType:     computeapi.GPU_TYPE,
+			wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE,
+			wantVendor:      "MTHREADS",
+		},
+		{
+			name:            "MTHREADS_GPU_HAMI",
+			in:              api.Device{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI},
+			wantDevType:     computeapi.GPU_TYPE,
+			wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI,
+			wantVendor:      "MTHREADS",
+		},
 		{
 			name:            "ILUVATAR vendor preserved and canonicalized",
 			in:              api.Device{DevType: computeapi.GPU_TYPE, Vendor: "iluvatar", Model: "BI-V150S"},
@@ -157,6 +171,30 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) {
 	}
 }

+func TestNormalizeLLMSkuDevicesRejectsMultipleMthreadsHami(t *testing.T) {
+	sliced := api.Devices{
+		{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI, Model: "MTT S5000"},
+		{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU_HAMI, Model: "MTT S5000"},
+	}
+	if err := normalizeLLMSkuDevices(&sliced); err == nil {
+		t.Fatal("expected multiple Moore Threads sGPU cards to be rejected")
+	}
+	exclusive := api.Devices{
+		{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU, Model: "MTT S5000"},
+		{DevType: computeapi.CONTAINER_DEV_MTHREADS_GPU, Model: "MTT S5000"},
+	}
+	if err := normalizeLLMSkuDevices(&exclusive); err != nil {
+		t.Fatalf("exclusive multi-card should be allowed: %v", err)
+	}
+	nvidia := api.Devices{
+		{DevType: computeapi.CONTAINER_DEV_NVIDIA_HAMI, Model: "A100"},
+		{DevType: computeapi.CONTAINER_DEV_NVIDIA_HAMI, Model: "A100"},
+	}
+	if err := normalizeLLMSkuDevices(&nvidia); err != nil {
+		t.Fatalf("NVIDIA HAMI multi-card should be allowed: %v", err)
+	}
+}
+
 func TestNormalizeLLMSkuDevicesAllowsHAMIWithoutStoredVram(t *testing.T) {
 	devs := api.Devices{{Model: "A100"}}
 	if err := normalizeLLMSkuDevices(&devs); err != nil {
@@ -41,6 +41,7 @@ var All = []SMeasurement{
 	ixsmi,
 	ppusmi,
 	xpusmi,
+	mthreadsGmi,

 	worker,
 	serviceHttpCode,
@@ -165,14 +165,14 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "nvidia_smi","Nvidia GPU metrics","host","telegraf","memory_used","GPU memory used size",""
 "nvidia_smi","Nvidia GPU metrics","host","telegraf","utilization_gpu","GPU utilization","%"
 "nvidia_smi","Nvidia GPU metrics","host","telegraf","utilization_memory","GPU memory utilization","%"
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","clocks_current_graphics","GPU current clocks, MHz",""
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","temperature_gpu","GPU temperature",""
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","memory_total","GPU memory total size",""
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","memory_free","GPU memory free size",""
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","memory_used","GPU memory used size",""
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","utilization_gpu","GPU utilization","%"
-"agent_nvidia_smi","Nvidia GPU metrics in guests","agent","telegraf","utilization_memory","GPU memory utilization","%"
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","clocks_current_graphics","GPU current clocks, MHz",""
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","temperature_gpu","GPU temperature",""
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","memory_total","GPU memory total size",""
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","memory_free","GPU memory free size",""
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","memory_used","GPU memory used size",""
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","utilization_gpu","GPU utilization","%"
+"agent_nvidia_smi","Nvidia GPU metrics","agent","telegraf","utilization_memory","GPU memory utilization","%"
 "processes","Processes status","host","telegraf","total","Total processes count","count"
 "processes","Processes status","host","telegraf","total_threads","Total threads count","count"
 "processes","Processes status","host","telegraf","zombies","Z state, zombie processes count","count"
@@ -247,16 +247,26 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "vasmi","Vasmi GPU metrics","host","telegraf","eclk","eclk, MHz",""
 "vasmi","Vasmi GPU metrics","host","telegraf","gclk","gclk, MHz",""
 "vasmi","Vasmi GPU metrics","host","telegraf","aic_power","AIC power",""
-"npu_smi","NPU metrics","host","telegraf","temperature","NPU temperature",""
-"npu_smi","NPU metrics","host","telegraf","npu_real_time_power","NPU real time power",""
-"npu_smi","NPU metrics","host","telegraf","npu_utilization","NPU utilization","%"
-"npu_smi","NPU metrics","host","telegraf","hbm_usage_rate","NPU hbm usage rate","%"
-"npu_smi","NPU metrics","host","telegraf","hbm_capacity","NPU hbm capacity",""
-"npu_smi","NPU metrics","host","telegraf","aicore_usage_rate","NPU aicore usage rate","%"
-"npu_smi","NPU metrics","host","telegraf","aivector_usage_rate","NPU aivector usage rate","%"
-"npu_smi","NPU metrics","host","telegraf","aicube_usage_rate","NPU aicube usage rate","%"
-"npu_smi","NPU metrics","host","telegraf","aicpu_usage_rate","NPU aicpu usage rate","%"
-"npu_smi","NPU metrics","host","telegraf","hbm_bandwidth_usage_rate","NPU hbm bandwidth usage rate",""
+"npu_smi","Ascend NPU metrics","host","telegraf","temperature","NPU temperature",""
+"npu_smi","Ascend NPU metrics","host","telegraf","npu_real_time_power","NPU real time power",""
+"npu_smi","Ascend NPU metrics","host","telegraf","npu_utilization","NPU utilization","%"
+"npu_smi","Ascend NPU metrics","host","telegraf","hbm_usage_rate","NPU hbm usage rate","%"
+"npu_smi","Ascend NPU metrics","host","telegraf","hbm_capacity","NPU hbm capacity",""
+"npu_smi","Ascend NPU metrics","host","telegraf","aicore_usage_rate","NPU aicore usage rate","%"
+"npu_smi","Ascend NPU metrics","host","telegraf","aivector_usage_rate","NPU aivector usage rate","%"
+"npu_smi","Ascend NPU metrics","host","telegraf","aicube_usage_rate","NPU aicube usage rate","%"
+"npu_smi","Ascend NPU metrics","host","telegraf","aicpu_usage_rate","NPU aicpu usage rate","%"
+"npu_smi","Ascend NPU metrics","host","telegraf","hbm_bandwidth_usage_rate","NPU hbm bandwidth usage rate",""
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","temperature","NPU temperature",""
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","npu_real_time_power","NPU real time power",""
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","npu_utilization","NPU utilization","%"
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","hbm_usage_rate","NPU hbm usage rate","%"
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","hbm_capacity","NPU hbm capacity",""
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aicore_usage_rate","NPU aicore usage rate","%"
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aivector_usage_rate","NPU aivector usage rate","%"
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aicube_usage_rate","NPU aicube usage rate","%"
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","aicpu_usage_rate","NPU aicpu usage rate","%"
+"agent_npu_smi","Ascend NPU metrics","agent","telegraf","hbm_bandwidth_usage_rate","NPU hbm bandwidth usage rate",""
 "hysmi","Hygon DCU metrics","host","telegraf","temperature_gpu","DCU temperature",""
 "hysmi","Hygon DCU metrics","host","telegraf","power_draw","DCU power draw",""
 "hysmi","Hygon DCU metrics","host","telegraf","power_cap","DCU power cap",""
@@ -264,6 +274,31 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "hysmi","Hygon DCU metrics","host","telegraf","utilization_memory","DCU memory utilization","%"
 "hysmi","Hygon DCU metrics","host","telegraf","utilization_encoder","DCU encoder utilization","%"
 "hysmi","Hygon DCU metrics","host","telegraf","utilization_decoder","DCU decoder utilization","%"
+"hysmi","Hygon DCU metrics","host","telegraf","memory_total","GPU memory total size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_free","GPU memory free size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_used","GPU memory used size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_gtt_total","GPU memory gtt size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_gtt_free","GPU memory gtt free size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_gtt_used","GPU memory gtt used size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_vis_vram_total","GPU memory vis vram size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_vis_vram_free","GPU memory vis vram free size",""
+"hysmi","Hygon DCU metrics","host","telegraf","memory_vis_vram_used","GPU memory vis vram used size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","temperature_gpu","DCU temperature",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","power_draw","DCU power draw",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","power_cap","DCU power cap",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_gpu","DCU utilization","%"
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_memory","DCU memory utilization","%"
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_encoder","DCU encoder utilization","%"
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","utilization_decoder","DCU decoder utilization","%"
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_total","GPU memory total size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_free","GPU memory free size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_used","GPU memory used size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_gtt_total","GPU memory gtt size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_gtt_free","GPU memory gtt free size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_gtt_used","GPU memory gtt used size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_vis_vram_total","GPU memory vis vram size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_vis_vram_free","GPU memory vis vram free size",""
+"agent_hysmi","Hygon DCU metrics","agent","telegraf","memory_vis_vram_used","GPU memory vis vram used size",""
 "ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_gpu","GPU temperature",""
 "ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_memory","GPU memory temperature",""
 "ixsmi","Iluvatar GPU metrics","host","telegraf","memory_total","GPU memory total size",""
@@ -274,6 +309,16 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "ixsmi","Iluvatar GPU metrics","host","telegraf","power_draw","GPU power draw",""
 "ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_sm","GPU current SM clocks, MHz",""
 "ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","temperature_gpu","GPU temperature",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","temperature_memory","GPU memory temperature",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","memory_total","GPU memory total size",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","memory_free","GPU memory free size",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","memory_used","GPU memory used size",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","utilization_gpu","GPU utilization","%"
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","utilization_memory","GPU memory utilization","%"
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","power_draw","GPU power draw",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","clocks_current_sm","GPU current SM clocks, MHz",""
+"agent_ixsmi","Iluvatar GPU metrics","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
 "ppusmi","T-Head PPU metrics","host","telegraf","temperature_gpu","PPU temperature",""
 "ppusmi","T-Head PPU metrics","host","telegraf","temperature_memory","PPU memory temperature",""
 "ppusmi","T-Head PPU metrics","host","telegraf","memory_total","PPU memory total size",""
@@ -284,6 +329,16 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "ppusmi","T-Head PPU metrics","host","telegraf","power_draw","PPU power draw",""
 "ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_sm","PPU current CU clocks, MHz",""
 "ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_memory","PPU current memory clocks, MHz",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","temperature_gpu","PPU temperature",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","temperature_memory","PPU memory temperature",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","memory_total","PPU memory total size",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","memory_free","PPU memory free size",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","memory_used","PPU memory used size",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","utilization_gpu","PPU utilization","%"
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","utilization_memory","PPU memory utilization","%"
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","power_draw","PPU power draw",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","clocks_current_sm","PPU current CU clocks, MHz",""
+"agent_ppusmi","T-Head PPU metrics","agent","telegraf","clocks_current_memory","PPU current memory clocks, MHz",""
 "xpusmi","Kunlunxin XPU metrics","host","telegraf","temperature_gpu","XPU temperature",""
 "xpusmi","Kunlunxin XPU metrics","host","telegraf","memory_total","XPU memory total size",""
 "xpusmi","Kunlunxin XPU metrics","host","telegraf","memory_free","XPU memory free size",""
@@ -302,6 +357,52 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "xpusmi","Kunlunxin XPU metrics","host","telegraf","ecc_errors_dram_uncorrectable","XPU uncorrectable DRAM ECC errors",""
 "xpusmi","Kunlunxin XPU metrics","host","telegraf","ecc_errors_dram_correctable_aggregate","XPU correctable DRAM ECC errors in total",""
 "xpusmi","Kunlunxin XPU metrics","host","telegraf","ecc_errors_dram_uncorrectable_aggregate","XPU uncorrectable DRAM ECC errors in total",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","temperature_gpu","XPU temperature",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","memory_total","XPU memory total size",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","memory_free","XPU memory free size",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","memory_used","XPU memory used size",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","l3_memory_total","XPU L3 memory total size",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","l3_memory_free","XPU L3 memory free size",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","l3_memory_used","XPU L3 memory used size",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","utilization_gpu","XPU utilization","%"
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","power_draw","XPU power draw",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","power_limit","XPU power limit",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","clocks_current_cluster","XPU current cluster clocks, MHz",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","clocks_current_cdnn","XPU current CDNN clocks, MHz",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","pcie_link_gen_current","XPU current PCIe link generation",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","pcie_link_width_current","XPU current PCIe link width",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_correctable","XPU correctable DRAM ECC errors",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_uncorrectable","XPU uncorrectable DRAM ECC errors",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_correctable_aggregate","XPU correctable DRAM ECC errors in total",""
+"agent_xpusmi","Kunlunxin XPU metrics","agent","telegraf","ecc_errors_dram_uncorrectable_aggregate","XPU uncorrectable DRAM ECC errors in total",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","temperature_gpu","GPU temperature",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","memory_total","GPU memory total size",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","memory_free","GPU memory free size",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","memory_used","GPU memory used size",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_gpu","GPU utilization","%"
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_memory","GPU memory utilization","%"
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_encoder","GPU encoder utilization","%"
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","utilization_decoder","GPU decoder utilization","%"
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","power_draw","GPU power draw",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","power_limit","GPU power limit",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","clocks_current_graphics","GPU current clocks, MHz",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","pcie_link_gen_current","GPU PCIe link generation",""
+"mthreads_gmi","Moore Threads GPU metrics","host","telegraf","pcie_link_width_current","GPU PCIe link width",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","temperature_gpu","GPU temperature",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","memory_total","GPU memory total size",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","memory_free","GPU memory free size",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","memory_used","GPU memory used size",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_gpu","GPU utilization","%"
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_memory","GPU memory utilization","%"
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_encoder","GPU encoder utilization","%"
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","utilization_decoder","GPU decoder utilization","%"
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","power_draw","GPU power draw",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","power_limit","GPU power limit",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","clocks_current_graphics","GPU current clocks, MHz",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","pcie_link_gen_current","GPU PCIe link generation",""
+"agent_mthreads_gmi","Moore Threads GPU metrics","agent","telegraf","pcie_link_width_current","GPU PCIe link width",""
 "worker","Worker queue","system","system","active_worker_cnt","Active Worker Count","NULL"
 "worker","Worker queue","system","system","max_worker_count","Max Worker Count","NULL"
 "worker","Worker queue","system","system","detach_worker_cnt","Detach worker Count","NULL"
@@ -613,6 +714,7 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "pod_cpu","Pod cpu","container","telegraf","usage_rate","Pod cpu usage rate","%"
 "pod_gpu","Pod gpu","container","telegraf","mem","Pod gpu mem used","Mb"
 "pod_gpu","Pod gpu","container","telegraf","mem_util","Pod gpu mem usage","%"
+"pod_gpu","Pod gpu","container","telegraf","gpu_util","Pod gpu utilization","%"
 "pod_mem","Pod memory","container","telegraf","usage_rate","Pod memory usage rate","%"
 "pod_mem","Pod memory","container","telegraf","working_set_rate","Container memory working set usage rate","%"
 "pod_mem","Pod memory","container","telegraf","usage_bytes","Container memory usage bytes","byte"
@@ -0,0 +1,65 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var mthreadsGmi = SMeasurement{
+	Context: newHostAgentMonitorContext("mthreads_gmi", "Moore Threads GPU metrics"),
+	Metrics: []SMetric{
+		{
+			"temperature_gpu", "GPU temperature", "",
+		},
+		{
+			"memory_total", "GPU memory total size", "",
+		},
+		{
+			"memory_free", "GPU memory free size", "",
+		},
+		{
+			"memory_used", "GPU memory used size", "",
+		},
+		{
+			"utilization_gpu", "GPU utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"utilization_memory", "GPU memory utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"utilization_encoder", "GPU encoder utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"utilization_decoder", "GPU decoder utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"power_draw", "GPU power draw", "",
+		},
+		{
+			"power_limit", "GPU power limit", "",
+		},
+		{
+			"clocks_current_graphics", "GPU current clocks, MHz", "",
+		},
+		{
+			"clocks_current_memory", "GPU current memory clocks, MHz", "",
+		},
+		{
+			"pcie_link_gen_current", "GPU PCIe link generation", "",
+		},
+		{
+			"pcie_link_width_current", "GPU PCIe link width", "",
+		},
+	},
+}
@@ -59,6 +59,11 @@ var podGpu = SMeasurement{
 			DisplayName: "Pod gpu mem usage",
 			Unit:        monitor.METRIC_UNIT_PERCENT,
 		},
+		{
+			Name:        "gpu_util",
+			DisplayName: "Pod gpu utilization",
+			Unit:        monitor.METRIC_UNIT_PERCENT,
+		},
 	},
 }