feat(host): container nvidia gpu metrics (#21950)

* feat(host): container nvidia gpu metrics * feat(host): add vastaitech gpu metrics support * feat(host): add cph amd gpu metrics * fix(host): add nvidia gpu no card envs * fix(host): add nvidia gpu telegraf conf on container host
2026-09-19 02:37:24 +08:00 · 2025-01-26 12:52:28 +08:00
parent 475f110256
commit a4377527c5
13 changed files with 1055 additions and 51 deletions
@@ -134,6 +134,8 @@ type PodInstance interface {

 	IsInternalStopped(ctrCriId string) (*ContainerExpectedStatus, bool)
 	IsInternalRemoved(ctrCriId string) bool
+
+	GetPodContainerCriIds() []string
 }

 type sContainer struct {
@@ -588,6 +590,24 @@ func (s *sPodGuestInstance) GetContainers() []*hostapi.ContainerDesc {
 	return s.GetDesc().Containers
 }

+func (s *sPodGuestInstance) GetPodContainerCriIds() []string {
+	criids := make([]string, 0)
+	for i := range s.containers {
+		criids = append(criids, s.containers[i].CRIId)
+	}
+	return criids
+}
+
+func (s *sPodGuestInstance) HasContainerNvidiaGpu() bool {
+	for i := range s.Desc.IsolatedDevices {
+		if s.Desc.IsolatedDevices[i].DevType == computeapi.CONTAINER_DEV_NVIDIA_MPS ||
+			s.Desc.IsolatedDevices[i].DevType == computeapi.CONTAINER_DEV_NVIDIA_GPU {
+			return true
+		}
+	}
+	return false
+}
+
 func (s *sPodGuestInstance) GetContainerById(ctrId string) *hostapi.ContainerDesc {
 	ctrs := s.GetContainers()
 	for i := range ctrs {
@@ -1704,7 +1724,12 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 		if err := s.getIsolatedDeviceExtraConfig(spec, ctrCfg); err != nil {
 			return "", err
 		}
+	} else {
+		if hostinfo.Instance().HasContainerNvidiaGpu() {
+			ctrCfg.Envs = append(ctrCfg.Envs, &runtimeapi.KeyValue{Key: "NVIDIA_VISIBLE_DEVICES", Value: "void"})
+		}
 	}
+
 	if len(spec.Command) != 0 {
 		ctrCfg.Command = spec.Command
 	}
@@ -22,7 +22,9 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"

+	apis "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/util/pod"
 	"yunion.io/x/onecloud/pkg/util/pod/cadvisor"
@@ -76,3 +78,70 @@ func (h *SHostInfo) GetContainerCPUMap() *pod.HostContainerCPUMap {
 func (h *SHostInfo) GetContainerStatsProvider() stats.ContainerStatsProvider {
 	return h.containerStatsProvider
 }
+
+type INvidiaGpuIndexMemoryInterface interface {
+	GetNvidiaDevMemSize() int
+	GetNvidiaDevIndex() string
+}
+
+func (h *SHostInfo) GetNvidiaGpuIndexMemoryMap() map[string]int {
+	res := map[string]int{}
+	for i := range h.containerNvidiaGpus {
+		iDev, ok := h.containerNvidiaGpus[i].(INvidiaGpuIndexMemoryInterface)
+		if !ok {
+			continue
+		}
+		index := iDev.GetNvidiaDevIndex()
+		memSize := iDev.GetNvidiaDevMemSize()
+		res[index] = memSize
+	}
+	return res
+}
+
+func (h *SHostInfo) HasContainerVastaitechGpu() bool {
+	if h.hasVastaitechGpus != nil {
+		return *h.hasVastaitechGpus
+	}
+	hasVastaitechGpus := false
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_VASTAITECH_GPU {
+			hasVastaitechGpus = true
+		}
+	}
+	h.hasVastaitechGpus = &hasVastaitechGpus
+	return *h.hasVastaitechGpus
+}
+
+func (h *SHostInfo) HasContainerCphAmdGpu() bool {
+	if h.hasCphAmdGpus != nil {
+		return *h.hasCphAmdGpus
+	}
+	hasCphAmdGpus := false
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_CPH_AMD_GPU {
+			hasCphAmdGpus = true
+		}
+	}
+	h.hasCphAmdGpus = &hasCphAmdGpus
+	return *h.hasCphAmdGpus
+}
+
+func (h *SHostInfo) HasContainerNvidiaGpu() bool {
+	if h.hasNvidiaGpus != nil {
+		return *h.hasNvidiaGpus
+	}
+	hasNvidiaGpus := false
+	nvDevs := make([]isolated_device.IDevice, 0)
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_NVIDIA_GPU || devs[i].GetDeviceType() == apis.CONTAINER_DEV_NVIDIA_MPS {
+			hasNvidiaGpus = true
+			nvDevs = append(nvDevs, devs[i])
+		}
+	}
+	h.hasNvidiaGpus = &hasNvidiaGpus
+	h.containerNvidiaGpus = nvDevs
+	return *h.hasNvidiaGpus
+}
@@ -129,6 +129,10 @@ type SHostInfo struct {
 	containerCPUMap                *pod.HostContainerCPUMap
 	containerStatsProvider         stats.ContainerStatsProvider
 	containerCpufreqSimulateConfig *jsonutils.JSONDict
+	containerNvidiaGpus            []isolated_device.IDevice
+	hasNvidiaGpus                  *bool
+	hasVastaitechGpus              *bool
+	hasCphAmdGpus                  *bool
 }

 func (h *SHostInfo) GetContainerDeviceConfigurationFilePath() string {
@@ -2552,6 +2556,7 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 	// group dev
 	hasNetint := false
 	hasVasmi := false
+	hasNvidiasmi := false
 	for _, dev := range devs {
 		devType := dev.GetDeviceType()
 		switch devType {
@@ -2575,18 +2580,23 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 		case string(isolated_device.ContainerDeviceTypeVastaitechGpu):
 			hasVasmi = true
 			continue
+		case string(isolated_device.ContainerDeviceTypeNvidiaGpu), string(isolated_device.ContainerDeviceTypeNvidiaMps):
+			hasNvidiasmi = true
 		}
 	}
 	if hasNetint {
-		conf[system_service.TELEGAF_INPUT_NETDEV] = map[string]interface{}{
+		conf[system_service.TELEGRAF_INPUT_NETDEV] = map[string]interface{}{
 			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: "/usr/bin/ni_rsrc_mon",
 		}
 	}
 	if hasVasmi {
-		conf[system_service.TELEGAF_INPUT_VASMI] = map[string]interface{}{
+		conf[system_service.TELEGRAF_INPUT_VASMI] = map[string]interface{}{
 			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: "/usr/bin/vasmi",
 		}
 	}
+	if hasNvidiasmi {
+		conf[system_service.TELEGRAF_INPUT_NVIDIASMI] = struct{}{}
+	}
 }

 func (h *SHostInfo) getNicsTelegrafConf() []map[string]interface{} {
@@ -0,0 +1,120 @@
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+)
+
+type CphAmdGpuProcessMetrics struct {
+	Pid   string // Process ID
+	DevId string
+	Mem   float64 // Memory Utilization
+}
+
+/*
+pid  2088269 command allocator@2.0-s:
+        0x00000001:         4096 byte  GTT CPU_ACCESS_REQUIRED
+        0x00000002:      2097152 byte  GTT CPU_ACCESS_REQUIRED
+        0x00000003:      2097152 byte VRAM VRAM_CLEARED
+        0x00000004:      2097152 byte VRAM NO_CPU_ACCESS VRAM_CLEARED
+        0x00000006:      2097152 byte  GTT CPU_ACCESS_REQUIRED VRAM_CLEARED
+        0x00000007:      2097152 byte  GTT CPU_ACCESS_REQUIRED VRAM_CLEARED
+*/
+
+func GetCphAmdGpuProcessMetrics() ([]CphAmdGpuProcessMetrics, error) {
+	debugDriDir := "/sys/kernel/debug/dri"
+	entrys, err := os.ReadDir(debugDriDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "os.ReadDir")
+	}
+
+	res := make([]CphAmdGpuProcessMetrics, 0)
+	for i := range entrys {
+		if entrys[i].IsDir() {
+			fpath := path.Join(debugDriDir, entrys[i].Name(), "amdgpu_gem_info")
+			if fileutils2.Exists(fpath) {
+				content, err := fileutils2.FileGetContents(fpath)
+				if err != nil {
+					log.Errorf("failed FileGetContents %s: %s", fpath, err)
+					continue
+				}
+				metrics := parseCphAmdGpuGemInfo(content, entrys[i].Name())
+				if len(metrics) > 0 {
+					res = append(res, metrics...)
+				}
+			}
+		}
+	}
+	return res, nil
+}
+
+func parseCphAmdGpuGemInfo(content string, devId string) []CphAmdGpuProcessMetrics {
+	res := make([]CphAmdGpuProcessMetrics, 0)
+	lines := strings.Split(content, "\n")
+	var i, length = 0, len(lines)
+	for i < length {
+		line := strings.TrimSpace(lines[i])
+		segs := strings.Fields(line)
+		if len(segs) < 2 {
+			i++
+			continue
+		}
+		if segs[0] != "pid" {
+			i++
+			continue
+		}
+		pid := segs[1]
+		var vramTotal int64 = 0
+		j := i + 1
+		for j < length {
+			line := strings.TrimSpace(lines[j])
+			if len(line) == 0 {
+				break
+			}
+			segs := strings.Fields(line)
+			if len(segs) < 4 {
+				log.Errorf("unknown output line %s", line)
+				break
+			}
+			if segs[0] == "pid" {
+				break
+			}
+			memUsedStr, memType := segs[1], segs[3]
+			if memType == "VRAM" {
+				memUsed, err := strconv.ParseInt(memUsedStr, 10, 64)
+				if err != nil {
+					log.Errorf("failed parse memused %s %s: %s", line, memUsedStr, err)
+					break
+				}
+				vramTotal += memUsed
+			}
+			j++
+		}
+		res = append(res, CphAmdGpuProcessMetrics{
+			Pid:   pid,
+			DevId: devId,
+			Mem:   float64(vramTotal) / 1024.0 / 1024.0,
+		})
+		i = j
+	}
+
+	return res
+}
@@ -16,12 +16,16 @@ package hostmetrics

 import (
 	"fmt"
+	"sort"
+	"strconv"
 	"strings"
 	"time"

 	"yunion.io/x/log"

+	apis "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/guestman"
+	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/util/pod/stats"
 )

@@ -49,6 +53,28 @@ const (
 	SOCKET_COUNT    = "socket_count"
 	THREADS_CURRENT = "threads_current"
 	THREADS_MAX     = "threads_max"
+
+	NVIDIA_GPU_MEMORY_TOTAL   = "memory_total"
+	NVIDIA_GPU_INDEX          = "index"
+	NVIDIA_GPU_PHYSICAL_INDEX = "physical_index"
+	NVIDIA_GPU_FRAME_BUFFER   = "frame_buffer"
+	NVIDIA_GPU_CCPM           = "ccpm"
+	NVIDIA_GPU_SM             = "sm"
+	NVIDIA_GPU_MEM_UTIL       = "mem_util"
+	NVIDIA_GPU_ENC            = "enc"
+	NVIDIA_GPU_DEC            = "dec"
+	NVIDIA_GPU_JPG            = "jpg"
+	NVIDIA_GPU_OFA            = "ofa"
+
+	VASTAITECH_GPU_DEV_ID   = "dev_id"
+	VASTAITECH_GPU_ENC      = "enc"
+	VASTAITECH_GPU_DEC      = "dec"
+	VASTAITECH_GPU_GFX      = "gfx"
+	VASTAITECH_GPU_MEM      = "mem"
+	VASTAITECH_GPU_MEM_UTIL = "mem_util"
+
+	CPH_AMD_GPU_DEV_ID = "dev_id"
+	CPH_AMD_GPU_MEM    = "mem"
 )

 type CadvisorProcessMetric struct {
@@ -75,12 +101,15 @@ func (m CadvisorProcessMetric) ToMap() map[string]interface{} {
 }

 type PodMetrics struct {
-	PodCpu     *PodCpuMetric       `json:"pod_cpu"`
-	PodMemory  *PodMemoryMetric    `json:"pod_memory"`
-	PodProcess *PodProcessMetric   `json:"pod_process"`
-	PodVolumes []*PodVolumeMetric  `json:"pod_volume"`
-	PodDiskIos PodDiskIoMetrics    `json:"pod_disk_ios"`
-	Containers []*ContainerMetrics `json:"containers"`
+	PodCpu           *PodCpuMetric              `json:"pod_cpu"`
+	PodMemory        *PodMemoryMetric           `json:"pod_memory"`
+	PodProcess       *PodProcessMetric          `json:"pod_process"`
+	PodVolumes       []*PodVolumeMetric         `json:"pod_volume"`
+	PodDiskIos       PodDiskIoMetrics           `json:"pod_disk_ios"`
+	PodNvidiaGpu     []*PodNvidiaGpuMetrics     `json:"pod_nvidia_gpu"`
+	PodVastaitechGpu []*PodVastaitechGpuMetrics `json:"pod_vastaitech_gpu"`
+	PodCphAmdGpu     []*PodCphAmdGpuMetrics     `json:"pod_cph_amd_gpu"`
+	Containers       []*ContainerMetrics        `json:"containers"`
 }

 type PodMetricMeta struct {
@@ -95,6 +124,132 @@ func (m PodMetricMeta) GetTag() map[string]string {
 	return nil
 }

+type PodCphAmdGpuMetrics struct {
+	PodMetricMeta
+
+	DevId string
+	Mem   float64 // MB
+}
+
+func (m PodCphAmdGpuMetrics) GetName() string {
+	return "pod_cph_amd_gpu"
+}
+
+func (m PodCphAmdGpuMetrics) GetUniformName() string {
+	return "pod_gpu"
+}
+
+func (m PodCphAmdGpuMetrics) GetTag() map[string]string {
+	return map[string]string{
+		"dev_id":   m.DevId,
+		"dev_type": apis.CONTAINER_DEV_CPH_AMD_GPU,
+	}
+}
+
+func (m PodCphAmdGpuMetrics) ToMap() map[string]interface{} {
+	ret := map[string]interface{}{
+		CPH_AMD_GPU_DEV_ID: m.DevId,
+		CPH_AMD_GPU_MEM:    m.Mem,
+	}
+	return ret
+}
+
+type PodVastaitechGpuMetrics struct {
+	PodMetricMeta
+
+	PciAddr string
+	DevId   string
+
+	Mem     float64 // MB
+	MemUtil float64
+	Gfx     float64
+	DecUtil float64
+	EncUtil float64
+}
+
+func (m PodVastaitechGpuMetrics) GetName() string {
+	return "pod_vastaitech_gpu"
+}
+
+func (m PodVastaitechGpuMetrics) GetUniformName() string {
+	return "pod_gpu"
+}
+
+func (m PodVastaitechGpuMetrics) GetTag() map[string]string {
+	return map[string]string{
+		"dev_id":   m.DevId,
+		"dev_type": apis.CONTAINER_DEV_VASTAITECH_GPU,
+	}
+}
+
+func (m PodVastaitechGpuMetrics) ToMap() map[string]interface{} {
+	ret := map[string]interface{}{
+		VASTAITECH_GPU_DEC:      m.DecUtil,
+		VASTAITECH_GPU_DEV_ID:   m.DevId,
+		VASTAITECH_GPU_ENC:      m.EncUtil,
+		VASTAITECH_GPU_GFX:      m.Gfx,
+		VASTAITECH_GPU_MEM:      m.Mem,
+		VASTAITECH_GPU_MEM_UTIL: m.MemUtil,
+	}
+	return ret
+}
+
+type PodNvidiaGpuMetrics struct {
+	PodMetricMeta
+
+	Index         int
+	PhysicalIndex int
+	MemTotal      int
+
+	Framebuffer int     // Framebuffer Memory Usage
+	Ccpm        int     // Current CUDA Contexts Per Measurement
+	SmUtil      float64 // Streaming Multiprocessor Utilization
+	Mem         int     // Mem Usage
+	MemUtil     float64 // Memory Utilization
+	EncUtil     float64 // Encoder Utilization
+	DecUtil     float64 // Decoder Utilization
+	JpgUtil     float64 // JPEG Decoder Utilization
+	OfaUtil     float64 // Other Feature Utilization
+}
+
+func (m PodNvidiaGpuMetrics) GetName() string {
+	return "pod_nvidia_gpu"
+}
+
+func (m PodNvidiaGpuMetrics) GetUniformName() string {
+	return "pod_gpu"
+}
+
+func (m PodNvidiaGpuMetrics) GetTag() map[string]string {
+	devType := apis.CONTAINER_DEV_NVIDIA_GPU
+	if options.HostOptions.EnableCudaMPS {
+		devType = apis.CONTAINER_DEV_NVIDIA_MPS
+	}
+	return map[string]string{
+		"index":          strconv.Itoa(m.Index),
+		"physical_index": strconv.Itoa(m.PhysicalIndex),
+		"dev_type":       devType,
+	}
+}
+
+func (m PodNvidiaGpuMetrics) ToMap() map[string]interface{} {
+	ret := map[string]interface{}{
+		NVIDIA_GPU_MEMORY_TOTAL:   m.MemTotal,
+		NVIDIA_GPU_INDEX:          m.Index,
+		NVIDIA_GPU_PHYSICAL_INDEX: m.PhysicalIndex,
+		NVIDIA_GPU_FRAME_BUFFER:   m.Framebuffer,
+		NVIDIA_GPU_CCPM:           m.Ccpm,
+		NVIDIA_GPU_SM:             m.SmUtil,
+		NVIDIA_GPU_MEM_UTIL:       m.MemUtil,
+		NVIDIA_GPU_ENC:            m.EncUtil,
+		NVIDIA_GPU_DEC:            m.DecUtil,
+		NVIDIA_GPU_JPG:            m.JpgUtil,
+		NVIDIA_GPU_OFA:            m.OfaUtil,
+	}
+
+	return ret
+}
+
 type PodCpuMetric struct {
 	PodMetricMeta
 	CpuUsageSecondsTotal float64  `json:"cpu_usage_seconds_total"`
@@ -311,7 +466,7 @@ type ContainerMetricMeta struct {
 func (m ContainerMetricMeta) GetTag() map[string]string {
 	ret := map[string]string{
 		"pod_id":         m.PodId,
-		"container_name": m.ContainerName,
+		"container_name": strings.ReplaceAll(m.ContainerName, " ", "+"),
 	}
 	if m.ContainerId != "" {
 		ret["container_id"] = m.ContainerId
@@ -382,18 +537,54 @@ func (m *ContainerDiskIoMetric) GetTag() map[string]string {
 	return baseTags
 }

-func GetPodStatsById(stats []stats.PodStats, podId string) *stats.PodStats {
-	for _, stat := range stats {
-		if stat.PodRef.UID == podId {
-			tmp := stat
-			return &tmp
+func GetPodStatsById(ss []stats.PodStats, gpuPodProcs map[string]map[string]struct{}, podId string) (*stats.PodStats, map[string]struct{}) {
+	var podStat *stats.PodStats
+	for i := range ss {
+		if ss[i].PodRef.UID == podId {
+			podStat = &ss[i]
+			break
 		}
 	}
-	return nil
+	podProcs, _ := gpuPodProcs[podId]
+	return podStat, podProcs
+}
+
+func GetPodNvidiaGpuMetrics(metrics []NvidiaGpuProcessMetrics, podProcs map[string]struct{}) []NvidiaGpuProcessMetrics {
+	podMetrics := make([]NvidiaGpuProcessMetrics, 0)
+	for i := range metrics {
+		pid := metrics[i].Pid
+		if _, ok := podProcs[pid]; ok {
+			podMetrics = append(podMetrics, metrics[i])
+		}
+	}
+	return podMetrics
+}
+
+func GetPodVastaitechGpuMetrics(metrics []VastaitechGpuProcessMetrics, podProcs map[string]struct{}) []VastaitechGpuProcessMetrics {
+	podMetrics := make([]VastaitechGpuProcessMetrics, 0)
+	for i := range metrics {
+		pid := metrics[i].Pid
+		if _, ok := podProcs[pid]; ok {
+			podMetrics = append(podMetrics, metrics[i])
+		}
+	}
+	return podMetrics
+}
+
+func GetPodCphAmdGpuMetrics(metrics []CphAmdGpuProcessMetrics, podProcs map[string]struct{}) []CphAmdGpuProcessMetrics {
+	podMetrics := make([]CphAmdGpuProcessMetrics, 0)
+	for i := range metrics {
+		pid := metrics[i].Pid
+		if _, ok := podProcs[pid]; ok {
+			podMetrics = append(podMetrics, metrics[i])
+		}
+	}
+	return podMetrics
 }

 func (s *SGuestMonitorCollector) collectPodMetrics(gm *SGuestMonitor, prevUsage *GuestMetrics) *GuestMetrics {
 	gmData := new(GuestMetrics)
+	s.hostInfo.GetContainerStatsProvider()
 	gmData.PodMetrics = gm.PodMetrics(prevUsage)

 	// netio
@@ -564,11 +755,14 @@ func (m *SGuestMonitor) PodMetrics(prevUsage *GuestMetrics) *PodMetrics {
 	}

 	pm := &PodMetrics{
-		PodCpu:     podCpu,
-		PodMemory:  podMemory,
-		PodProcess: podProcess,
-		PodVolumes: m.getVolumeMetrics(),
-		Containers: containers,
+		PodCpu:           podCpu,
+		PodMemory:        podMemory,
+		PodProcess:       podProcess,
+		PodVolumes:       m.getVolumeMetrics(),
+		PodNvidiaGpu:     m.getPodNvidiaGpuMetrics(),
+		PodVastaitechGpu: m.getPodVastaitechGpuMetrics(),
+		PodCphAmdGpu:     m.getPodCphAmdGpuMetrics(),
+		Containers:       containers,
 	}

 	if stat.DiskIo != nil {
@@ -587,12 +781,111 @@ func (m *SGuestMonitor) PodMetrics(prevUsage *GuestMetrics) *PodMetrics {
 	return pm
 }

+func (m *SGuestMonitor) getPodCphAmdGpuMetrics() []*PodCphAmdGpuMetrics {
+	if len(m.cphAmdGpuMetrics) == 0 {
+		return nil
+	}
+	addrGpuMap := map[string]*PodCphAmdGpuMetrics{}
+	for i := range m.cphAmdGpuMetrics {
+		devId := m.cphAmdGpuMetrics[i].DevId
+		gms, ok := addrGpuMap[devId]
+		if !ok {
+			gms = new(PodCphAmdGpuMetrics)
+			gms.DevId = devId
+		}
+		gms.Mem += m.cphAmdGpuMetrics[i].Mem
+		addrGpuMap[devId] = gms
+	}
+	res := make([]*PodCphAmdGpuMetrics, 0)
+	for _, gms := range addrGpuMap {
+		res = append(res, gms)
+	}
+
+	return res
+}
+
+func (m *SGuestMonitor) getPodVastaitechGpuMetrics() []*PodVastaitechGpuMetrics {
+	if len(m.vastaitechGpuMetrics) == 0 {
+		return nil
+	}
+	addrGpuMap := map[string]*PodVastaitechGpuMetrics{}
+	for i := range m.vastaitechGpuMetrics {
+		pciAddr := m.vastaitechGpuMetrics[i].PciAddr
+		gms, ok := addrGpuMap[pciAddr]
+		if !ok {
+			gms = new(PodVastaitechGpuMetrics)
+			gms.DevId = m.vastaitechGpuMetrics[i].DevId
+			gms.PciAddr = m.vastaitechGpuMetrics[i].PciAddr
+		}
+		gms.Mem += m.vastaitechGpuMetrics[i].GfxMem
+		gms.MemUtil += m.vastaitechGpuMetrics[i].GfxMemUsage
+		gms.Gfx += m.vastaitechGpuMetrics[i].Gfx
+		gms.DecUtil += m.vastaitechGpuMetrics[i].Dec
+		gms.EncUtil += m.vastaitechGpuMetrics[i].Enc
+		addrGpuMap[pciAddr] = gms
+	}
+	res := make([]*PodVastaitechGpuMetrics, 0)
+	for _, gms := range addrGpuMap {
+		res = append(res, gms)
+	}
+
+	return res
+}
+
+func (m *SGuestMonitor) getPodNvidiaGpuMetrics() []*PodNvidiaGpuMetrics {
+	if len(m.nvidiaGpuMetrics) == 0 {
+		return nil
+	}
+	indexGpuMap := map[int]*PodNvidiaGpuMetrics{}
+	for i := range m.nvidiaGpuMetrics {
+		index := m.nvidiaGpuMetrics[i].Index
+		gms, ok := indexGpuMap[index]
+		if !ok {
+			gms = new(PodNvidiaGpuMetrics)
+		}
+		gms.Framebuffer += m.nvidiaGpuMetrics[i].FB
+		gms.Ccpm += m.nvidiaGpuMetrics[i].Ccpm
+		gms.SmUtil += m.nvidiaGpuMetrics[i].Sm
+		gms.EncUtil += m.nvidiaGpuMetrics[i].Enc
+		gms.DecUtil += m.nvidiaGpuMetrics[i].Dec
+		gms.JpgUtil += m.nvidiaGpuMetrics[i].Jpg
+		gms.OfaUtil += m.nvidiaGpuMetrics[i].Ofa
+		indexGpuMap[index] = gms
+	}
+
+	indexs := make([]int, 0)
+	for index, gms := range indexGpuMap {
+		indexs = append(indexs, index)
+		indexStr := strconv.Itoa(index)
+		memSizeTotal, ok := m.nvidiaGpuIndexMemoryMap[indexStr]
+		if !ok {
+			continue
+		}
+		gms.MemTotal = memSizeTotal
+		gms.Mem = gms.Framebuffer
+		gms.MemUtil = float64(gms.Framebuffer) / float64(gms.MemTotal)
+	}
+	sort.Ints(indexs)
+	res := make([]*PodNvidiaGpuMetrics, len(indexs))
+	for i := range indexs {
+		gms := indexGpuMap[indexs[i]]
+		gms.PhysicalIndex = gms.Index
+		gms.Index = i
+		res[i] = gms
+	}
+	return res
+}
+
 type iPodMetric interface {
 	GetName() string
 	GetTag() map[string]string
 	ToMap() map[string]interface{}
 }

+type iPodUniformName interface {
+	GetUniformName() string
+}
+
 func (d *GuestMetrics) toPodTelegrafData(tagStr string) []string {
 	m := d.PodMetrics
 	ims := []iPodMetric{m.PodCpu, m.PodMemory}
@@ -607,6 +900,16 @@ func (d *GuestMetrics) toPodTelegrafData(tagStr string) []string {
 			ims = append(ims, d)
 		}
 	}
+	for i := range m.PodNvidiaGpu {
+		ims = append(ims, m.PodNvidiaGpu[i])
+	}
+	for i := range m.PodVastaitechGpu {
+		ims = append(ims, m.PodVastaitechGpu[i])
+	}
+	for i := range m.PodCphAmdGpu {
+		ims = append(ims, m.PodCphAmdGpu[i])
+	}
+
 	for _, c := range m.Containers {
 		ims = append(ims, c.ContainerCpu)
 		ims = append(ims, c.ContainerMemory)
@@ -629,6 +932,11 @@ func (d *GuestMetrics) toPodTelegrafData(tagStr string) []string {
 			newTagStr = strings.Join([]string{tagStr, strings.Join(newTagArr, ",")}, ",")
 		}
 		res = append(res, fmt.Sprintf("%s,%s %s", im.GetName(), newTagStr, d.mapToStatStr(im.ToMap())))
+		if imu, ok := im.(iPodUniformName); ok {
+			if un := imu.GetUniformName(); un != "" {
+				res = append(res, fmt.Sprintf("%s,%s %s", un, newTagStr, d.mapToStatStr(im.ToMap())))
+			}
+		}
 	}
 	res = append(res, d.netioToTelegrafData("pod_netio", tagStr)...)
 	return res
@@ -0,0 +1,200 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/guestman"
+	"yunion.io/x/onecloud/pkg/util/cgrouputils"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type NvidiaGpuProcessMetrics struct {
+	Index   int     // Gpu Index
+	Pid     string  // Process ID
+	Type    string  // Process Type C/G, Compute or Graphics
+	FB      int     // Framebuffer Memory Usage
+	Ccpm    int     // Current CUDA Contexts Per Measurement
+	Sm      float64 // Streaming Multiprocessor Utilization
+	Mem     float64 // Memory Utilization
+	Enc     float64 // Encoder Utilization
+	Dec     float64 // Decoder Utilization
+	Jpg     float64 // JPEG Decoder Utilization
+	Ofa     float64 // Other Feature Utilization
+	Command string  // Process Command Name
+}
+
+func GetNvidiaGpuProcessMetrics() ([]NvidiaGpuProcessMetrics, error) {
+	cmd := "nvidia-smi pmon -s mu -c 1"
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "Execute %s failed: %s", cmd, output)
+	}
+	return parseNvidiaGpuProcessMetrics(string(output)), nil
+}
+
+/*
+# gpu         pid   type     fb   ccpm     sm    mem    enc    dec    jpg    ofa    command
+# Idx           #    C/G     MB     MB      %      %      %      %      %      %    name
+*/
+func parseNvidiaGpuProcessMetrics(gpuMetricsStr string) []NvidiaGpuProcessMetrics {
+	gpuProcessMetrics := make([]NvidiaGpuProcessMetrics, 0)
+
+	lines := strings.Split(gpuMetricsStr, "\n")
+	for _, line := range lines {
+
+		// Skip comments and blank lines
+		if strings.HasPrefix(line, "#") || len(strings.TrimSpace(line)) == 0 {
+			continue
+		}
+
+		var processMetrics NvidiaGpuProcessMetrics
+		var fb, ccpm, sm, mem, enc, dec, jpg, ofa string
+		_, err := fmt.Sscanf(line, "%d %s %s %s %s %s %s %s %s %s %s %s",
+			&processMetrics.Index, &processMetrics.Pid, &processMetrics.Type, &fb, &ccpm,
+			&sm, &mem, &enc, &dec, &jpg, &ofa, &processMetrics.Command)
+		if err != nil {
+			log.Errorf("failed parse nvidia gpu metrics %s: %s", line, err)
+			continue
+		}
+		if processMetrics.Command == "nvidia-cuda-mps" || processMetrics.Command == "-" {
+			continue
+		}
+		if fb != "-" {
+			val, err := strconv.Atoi(fb)
+			if err != nil {
+				log.Errorf("failed parse sm %s: %s", sm, err)
+			}
+			processMetrics.FB = val
+		}
+		if ccpm != "-" {
+			val, err := strconv.Atoi(ccpm)
+			if err != nil {
+				log.Errorf("failed parse sm %s: %s", sm, err)
+			}
+			processMetrics.Ccpm = val
+		}
+		if sm != "-" {
+			val, err := strconv.ParseFloat(sm, 64)
+			if err != nil {
+				log.Errorf("failed parse sm %s: %s", sm, err)
+			}
+			processMetrics.Sm = val
+		}
+		if mem != "-" {
+			val, err := strconv.ParseFloat(mem, 64)
+			if err != nil {
+				log.Errorf("failed parse mem %s: %s", mem, err)
+			}
+			processMetrics.Mem = val
+		}
+		if enc != "-" {
+			val, err := strconv.ParseFloat(enc, 64)
+			if err != nil {
+				log.Errorf("failed parse enc %s: %s", enc, err)
+			}
+			processMetrics.Enc = val
+		}
+		if dec != "-" {
+			val, err := strconv.ParseFloat(dec, 64)
+			if err != nil {
+				log.Errorf("failed parse dec %s: %s", dec, err)
+			}
+			processMetrics.Dec = val
+		}
+		if jpg != "-" {
+			val, err := strconv.ParseFloat(jpg, 64)
+			if err != nil {
+				log.Errorf("failed parse jpg %s: %s", jpg, err)
+			}
+			processMetrics.Jpg = val
+		}
+		if ofa != "-" {
+			val, err := strconv.ParseFloat(ofa, 64)
+			if err != nil {
+				log.Errorf("failed parse ofa %s: %s", ofa, err)
+			}
+			processMetrics.Ofa = val
+		}
+
+		gpuProcessMetrics = append(gpuProcessMetrics, processMetrics)
+	}
+	return gpuProcessMetrics
+}
+
+func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string]struct{} {
+	podProcIds := map[string]map[string]struct{}{}
+	guestmanager := guestman.GetGuestManager()
+	cgroupRoot := path.Join(cgrouputils.RootTaskPath("cpuset"), "cloudpods")
+	guestmanager.Servers.Range(func(k, v interface{}) bool {
+		pod, ok := v.(guestman.PodInstance)
+		if !ok {
+			return true
+		}
+		if !pod.IsRunning() {
+			return true
+		}
+		podDesc := pod.GetDesc()
+		hasGpu := false
+		for i := range podDesc.IsolatedDevices {
+			if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, []string{compute.CONTAINER_DEV_NVIDIA_GPU, compute.CONTAINER_DEV_NVIDIA_MPS, compute.CONTAINER_DEV_VASTAITECH_GPU, compute.CONTAINER_DEV_CPH_AMD_GPU}) {
+				hasGpu = true
+				break
+			}
+		}
+		if !hasGpu {
+			return true
+		}
+
+		criIds := pod.GetPodContainerCriIds()
+		procs := map[string]struct{}{}
+		for i := range criIds {
+			cgroupPath := path.Join(cgroupRoot, criIds[i], "cgroup.procs")
+			pids, err := ReadProccessFromCgroupProcs(cgroupPath)
+			if err != nil {
+				log.Errorf("collectNvidiaGpuPodsProcesses: %s", err)
+				continue
+			}
+			for _, pid := range pids {
+				procs[pid] = struct{}{}
+			}
+		}
+		if len(procs) > 0 {
+			podProcIds[pod.GetId()] = procs
+		}
+		return true
+	})
+	return podProcIds
+}
+
+func ReadProccessFromCgroupProcs(procFilePath string) ([]string, error) {
+	out, err := os.ReadFile(procFilePath)
+	if err != nil {
+		return nil, errors.Wrap(err, "os.ReadFile")
+	}
+
+	pids := strings.Split(string(out), "\n")
+	return pids, nil
+}
@@ -0,0 +1,147 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type VastaitechGpuProcessMetrics struct {
+	DevId       string
+	PciAddr     string // Device Pci Addr
+	Pid         string // Process ID
+	Enc         float64
+	Dec         float64
+	Gfx         float64
+	GfxMem      float64
+	GfxMemUsage float64
+}
+
+func GetVastaitechGpuProcessMetrics() ([]VastaitechGpuProcessMetrics, error) {
+	outputFile := "/tmp/vasmi_pmon.csv"
+	cmd := fmt.Sprintf("/usr/bin/vasmi pmon --outputformat=csv --outputfile=%s --loop 1", outputFile)
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "Execute %s failed: %s", cmd, out)
+	}
+	output, err := fileutils2.FileGetContents(outputFile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "FileGetContents %s failed", outputFile)
+	}
+
+	return parseVastaitechGpuProcessMetrics(output), nil
+}
+
+/*
+LoopTimes,AIC,DevId,PCIe_Bus_Id,PID,VPID,Command,Container_Name,Enc,Dec,Gfx,Gfx_Mem,Gfx_Mem_Usage,Reserved_Mem
+1,0,0,0000:18:00.0,3112054,388,android.hardware.graphics.allocator@2.0-service,va_androidxx,0,0,0,61.68MB,0.827308,0.00B
+*/
+
+func parseVastaitechGpuProcessMetrics(gpuMetricsStr string) []VastaitechGpuProcessMetrics {
+	gpuProcessMetrics := make([]VastaitechGpuProcessMetrics, 0)
+	lines := strings.Split(gpuMetricsStr, "\n")
+	for i := 1; i < len(lines); i++ {
+		segs := strings.Split(lines[i], ",")
+		segLens := len(segs)
+		if segLens < 14 {
+			continue
+		}
+
+		devId, pciAddr, pidStr := segs[2], segs[3], segs[4]
+		gfxMemUsageStr, gfxMemStr, gfxStr, decStr, encStr := segs[segLens-2], segs[segLens-3], segs[segLens-4], segs[segLens-5], segs[segLens-6]
+		gfxMemUsage, err := strconv.ParseFloat(gfxMemUsageStr, 64)
+		if err != nil {
+			log.Errorf("failed parse gfxMemUsageStr %s: %s", gfxMemUsageStr, err)
+		}
+		gfxMem, err := parseSizeToFloat64(gfxMemStr)
+		if err != nil {
+			log.Errorf("failed parse gfxMemStr %s: %s", gfxMemStr, err)
+			continue
+		}
+		pciAddr = strings.ReplaceAll(pciAddr, ":", "_")
+		gfxMem = gfxMem / 1024.0 / 1024.0
+		gfx, err := strconv.ParseFloat(gfxStr, 64)
+		if err != nil {
+			log.Errorf("failed parse gfxStr %s: %s", gfxStr, err)
+		}
+		dec, err := strconv.ParseFloat(decStr, 64)
+		if err != nil {
+			log.Errorf("failed parse decStr %s: %s", decStr, err)
+		}
+		enc, err := strconv.ParseFloat(encStr, 64)
+		if err != nil {
+			log.Errorf("failed parse encStr %s: %s", encStr, err)
+		}
+		procMetrics := VastaitechGpuProcessMetrics{
+			DevId:       devId,
+			PciAddr:     pciAddr,
+			Pid:         pidStr,
+			Gfx:         gfx,
+			GfxMem:      gfxMem,
+			GfxMemUsage: gfxMemUsage,
+			Enc:         enc,
+			Dec:         dec,
+		}
+		gpuProcessMetrics = append(gpuProcessMetrics, procMetrics)
+	}
+	return gpuProcessMetrics
+}
+
+// See: http://en.wikipedia.org/wiki/Binary_prefix
+const (
+	// Decimal
+
+	KB = 1000
+	MB = 1000 * KB
+	GB = 1000 * MB
+	TB = 1000 * GB
+	PB = 1000 * TB
+)
+
+type unitMap map[string]int64
+
+var (
+	decimalMap = unitMap{"k": KB, "m": MB, "g": GB, "t": TB, "p": PB}
+	sizeRegex  = regexp.MustCompile(`^(\d+(\.\d+)*) ?([kKmMgGtTpP])?[iI]?[bB]?$`)
+)
+
+// Parses the human-readable size string into the amount it represents.
+func parseSizeToFloat64(sizeStr string) (float64, error) {
+	matches := sizeRegex.FindStringSubmatch(sizeStr)
+	if len(matches) != 4 {
+		return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
+	}
+
+	size, err := strconv.ParseFloat(matches[1], 64)
+	if err != nil {
+		return -1, err
+	}
+
+	unitPrefix := strings.ToLower(matches[3])
+	if mul, ok := decimalMap[unitPrefix]; ok {
+		size *= float64(mul)
+	}
+
+	return size, nil
+}
@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/httputils"
 	"yunion.io/x/pkg/util/netutils"
+	"yunion.io/x/pkg/utils"

 	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/guestman"
@@ -58,6 +59,10 @@ var hostMetricsCollector *SHostMetricsCollector

 type IHostInfo interface {
 	GetContainerStatsProvider() stats.ContainerStatsProvider
+	HasContainerNvidiaGpu() bool
+	HasContainerVastaitechGpu() bool
+	HasContainerCphAmdGpu() bool
+	GetNvidiaGpuIndexMemoryMap() map[string]int
 }

 func Init(hostInfo IHostInfo) {
@@ -178,6 +183,10 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor {
 	guestmanager := guestman.GetGuestManager()

 	var podStats []stats.PodStats = nil
+	var nvidiaGpuMetrics []NvidiaGpuProcessMetrics = nil
+	var vastaitechGpuMetrics []VastaitechGpuProcessMetrics = nil
+	var cphAmdGpuMetrics []CphAmdGpuProcessMetrics = nil
+	var gpuPodProcs = s.collectGpuPodsProcesses()

 	guestmanager.Servers.Range(func(k, v interface{}) bool {
 		instance, ok := v.(guestman.GuestRuntimeInstance)
@@ -234,13 +243,37 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor {
 					log.Errorf("ListPodCPUAndMemoryStats: %s", err)
 					return true
 				}
+				if s.hostInfo.HasContainerNvidiaGpu() {
+					nvidiaGpuMetrics, err = GetNvidiaGpuProcessMetrics()
+					if err != nil {
+						log.Errorf("GetNvidiaGpuProcessMetrics %s", err)
+					}
+				}
+				if s.hostInfo.HasContainerVastaitechGpu() {
+					vastaitechGpuMetrics, err = GetVastaitechGpuProcessMetrics()
+					if err != nil {
+						log.Errorf("GetVastaitechGpuProcessMetrics %s", err)
+					}
+				}
+				if s.hostInfo.HasContainerCphAmdGpu() {
+					cphAmdGpuMetrics, err = GetCphAmdGpuProcessMetrics()
+					if err != nil {
+						log.Errorf("GetCphAmdGpuProcessMetrics %s", err)
+					}
+				}
 			}
-			podStat := GetPodStatsById(podStats, guestId)
+
+			podStat, podProcs := GetPodStatsById(podStats, gpuPodProcs, guestId)
 			if podStat != nil {
-				gm, err := NewGuestPodMonitor(instance, guestName, guestId, podStat, nicsDesc, int(vcpuCount))
+				gm, err := NewGuestPodMonitor(
+					instance, guestName, guestId, podStat,
+					nvidiaGpuMetrics, vastaitechGpuMetrics, cphAmdGpuMetrics,
+					s.hostInfo, podProcs, nicsDesc, int(vcpuCount),
+				)
 				if err != nil {
 					return true
 				}
+
 				gm.UpdateByInstance(instance)
 				gms[guestId] = gm
 				return true
@@ -548,22 +581,26 @@ func (s *SGuestMonitorCollector) reportNetIo(cur, prev *NetIOMetric) {
 }

 type SGuestMonitor struct {
-	Name           string
-	Id             string
-	Pid            int
-	Nics           []*desc.SGuestNetwork
-	CpuCnt         int
-	MemMB          int64
-	Ip             string
-	Process        *process.Process
-	ScalingGroupId string
-	Tenant         string
-	TenantId       string
-	DomainId       string
-	ProjectDomain  string
-	podStat        *stats.PodStats
-	instance       guestman.GuestRuntimeInstance
-	sysFs          sysfs.SysFs
+	Name                    string
+	Id                      string
+	Pid                     int
+	Nics                    []*desc.SGuestNetwork
+	CpuCnt                  int
+	MemMB                   int64
+	Ip                      string
+	Process                 *process.Process
+	ScalingGroupId          string
+	Tenant                  string
+	TenantId                string
+	DomainId                string
+	ProjectDomain           string
+	podStat                 *stats.PodStats
+	nvidiaGpuMetrics        []NvidiaGpuProcessMetrics
+	nvidiaGpuIndexMemoryMap map[string]int
+	vastaitechGpuMetrics    []VastaitechGpuProcessMetrics
+	cphAmdGpuMetrics        []CphAmdGpuProcessMetrics
+	instance                guestman.GuestRuntimeInstance
+	sysFs                   sysfs.SysFs
 }

 func NewGuestMonitor(instance guestman.GuestRuntimeInstance, name, id string, pid int, nics []*desc.SGuestNetwork, cpuCount int) (*SGuestMonitor, error) {
@@ -574,12 +611,41 @@ func NewGuestMonitor(instance guestman.GuestRuntimeInstance, name, id string, pi
 	return newGuestMonitor(instance, name, id, proc, nics, cpuCount)
 }

-func NewGuestPodMonitor(instance guestman.GuestRuntimeInstance, name, id string, stat *stats.PodStats, nics []*desc.SGuestNetwork, cpuCount int) (*SGuestMonitor, error) {
+func NewGuestPodMonitor(
+	instance guestman.GuestRuntimeInstance, name, id string, stat *stats.PodStats,
+	nvidiaGpuMetrics []NvidiaGpuProcessMetrics, vastaitechGpuMetrics []VastaitechGpuProcessMetrics, cphAmdGpuMetrics []CphAmdGpuProcessMetrics,
+	hostInstance IHostInfo, podProcs map[string]struct{}, nics []*desc.SGuestNetwork, cpuCount int,
+) (*SGuestMonitor, error) {
 	m, err := newGuestMonitor(instance, name, id, nil, nics, cpuCount)
 	if err != nil {
 		return nil, errors.Wrap(err, "new pod GuestMonitor")
 	}
 	m.podStat = stat
+	podDesc := instance.GetDesc()
+
+	hasNvGpu := false
+	hasCphAmdGpu := false
+	hasVastaitechGpu := false
+	for i := range podDesc.IsolatedDevices {
+		if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, []string{compute.CONTAINER_DEV_NVIDIA_MPS, compute.CONTAINER_DEV_NVIDIA_GPU}) {
+			hasNvGpu = true
+		} else if podDesc.IsolatedDevices[i].DevType == compute.CONTAINER_DEV_VASTAITECH_GPU {
+			hasVastaitechGpu = true
+		} else if podDesc.IsolatedDevices[i].DevType == compute.CONTAINER_DEV_CPH_AMD_GPU {
+			hasCphAmdGpu = true
+		}
+	}
+
+	if hasNvGpu {
+		m.nvidiaGpuMetrics = GetPodNvidiaGpuMetrics(nvidiaGpuMetrics, podProcs)
+		m.nvidiaGpuIndexMemoryMap = hostInstance.GetNvidiaGpuIndexMemoryMap()
+	}
+	if hasVastaitechGpu {
+		m.vastaitechGpuMetrics = GetPodVastaitechGpuMetrics(vastaitechGpuMetrics, podProcs)
+	}
+	if hasCphAmdGpu {
+		m.cphAmdGpuMetrics = GetPodCphAmdGpuMetrics(cphAmdGpuMetrics, podProcs)
+	}
 	return m, nil
 }

@@ -82,6 +82,17 @@ func (m *nvidiaGPUManager) GetContainerExtraConfigures(devs []*hostapi.Container

 type nvidiaGPU struct {
 	*BaseDevice
+
+	memSize  int
+	gpuIndex string
+}
+
+func (dev *nvidiaGPU) GetNvidiaDevMemSize() int {
+	return dev.memSize
+}
+
+func (dev *nvidiaGPU) GetNvidiaDevIndex() string {
+	return dev.gpuIndex
 }

 func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
@@ -91,7 +102,7 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
 	// GPU-bc1a3bb9-55cb-8c52-c374-4f8b4f388a20, NVIDIA A800-SXM4-80GB, 00000000:10:00.0

 	// nvidia-smi  --query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode --format=csv
-	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,compute_mode", "--format=csv").Output()
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,compute_mode,memory.total,index", "--format=csv").Output()
 	if err != nil {
 		return nil, errors.Wrap(err, "nvidia-smi")
 	}
@@ -101,15 +112,19 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
 			continue
 		}
 		segs := strings.Split(line, ",")
-		if len(segs) != 4 {
+		if len(segs) != 6 {
 			log.Errorf("unknown nvidia-smi out line %s", line)
 			continue
 		}
-		gpuId, gpuName, gpuPciAddr, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3])
+		gpuId, gpuName, gpuPciAddr, computeMode, memTotal, index := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3]), strings.TrimSpace(segs[4]), strings.TrimSpace(segs[5])
 		if computeMode != "Default" {
 			log.Warningf("gpu device %s compute mode %s, skip.", gpuId, computeMode)
 			continue
 		}
+		memSize, err := parseMemSize(memTotal)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed parse memSize %s", memTotal)
+		}

 		pciOutput, err := isolated_device.GetPCIStrByAddr(gpuPciAddr)
 		if err != nil {
@@ -118,6 +133,8 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
 		dev := isolated_device.NewPCIDevice2(pciOutput[0])
 		gpuDev := &nvidiaGPU{
 			BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaGpu, gpuId),
+			memSize:    memSize,
+			gpuIndex:   index,
 		}
 		gpuDev.SetModelName(gpuName)

@@ -113,6 +113,16 @@ type nvidiaMPS struct {
 	MemSizeMB        int
 	MemTotalMB       int
 	ThreadPercentage int
+
+	gpuIndex string
+}
+
+func (dev *nvidiaMPS) GetNvidiaDevMemSize() int {
+	return dev.MemSizeMB
+}
+
+func (dev *nvidiaMPS) GetNvidiaDevIndex() string {
+	return dev.gpuIndex
 }

 func (c *nvidiaMPS) GetNvidiaMpsMemoryLimit() int {
@@ -139,7 +149,7 @@ func getNvidiaMPSGpus() ([]isolated_device.IDevice, error) {
 	devs := make([]isolated_device.IDevice, 0)
 	// nvidia-smi  --query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode --format=csv
 	// GPU-76aef7ff-372d-2432-b4b4-beca4d8d3400, Tesla P40, 00000000:00:08.0, 23040 MiB, Exclusive_Process
-	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode", "--format=csv").Output()
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode,index", "--format=csv").Output()
 	if err != nil {
 		return nil, errors.Wrap(err, "nvidia-smi")
 	}
@@ -149,11 +159,11 @@ func getNvidiaMPSGpus() ([]isolated_device.IDevice, error) {
 			continue
 		}
 		segs := strings.Split(line, ",")
-		if len(segs) != 5 {
+		if len(segs) != 6 {
 			log.Errorf("unknown nvidia-smi out line %s", line)
 			continue
 		}
-		gpuId, gpuName, gpuPciAddr, memTotal, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3]), strings.TrimSpace(segs[4])
+		gpuId, gpuName, gpuPciAddr, memTotal, computeMode, index := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3]), strings.TrimSpace(segs[4]), strings.TrimSpace(segs[5])
 		if computeMode != "Exclusive_Process" {
 			log.Warningf("gpu device %s compute mode %s, skip.", gpuId, computeMode)
 			continue
@@ -174,6 +184,7 @@ func getNvidiaMPSGpus() ([]isolated_device.IDevice, error) {
 				MemSizeMB:        memSize / options.HostOptions.CudaMPSReplicas,
 				MemTotalMB:       memSize,
 				ThreadPercentage: 100 / options.HostOptions.CudaMPSReplicas,
+				gpuIndex:         index,
 			}
 			gpuDev.SetModelName(gpuName)
 			devAddr := gpuDev.GetAddr()
@@ -33,8 +33,9 @@ const (
 	TELEGRAF_INPUT_RADEONTOP           = "radeontop"
 	TELEGRAF_INPUT_RADEONTOP_DEV_PATHS = "device_paths"
 	TELEGRAF_INPUT_CONF_BIN_PATH       = "bin_path"
-	TELEGAF_INPUT_NETDEV               = "ni_rsrc_mon"
-	TELEGAF_INPUT_VASMI                = "vasmi"
+	TELEGRAF_INPUT_NETDEV              = "ni_rsrc_mon"
+	TELEGRAF_INPUT_VASMI               = "vasmi"
+	TELEGRAF_INPUT_NVIDIASMI           = "nvidia-smi"
 )

 type STelegraf struct {
@@ -315,19 +316,25 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string {
 		conf += "\n"
 	}

-	if netdev, ok := kwargs[TELEGAF_INPUT_NETDEV]; ok {
+	if netdev, ok := kwargs[TELEGRAF_INPUT_NETDEV]; ok {
 		netdevMap, _ := netdev.(map[string]interface{})
-		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGAF_INPUT_NETDEV)
+		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_NETDEV)
 		conf += fmt.Sprintf("  bin_path = \"%s\"\n", netdevMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string))
 		conf += "\n"
 	}

-	if vasmi, ok := kwargs[TELEGAF_INPUT_VASMI]; ok {
+	if vasmi, ok := kwargs[TELEGRAF_INPUT_VASMI]; ok {
 		vasmiMap, _ := vasmi.(map[string]interface{})
-		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGAF_INPUT_VASMI)
+		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_VASMI)
 		conf += fmt.Sprintf("  bin_path = \"%s\"\n", vasmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string))
 		conf += "\n"
 	}
+
+	if _, ok := kwargs[TELEGRAF_INPUT_NVIDIASMI]; ok {
+		conf += "[[inputs.nvidia_smi]]\n"
+		conf += "\n"
+	}
+
 	return conf
 }

@@ -82,6 +82,7 @@ var All = []SMeasurement{
 	ping,

 	podCpu,
+	podGpu,
 	podMem,
 	podVolume,
 	podProcess,
@@ -39,6 +39,29 @@ var podCpu = SMeasurement{
 	},
 }

+var podGpu = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			Name:         "pod_gpu",
+			DisplayName:  "Pod gpu",
+			ResourceType: monitor.METRIC_RES_TYPE_CONTAINER,
+			Database:     monitor.METRIC_DATABASE_TELE,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			Name:        "mem",
+			DisplayName: "Pod gpu mem used",
+			Unit:        monitor.METRIC_UNIT_MB,
+		},
+		{
+			Name:        "mem_util",
+			DisplayName: "Pod gpu mem usage",
+			Unit:        monitor.METRIC_UNIT_PERCENT,
+		},
+	},
+}
+
 var podMem = SMeasurement{
 	Context: []SMonitorContext{
 		{