feat(account): support custom gpu monitor (#6636)

add enum

controllers/resources/controllers/monitor_controller.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
 	"os"
@@ -76,8 +77,9 @@ type MonitorReconciler struct {
 	stopCh                   chan struct{}
 	wg                       sync.WaitGroup
 	periodicReconcile        time.Duration
-	NvidiaGpu                map[string]gpu.NvidiaGPU
-	gpuMutex                 sync.Mutex
+	gpuAliasCard             map[string]corev1.ResourceName
+	gpuNodeAlias             map[string]string
+	gpuMutex                 sync.RWMutex
 	DBClient                 database.Interface
 	TrafficClient            database.Interface
 	Properties               *resources.PropertyTypeLS
@@ -95,12 +97,30 @@ type quantity struct {
 	detail string
 }
 
+type gpuAliasResource struct {
+	Card string `json:"card"`
+}
+
+type gpuAliasConfig struct {
+	Resource gpuAliasResource `json:"resource"`
+}
+
+type gpuNodeConfig struct {
+	Ref     string `json:"gpu.ref"`
+	Product string `json:"gpu.product"`
+}
+
 const (
 	PrometheusURL         = "PROM_URL"
 	ObjectStorageInstance = "OBJECT_STORAGE_INSTANCE"
 	ConcurrentLimit       = "CONCURRENT_LIMIT"
 )
 
+const (
+	gpuAliasConfigKey = "alias"
+	gpuInfoConfigKey  = "gpu"
+)
+
 var concurrentLimit = int64(DefaultConcurrencyLimit)
 
 const (
@@ -128,21 +148,25 @@ func NewMonitorReconciler(mgr ctrl.Manager) (*MonitorReconciler, error) {
 		periodicReconcile:     1 * time.Minute,
 		PromURL:               os.Getenv(PrometheusURL),
 		ObjectStorageInstance: os.Getenv(ObjectStorageInstance),
-		NvidiaGpu:             make(map[string]gpu.NvidiaGPU),
+		gpuAliasCard:          make(map[string]corev1.ResourceName),
+		gpuNodeAlias:          make(map[string]string),
 	}
 	concurrentLimit = env.GetInt64EnvWithDefault(ConcurrentLimit, DefaultConcurrencyLimit)
 	var err error
 	err = retry.Retry(2, 1*time.Second, func() error {
-		r.NvidiaGpu, err = gpu.GetNodeGpuModel(mgr.GetClient())
-		if err != nil {
-			return fmt.Errorf("failed to get node gpu model: %v", err)
+		if err = r.refreshGPUConfig(context.Background()); err != nil {
+			return fmt.Errorf("failed to refresh gpu config: %v", err)
 		}
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
-	r.Logger.Info("get gpu model", "gpu model", r.NvidiaGpu)
+	r.gpuMutex.RLock()
+	aliasCount := len(r.gpuAliasCard)
+	nodeCount := len(r.gpuNodeAlias)
+	r.gpuMutex.RUnlock()
+	r.Logger.Info("refresh gpu config", "aliasCount", aliasCount, "nodeCount", nodeCount)
 	return r, nil
 }
 
@@ -185,6 +209,9 @@ func (r *MonitorReconciler) startPeriodicReconcile() {
 			select {
 			case <-ticker.C:
 				r.enqueueNamespacesForReconcile()
+				if err := r.refreshGPUConfig(context.Background()); err != nil {
+					r.Logger.Error(err, "refresh gpu config failed")
+				}
 			case <-r.stopCh:
 				ticker.Stop()
 				return
@@ -434,6 +461,7 @@ func (r *MonitorReconciler) monitorPodResourceUsage(namespace string, resUsed ma
 		return fmt.Errorf("failed to list pods: %v", err)
 	}
 
+	knownCardResources := r.getGPUCardResources()
 	for i := range podList.Items {
 		pod := &podList.Items[i]
 		if pod.Spec.NodeName == "" || pod.Status.Phase == corev1.PodSucceeded && time.Since(pod.Status.StartTime.Time) > 1*time.Minute {
@@ -445,13 +473,25 @@ func (r *MonitorReconciler) monitorPodResourceUsage(namespace string, resUsed ma
 		if resUsed[podResNamed.String()] == nil {
 			resUsed[podResNamed.String()] = initResources()
 		}
+		usesGPU := podUsesGPU(pod, knownCardResources)
+		var aliasKey string
+		var cardResource corev1.ResourceName
+		if usesGPU {
+			var err error
+			aliasKey, cardResource, err = r.getGPUConfigForNode(pod.Spec.NodeName)
+			if err != nil {
+				r.Logger.Error(err, "get gpu config failed", "pod", pod.Name, "namespace", pod.Namespace, "node", pod.Spec.NodeName)
+			}
+		}
 		// skip pods that do not start for more than 1 minute
 		skip := pod.Status.Phase != corev1.PodRunning && (pod.Status.StartTime == nil || time.Since(pod.Status.StartTime.Time) > 1*time.Minute)
 		for _, container := range pod.Spec.Containers {
 			// gpu only use limit and not ignore pod pending status
-			if gpuRequest, ok := container.Resources.Limits[gpu.NvidiaGpuKey]; ok {
-				if err := r.getGPUResourceUsage(pod, gpuRequest, resUsed[podResNamed.String()]); err != nil {
-					r.Logger.Error(err, "get gpu resource usage failed", "pod", pod.Name)
+			if usesGPU && cardResource != "" {
+				if gpuRequest, ok := container.Resources.Limits[cardResource]; ok {
+					if err := r.getGPUResourceUsage(pod, aliasKey, gpuRequest, resUsed[podResNamed.String()]); err != nil {
+						r.Logger.Error(err, "get gpu resource usage failed", "pod", pod.Name)
+					}
 				}
 			}
 			if skip {
@@ -735,24 +775,110 @@ func (r *MonitorReconciler) handlerTrafficUsed(startTime, endTime time.Time, mon
 	return nil
 }
 
-func (r *MonitorReconciler) getGPUResourceUsage(pod *corev1.Pod, gpuReq resource.Quantity, rs map[corev1.ResourceName]*quantity) (err error) {
-	nodeName := pod.Spec.NodeName
+func (r *MonitorReconciler) refreshGPUConfig(ctx context.Context) error {
+	configmap := &corev1.ConfigMap{}
+	if err := r.Get(ctx, client.ObjectKey{
+		Namespace: gpu.NodeInfoConfigmapNamespace,
+		Name:      gpu.NodeInfoConfigmapName,
+	}, configmap); err != nil {
+		return err
+	}
+	aliasRaw := strings.TrimSpace(configmap.Data[gpuAliasConfigKey])
+	gpuRaw := strings.TrimSpace(configmap.Data[gpuInfoConfigKey])
+	if aliasRaw == "" || gpuRaw == "" {
+		return fmt.Errorf("gpu configmap %s/%s missing data", gpu.NodeInfoConfigmapNamespace, gpu.NodeInfoConfigmapName)
+	}
+	aliasMap := make(map[string]gpuAliasConfig)
+	if err := json.Unmarshal([]byte(aliasRaw), &aliasMap); err != nil {
+		return fmt.Errorf("unmarshal gpu alias failed: %w", err)
+	}
+	gpuMap := make(map[string]gpuNodeConfig)
+	if err := json.Unmarshal([]byte(gpuRaw), &gpuMap); err != nil {
+		return fmt.Errorf("unmarshal gpu info failed: %w", err)
+	}
+	aliasCard := make(map[string]corev1.ResourceName, len(aliasMap))
+	for aliasKey, alias := range aliasMap {
+		if alias.Resource.Card == "" {
+			continue
+		}
+		aliasCard[aliasKey] = corev1.ResourceName(alias.Resource.Card)
+	}
+	nodeAlias := make(map[string]string, len(gpuMap))
+	for nodeName, info := range gpuMap {
+		ref := strings.TrimSpace(info.Ref)
+		if ref == "" {
+			ref = strings.TrimSpace(info.Product)
+		}
+		if ref == "" {
+			continue
+		}
+		nodeAlias[nodeName] = ref
+	}
 	r.gpuMutex.Lock()
-	defer r.gpuMutex.Unlock()
-	gpuModel, exist := r.NvidiaGpu[nodeName]
-	if !exist {
-		if r.NvidiaGpu, err = gpu.GetNodeGpuModel(r.Client); err != nil {
-			return fmt.Errorf("get node gpu model failed: %w", err)
+	r.gpuAliasCard = aliasCard
+	r.gpuNodeAlias = nodeAlias
+	r.gpuMutex.Unlock()
+	return nil
+}
+
+func (r *MonitorReconciler) getGPUConfigForNode(nodeName string) (string, corev1.ResourceName, error) {
+	r.gpuMutex.RLock()
+	aliasKey, ok := r.gpuNodeAlias[nodeName]
+	cardResource := r.gpuAliasCard[aliasKey]
+	r.gpuMutex.RUnlock()
+	if ok && cardResource != "" {
+		return aliasKey, cardResource, nil
+	}
+	if err := r.refreshGPUConfig(context.Background()); err != nil {
+		return "", "", err
+	}
+	r.gpuMutex.RLock()
+	aliasKey, ok = r.gpuNodeAlias[nodeName]
+	cardResource = r.gpuAliasCard[aliasKey]
+	r.gpuMutex.RUnlock()
+	if !ok || cardResource == "" {
+		return "", "", fmt.Errorf("node %s not found gpu config", nodeName)
+	}
+	return aliasKey, cardResource, nil
+}
+
+func (r *MonitorReconciler) getGPUCardResources() []corev1.ResourceName {
+	r.gpuMutex.RLock()
+	defer r.gpuMutex.RUnlock()
+	cardResources := make([]corev1.ResourceName, 0, len(r.gpuAliasCard))
+	for _, card := range r.gpuAliasCard {
+		if card == "" {
+			continue
 		}
-		if gpuModel, exist = r.NvidiaGpu[nodeName]; !exist {
-			return fmt.Errorf("node %s not found gpu model", nodeName)
+		cardResources = append(cardResources, card)
+	}
+	return cardResources
+}
+
+func podUsesGPU(pod *corev1.Pod, cardResources []corev1.ResourceName) bool {
+	if len(cardResources) == 0 {
+		return false
+	}
+	for _, container := range pod.Spec.Containers {
+		for _, card := range cardResources {
+			if _, ok := container.Resources.Limits[card]; ok {
+				return true
+			}
 		}
 	}
-	if _, ok := rs[resources.NewGpuResource(gpuModel.GpuInfo.GpuProduct)]; !ok {
-		rs[resources.NewGpuResource(gpuModel.GpuInfo.GpuProduct)] = initGpuResources()
+	return false
+}
+
+func (r *MonitorReconciler) getGPUResourceUsage(pod *corev1.Pod, aliasKey string, gpuReq resource.Quantity, rs map[corev1.ResourceName]*quantity) (err error) {
+	if aliasKey == "" {
+		return fmt.Errorf("gpu alias is empty")
 	}
-	logger.Info("gpu request", "pod", pod.Name, "namespace", pod.Namespace, "gpu req", gpuReq.String(), "node", nodeName, "gpu model", gpuModel.GpuInfo.GpuProduct)
-	rs[resources.NewGpuResource(gpuModel.GpuInfo.GpuProduct)].Add(gpuReq)
+	gpuResource := resources.NewGpuResource(aliasKey)
+	if _, ok := rs[gpuResource]; !ok {
+		rs[gpuResource] = initGpuResources()
+	}
+	logger.Info("gpu request", "pod", pod.Name, "namespace", pod.Namespace, "gpu req", gpuReq.String(), "node", pod.Spec.NodeName, "gpu alias", aliasKey)
+	rs[gpuResource].Add(gpuReq)
 	return nil
 }
 

controllers/resources/deploy/charts/resources-controller/templates/rbac.yaml

@@ -138,6 +138,18 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - events
+      - configmaps
+    verbs:
+      - create
+      - get
+      - list
+      - patch
+      - update
+      - watch
   - apiGroups:
       - app.sealos.io
     resources:

service/account/common/account.go

@@ -7,6 +7,7 @@ type PropertyQuery struct {
 	Alias     string  `json:"alias,omitempty" bson:"alias,omitempty" example:"gpu-tesla-v100"`
 	UnitPrice float64 `json:"unit_price"      bson:"unit_price"      example:"10000"`
 	Unit      string  `json:"unit"            bson:"unit"            example:"1m"`
+	Enum      uint8   `json:"enum,omitempty"  bson:"enum,omitempty"  example:"1"`
 }
 
 type TimeCostsMap [][]any

service/account/dao/interface.go

@@ -662,6 +662,7 @@ func (m *MongoDB) GetProperties() ([]common.PropertyQuery, error) {
 			UnitPrice: propertyType.UnitPrice,
 			Unit:      propertyType.UnitString,
 			Alias:     propertyType.Alias,
+			Enum:      propertyType.Enum,
 		}
 		if propertyType.ViewPrice > 0 {
 			property.UnitPrice = propertyType.ViewPrice