diff --git a/controllers/resources/controllers/monitor_controller.go b/controllers/resources/controllers/monitor_controller.go index 6a270c2a9..218d75f7c 100644 --- a/controllers/resources/controllers/monitor_controller.go +++ b/controllers/resources/controllers/monitor_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "encoding/json" "fmt" "math" "os" @@ -76,8 +77,9 @@ type MonitorReconciler struct { stopCh chan struct{} wg sync.WaitGroup periodicReconcile time.Duration - NvidiaGpu map[string]gpu.NvidiaGPU - gpuMutex sync.Mutex + gpuAliasCard map[string]corev1.ResourceName + gpuNodeAlias map[string]string + gpuMutex sync.RWMutex DBClient database.Interface TrafficClient database.Interface Properties *resources.PropertyTypeLS @@ -95,12 +97,30 @@ type quantity struct { detail string } +type gpuAliasResource struct { + Card string `json:"card"` +} + +type gpuAliasConfig struct { + Resource gpuAliasResource `json:"resource"` +} + +type gpuNodeConfig struct { + Ref string `json:"gpu.ref"` + Product string `json:"gpu.product"` +} + const ( PrometheusURL = "PROM_URL" ObjectStorageInstance = "OBJECT_STORAGE_INSTANCE" ConcurrentLimit = "CONCURRENT_LIMIT" ) +const ( + gpuAliasConfigKey = "alias" + gpuInfoConfigKey = "gpu" +) + var concurrentLimit = int64(DefaultConcurrencyLimit) const ( @@ -128,21 +148,25 @@ func NewMonitorReconciler(mgr ctrl.Manager) (*MonitorReconciler, error) { periodicReconcile: 1 * time.Minute, PromURL: os.Getenv(PrometheusURL), ObjectStorageInstance: os.Getenv(ObjectStorageInstance), - NvidiaGpu: make(map[string]gpu.NvidiaGPU), + gpuAliasCard: make(map[string]corev1.ResourceName), + gpuNodeAlias: make(map[string]string), } concurrentLimit = env.GetInt64EnvWithDefault(ConcurrentLimit, DefaultConcurrencyLimit) var err error err = retry.Retry(2, 1*time.Second, func() error { - r.NvidiaGpu, err = gpu.GetNodeGpuModel(mgr.GetClient()) - if err != nil { - return fmt.Errorf("failed to get node gpu model: %v", err) + if err = r.refreshGPUConfig(context.Background()); err != nil { + return fmt.Errorf("failed to refresh gpu config: %v", err) } return nil }) if err != nil { return nil, err } - r.Logger.Info("get gpu model", "gpu model", r.NvidiaGpu) + r.gpuMutex.RLock() + aliasCount := len(r.gpuAliasCard) + nodeCount := len(r.gpuNodeAlias) + r.gpuMutex.RUnlock() + r.Logger.Info("refresh gpu config", "aliasCount", aliasCount, "nodeCount", nodeCount) return r, nil } @@ -185,6 +209,9 @@ func (r *MonitorReconciler) startPeriodicReconcile() { select { case <-ticker.C: r.enqueueNamespacesForReconcile() + if err := r.refreshGPUConfig(context.Background()); err != nil { + r.Logger.Error(err, "refresh gpu config failed") + } case <-r.stopCh: ticker.Stop() return @@ -434,6 +461,7 @@ func (r *MonitorReconciler) monitorPodResourceUsage(namespace string, resUsed ma return fmt.Errorf("failed to list pods: %v", err) } + knownCardResources := r.getGPUCardResources() for i := range podList.Items { pod := &podList.Items[i] if pod.Spec.NodeName == "" || pod.Status.Phase == corev1.PodSucceeded && time.Since(pod.Status.StartTime.Time) > 1*time.Minute { @@ -445,13 +473,25 @@ func (r *MonitorReconciler) monitorPodResourceUsage(namespace string, resUsed ma if resUsed[podResNamed.String()] == nil { resUsed[podResNamed.String()] = initResources() } + usesGPU := podUsesGPU(pod, knownCardResources) + var aliasKey string + var cardResource corev1.ResourceName + if usesGPU { + var err error + aliasKey, cardResource, err = r.getGPUConfigForNode(pod.Spec.NodeName) + if err != nil { + r.Logger.Error(err, "get gpu config failed", "pod", pod.Name, "namespace", pod.Namespace, "node", pod.Spec.NodeName) + } + } // skip pods that do not start for more than 1 minute skip := pod.Status.Phase != corev1.PodRunning && (pod.Status.StartTime == nil || time.Since(pod.Status.StartTime.Time) > 1*time.Minute) for _, container := range pod.Spec.Containers { // gpu only use limit and not ignore pod pending status - if gpuRequest, ok := container.Resources.Limits[gpu.NvidiaGpuKey]; ok { - if err := r.getGPUResourceUsage(pod, gpuRequest, resUsed[podResNamed.String()]); err != nil { - r.Logger.Error(err, "get gpu resource usage failed", "pod", pod.Name) + if usesGPU && cardResource != "" { + if gpuRequest, ok := container.Resources.Limits[cardResource]; ok { + if err := r.getGPUResourceUsage(pod, aliasKey, gpuRequest, resUsed[podResNamed.String()]); err != nil { + r.Logger.Error(err, "get gpu resource usage failed", "pod", pod.Name) + } } } if skip { @@ -735,24 +775,110 @@ func (r *MonitorReconciler) handlerTrafficUsed(startTime, endTime time.Time, mon return nil } -func (r *MonitorReconciler) getGPUResourceUsage(pod *corev1.Pod, gpuReq resource.Quantity, rs map[corev1.ResourceName]*quantity) (err error) { - nodeName := pod.Spec.NodeName +func (r *MonitorReconciler) refreshGPUConfig(ctx context.Context) error { + configmap := &corev1.ConfigMap{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: gpu.NodeInfoConfigmapNamespace, + Name: gpu.NodeInfoConfigmapName, + }, configmap); err != nil { + return err + } + aliasRaw := strings.TrimSpace(configmap.Data[gpuAliasConfigKey]) + gpuRaw := strings.TrimSpace(configmap.Data[gpuInfoConfigKey]) + if aliasRaw == "" || gpuRaw == "" { + return fmt.Errorf("gpu configmap %s/%s missing data", gpu.NodeInfoConfigmapNamespace, gpu.NodeInfoConfigmapName) + } + aliasMap := make(map[string]gpuAliasConfig) + if err := json.Unmarshal([]byte(aliasRaw), &aliasMap); err != nil { + return fmt.Errorf("unmarshal gpu alias failed: %w", err) + } + gpuMap := make(map[string]gpuNodeConfig) + if err := json.Unmarshal([]byte(gpuRaw), &gpuMap); err != nil { + return fmt.Errorf("unmarshal gpu info failed: %w", err) + } + aliasCard := make(map[string]corev1.ResourceName, len(aliasMap)) + for aliasKey, alias := range aliasMap { + if alias.Resource.Card == "" { + continue + } + aliasCard[aliasKey] = corev1.ResourceName(alias.Resource.Card) + } + nodeAlias := make(map[string]string, len(gpuMap)) + for nodeName, info := range gpuMap { + ref := strings.TrimSpace(info.Ref) + if ref == "" { + ref = strings.TrimSpace(info.Product) + } + if ref == "" { + continue + } + nodeAlias[nodeName] = ref + } r.gpuMutex.Lock() - defer r.gpuMutex.Unlock() - gpuModel, exist := r.NvidiaGpu[nodeName] - if !exist { - if r.NvidiaGpu, err = gpu.GetNodeGpuModel(r.Client); err != nil { - return fmt.Errorf("get node gpu model failed: %w", err) + r.gpuAliasCard = aliasCard + r.gpuNodeAlias = nodeAlias + r.gpuMutex.Unlock() + return nil +} + +func (r *MonitorReconciler) getGPUConfigForNode(nodeName string) (string, corev1.ResourceName, error) { + r.gpuMutex.RLock() + aliasKey, ok := r.gpuNodeAlias[nodeName] + cardResource := r.gpuAliasCard[aliasKey] + r.gpuMutex.RUnlock() + if ok && cardResource != "" { + return aliasKey, cardResource, nil + } + if err := r.refreshGPUConfig(context.Background()); err != nil { + return "", "", err + } + r.gpuMutex.RLock() + aliasKey, ok = r.gpuNodeAlias[nodeName] + cardResource = r.gpuAliasCard[aliasKey] + r.gpuMutex.RUnlock() + if !ok || cardResource == "" { + return "", "", fmt.Errorf("node %s not found gpu config", nodeName) + } + return aliasKey, cardResource, nil +} + +func (r *MonitorReconciler) getGPUCardResources() []corev1.ResourceName { + r.gpuMutex.RLock() + defer r.gpuMutex.RUnlock() + cardResources := make([]corev1.ResourceName, 0, len(r.gpuAliasCard)) + for _, card := range r.gpuAliasCard { + if card == "" { + continue } - if gpuModel, exist = r.NvidiaGpu[nodeName]; !exist { - return fmt.Errorf("node %s not found gpu model", nodeName) + cardResources = append(cardResources, card) + } + return cardResources +} + +func podUsesGPU(pod *corev1.Pod, cardResources []corev1.ResourceName) bool { + if len(cardResources) == 0 { + return false + } + for _, container := range pod.Spec.Containers { + for _, card := range cardResources { + if _, ok := container.Resources.Limits[card]; ok { + return true + } } } - if _, ok := rs[resources.NewGpuResource(gpuModel.GpuInfo.GpuProduct)]; !ok { - rs[resources.NewGpuResource(gpuModel.GpuInfo.GpuProduct)] = initGpuResources() + return false +} + +func (r *MonitorReconciler) getGPUResourceUsage(pod *corev1.Pod, aliasKey string, gpuReq resource.Quantity, rs map[corev1.ResourceName]*quantity) (err error) { + if aliasKey == "" { + return fmt.Errorf("gpu alias is empty") } - logger.Info("gpu request", "pod", pod.Name, "namespace", pod.Namespace, "gpu req", gpuReq.String(), "node", nodeName, "gpu model", gpuModel.GpuInfo.GpuProduct) - rs[resources.NewGpuResource(gpuModel.GpuInfo.GpuProduct)].Add(gpuReq) + gpuResource := resources.NewGpuResource(aliasKey) + if _, ok := rs[gpuResource]; !ok { + rs[gpuResource] = initGpuResources() + } + logger.Info("gpu request", "pod", pod.Name, "namespace", pod.Namespace, "gpu req", gpuReq.String(), "node", pod.Spec.NodeName, "gpu alias", aliasKey) + rs[gpuResource].Add(gpuReq) return nil } diff --git a/controllers/resources/deploy/charts/resources-controller/templates/rbac.yaml b/controllers/resources/deploy/charts/resources-controller/templates/rbac.yaml index 5f2a72115..efc9366aa 100644 --- a/controllers/resources/deploy/charts/resources-controller/templates/rbac.yaml +++ b/controllers/resources/deploy/charts/resources-controller/templates/rbac.yaml @@ -138,6 +138,18 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch - apiGroups: - app.sealos.io resources: diff --git a/service/account/common/account.go b/service/account/common/account.go index 0ab205765..f124d3272 100644 --- a/service/account/common/account.go +++ b/service/account/common/account.go @@ -7,6 +7,7 @@ type PropertyQuery struct { Alias string `json:"alias,omitempty" bson:"alias,omitempty" example:"gpu-tesla-v100"` UnitPrice float64 `json:"unit_price" bson:"unit_price" example:"10000"` Unit string `json:"unit" bson:"unit" example:"1m"` + Enum uint8 `json:"enum,omitempty" bson:"enum,omitempty" example:"1"` } type TimeCostsMap [][]any diff --git a/service/account/dao/interface.go b/service/account/dao/interface.go index fa5336352..fee3f7791 100644 --- a/service/account/dao/interface.go +++ b/service/account/dao/interface.go @@ -662,6 +662,7 @@ func (m *MongoDB) GetProperties() ([]common.PropertyQuery, error) { UnitPrice: propertyType.UnitPrice, Unit: propertyType.UnitString, Alias: propertyType.Alias, + Enum: propertyType.Enum, } if propertyType.ViewPrice > 0 { property.UnitPrice = propertyType.ViewPrice