From 205a5bb4fdeeb05d8ca27bd25cda37b8eeddb8b1 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Tue, 22 Aug 2023 11:11:31 +0800 Subject: [PATCH] feat: add initGPUInfoCM func (#3633) * fix: add env for Kubefile CMD * feat: add cm & role & rb config * feat: watch create event of node-gpu-info configmap to init it * fix name error * update yaml * recover Kubefile * Remove the watch CM logic and directly access the api-server to init the CM * fix review * feat: add alias data to node-gpu-info configmap * rename gpu short name --- .../node/config/configmap/configmap.yaml | 9 ++ .../node/config/configmap/kustomization.yaml | 2 + .../node/config/default/kustomization.yaml | 1 + controllers/node/config/rbac/role.yaml | 11 ++ .../node/config/rbac/role_binding.yaml | 14 ++ .../node/controllers/gpu_controller.go | 123 +++++++++++++----- controllers/node/deploy/manifests/deploy.yaml | 39 ++++++ 7 files changed, 166 insertions(+), 33 deletions(-) create mode 100644 controllers/node/config/configmap/configmap.yaml create mode 100644 controllers/node/config/configmap/kustomization.yaml diff --git a/controllers/node/config/configmap/configmap.yaml b/controllers/node/config/configmap/configmap.yaml new file mode 100644 index 000000000..4646091a2 --- /dev/null +++ b/controllers/node/config/configmap/configmap.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gpu-info + namespace: node-system +data: + gpu: "" + alias: '{"NVIDIA-GeForce-RTX-4090":"GeForce-RTX-4090"}' diff --git a/controllers/node/config/configmap/kustomization.yaml b/controllers/node/config/configmap/kustomization.yaml new file mode 100644 index 000000000..80a3ead29 --- /dev/null +++ b/controllers/node/config/configmap/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - configmap.yaml \ No newline at end of file diff --git a/controllers/node/config/default/kustomization.yaml b/controllers/node/config/default/kustomization.yaml index b37b4c606..070f572b6 100644 --- a/controllers/node/config/default/kustomization.yaml +++ b/controllers/node/config/default/kustomization.yaml @@ -18,6 +18,7 @@ resources: #- ../crd - ../rbac - ../manager +- ../configmap # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- ../webhook diff --git a/controllers/node/config/rbac/role.yaml b/controllers/node/config/rbac/role.yaml index 4a9b68c50..f9f61a0e6 100644 --- a/controllers/node/config/rbac/role.yaml +++ b/controllers/node/config/rbac/role.yaml @@ -79,3 +79,14 @@ rules: - get - patch - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gpu-info-cm-reader + namespace: node-system +rules: + - apiGroups: [ "" ] # "" indicates the core API group + resources: [ "configmaps" ] + resourceNames: [ "gpu-info" ] + verbs: [ "get", "watch", "list" ] \ No newline at end of file diff --git a/controllers/node/config/rbac/role_binding.yaml b/controllers/node/config/rbac/role_binding.yaml index 67989a75b..38b9c3354 100644 --- a/controllers/node/config/rbac/role_binding.yaml +++ b/controllers/node/config/rbac/role_binding.yaml @@ -17,3 +17,17 @@ subjects: - kind: ServiceAccount name: controller-manager namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gpu-info-cm-reader-rolebinding + namespace: node-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: node-gpu-info-cm-reader +subjects: + - kind: Group + name: system:serviceaccounts + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/controllers/node/controllers/gpu_controller.go b/controllers/node/controllers/gpu_controller.go index 88ad58a9a..787cb2faa 100644 --- a/controllers/node/controllers/gpu_controller.go +++ b/controllers/node/controllers/gpu_controller.go @@ -25,9 +25,14 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -41,8 +46,8 @@ type GpuReconciler struct { const ( GPU = "gpu" - GPUInfo = "gpu-info" - GPUInfoNameSpace = "sealos" + GPUInfo = "node-gpu-info" + GPUInfoNameSpace = "node-system" NvidiaGPUProduct = "nvidia.com/gpu.product" NvidiaGPUMemory = "nvidia.com/gpu.memory" NvidiaGPU corev1.ResourceName = "nvidia.com/gpu" @@ -73,7 +78,10 @@ func (r *GpuReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Res r.Logger.Error(err, "failed to get pod list") return ctrl.Result{}, err } + return r.applyGPUInfoCM(ctx, nodeList, podList, nil) +} +func (r *GpuReconciler) applyGPUInfoCM(ctx context.Context, nodeList *corev1.NodeList, podList *corev1.PodList, clientSet *kubernetes.Clientset) (ctrl.Result, error) { /* "nodeMap": { "sealos-poc-gpu-master-0":{}, @@ -137,7 +145,13 @@ func (r *GpuReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Res // create or update gpu-info configmap configmap := &corev1.ConfigMap{} - err = r.Get(ctx, types.NamespacedName{Name: GPUInfo, Namespace: GPUInfoNameSpace}, configmap) + + if clientSet != nil { + configmap, err = clientSet.CoreV1().ConfigMaps(GPUInfoNameSpace).Get(ctx, GPUInfo, metaV1.GetOptions{}) + } else { + err = r.Get(ctx, types.NamespacedName{Name: GPUInfo, Namespace: GPUInfoNameSpace}, configmap) + } + if errors.IsNotFound(err) { configmap = &corev1.ConfigMap{ ObjectMeta: metaV1.ObjectMeta{ @@ -152,27 +166,73 @@ func (r *GpuReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Res r.Logger.Error(err, "failed to create gpu-info configmap") return ctrl.Result{}, err } - } else if err == nil { - if configmap.Data[GPU] != nodeMapStr { - configmap.Data[GPU] = nodeMapStr - if err := r.Update(ctx, configmap); err != nil && !errors.IsConflict(err) { - r.Logger.Error(err, "failed to update gpu-info configmap") - return ctrl.Result{}, err - } - } - } else { + } else if err != nil { r.Logger.Error(err, "failed to get gpu-info configmap") return ctrl.Result{}, err } + + if configmap.Data == nil { + configmap.Data = map[string]string{} + } + if configmap.Data[GPU] != nodeMapStr { + configmap.Data[GPU] = nodeMapStr + if err := r.Update(ctx, configmap); err != nil && !errors.IsConflict(err) { + r.Logger.Error(err, "failed to update gpu-info configmap") + return ctrl.Result{}, err + } + } + r.Logger.V(1).Info("gpu-info configmap status", "gpu", configmap.Data[GPU]) return ctrl.Result{}, nil } +func (r *GpuReconciler) initGPUInfoCM(ctx context.Context, clientSet *kubernetes.Clientset) error { + // filter for nodes that have GPU + req1, _ := labels.NewRequirement(NvidiaGPUProduct, selection.Exists, []string{}) + req2, _ := labels.NewRequirement(NvidiaGPUMemory, selection.Exists, []string{}) + selector := labels.NewSelector().Add(*req1, *req2) + listOpts := metaV1.ListOptions{ + LabelSelector: selector.String(), + } + + nodeList, err := clientSet.CoreV1().Nodes().List(ctx, listOpts) + if err != nil { + return err + } + + podList := &corev1.PodList{} + for _, item := range nodeList.Items { + list, err := clientSet.CoreV1().Pods("").List(context.TODO(), metaV1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.nodeName", item.Name).String(), + }) + if err != nil { + return err + } + podList.Items = append(podList.Items, list.Items...) + } + + _, err = r.applyGPUInfoCM(ctx, nodeList, podList, clientSet) + return err +} + // SetupWithManager sets up the controller with the Manager. func (r *GpuReconciler) SetupWithManager(mgr ctrl.Manager) error { r.Logger = ctrl.Log.WithName("gpu-controller") r.Logger.V(1).Info("starting gpu controller") + // use clientSet to get resources from the API Server, not from Informer's cache + clientSet, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + r.Logger.Error(err, "failed to init") + return nil + } + + // init node-gpu-info configmap + r.Logger.V(1).Info("initializing node-gpu-info configmap") + if err := r.initGPUInfoCM(context.Background(), clientSet); err != nil { + return err + } + // build index for node which have GPU if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Node{}, NodeIndexKey, func(rawObj client.Object) []string { node := rawObj.(*corev1.Node) @@ -198,27 +258,24 @@ func (r *GpuReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Pod{}). - WithEventFilter( - predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - _, ok := event.Object.(*corev1.Pod).Spec.NodeSelector[NvidiaGPUProduct] - return ok - }, - UpdateFunc: func(event event.UpdateEvent) bool { - _, ok := event.ObjectNew.(*corev1.Pod).Spec.NodeSelector[NvidiaGPUProduct] - if !ok { - return false - } - phaseOld := event.ObjectOld.(*corev1.Pod).Status.Phase - phaseNew := event.ObjectNew.(*corev1.Pod).Status.Phase - return phaseOld != phaseNew - }, - DeleteFunc: func(event event.DeleteEvent) bool { - _, ok := event.Object.(*corev1.Pod).Spec.NodeSelector[NvidiaGPUProduct] - return ok - }, + For(&corev1.Pod{}, builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + _, ok := event.Object.(*corev1.Pod).Spec.NodeSelector[NvidiaGPUProduct] + return ok }, - ). + UpdateFunc: func(event event.UpdateEvent) bool { + _, ok := event.ObjectNew.(*corev1.Pod).Spec.NodeSelector[NvidiaGPUProduct] + if !ok { + return false + } + phaseOld := event.ObjectOld.(*corev1.Pod).Status.Phase + phaseNew := event.ObjectNew.(*corev1.Pod).Status.Phase + return phaseOld != phaseNew + }, + DeleteFunc: func(event event.DeleteEvent) bool { + _, ok := event.Object.(*corev1.Pod).Spec.NodeSelector[NvidiaGPUProduct] + return ok + }, + })). Complete(r) } diff --git a/controllers/node/deploy/manifests/deploy.yaml b/controllers/node/deploy/manifests/deploy.yaml index 02a007986..719baebb9 100644 --- a/controllers/node/deploy/manifests/deploy.yaml +++ b/controllers/node/deploy/manifests/deploy.yaml @@ -26,6 +26,23 @@ metadata: --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role +metadata: + name: node-gpu-info-cm-reader + namespace: node-system +rules: + - apiGroups: + - "" + resourceNames: + - node-gpu-info + resources: + - configmaps + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: labels: app.kubernetes.io/component: rbac @@ -194,6 +211,20 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding +metadata: + name: node-gpu-info-cm-reader-rolebinding + namespace: node-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: node-gpu-info-cm-reader +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: labels: app.kubernetes.io/component: rbac @@ -254,6 +285,14 @@ subjects: namespace: node-system --- apiVersion: v1 +data: + gpu: "" +kind: ConfigMap +metadata: + name: node-gpu-info + namespace: node-system +--- +apiVersion: v1 kind: Service metadata: labels: