Ci e2e metering (#2572)

.github/workflows/controllers.yml

@@ -88,7 +88,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        module: [ app,cluster,infra,metering,user, terminal,imagehub ]
+        module: [ app, cluster, infra, metering, user, terminal, imagehub, account]
     permissions:
       issues: write
     steps:
@@ -97,7 +97,7 @@ jobs:
         if: github.repository == 'labring/sealos'
         with:
           issue-number: 140
-          repository: labring/cluster-image
+          repository: labring-actions/cluster-image
           body: |
             /imagebuild_apps sealos-${{ matrix.module }}-controller dev
           token: "${{ secrets.GH_TOKEN }}"

.github/workflows/e2e_metering.yml

@@ -0,0 +1,125 @@
+name: Test Controllers Metering
+env:
+  # Common versions
+  GO_VERSION: "1.20"
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+    paths:
+      - "controllers/metering/**"
+      - "!.github/workflows/controllers.yml"
+      - "!**/*.md"
+      - "!**/*.yaml"
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        module: [ metering ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Auto install sealos
+        uses: labring/sealos-action@v0.0.4
+        with:
+          sealosVersion: 4.1.5-alpha2
+
+      - name: Build ${{ matrix.module }} amd64
+        working-directory: controllers/${{ matrix.module }}
+        run: |
+          GOARCH=amd64 make build
+          mv bin/manager bin/controller-${{ matrix.module }}-amd64
+          chmod +x bin/controller-${{ matrix.module }}-amd64
+      - name: Prepare
+        id: prepare
+        run: |
+          TAG=deploy
+          echo tag_name=${TAG} >> $GITHUB_OUTPUT
+      - name: build   ${{ matrix.module }} main controller image
+        env:
+          # fork friendly ^^
+          DOCKER_REPO: ghcr.io/${{ github.repository_owner }}/sealos-${{ matrix.module }}-controller
+        working-directory: controllers/${{ matrix.module }}/deploy
+        run: |
+          ls -l
+          sealos build --debug  -t ${DOCKER_REPO}:${{ steps.prepare.outputs.tag_name }}-cluster .
+          sealos login -u ${{ github.repository_owner }} -p ${{ secrets.GITHUB_TOKEN }} --debug ghcr.io
+          sealos push  ${DOCKER_REPO}:${{ steps.prepare.outputs.tag_name }}-cluster
+      - name: docker images
+        run: |
+          sudo sealos images
+
+  e2e-test:
+    needs:
+      - build-image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Auto install sealos
+        uses: labring/sealos-action@v0.0.4
+        with:
+          sealosVersion: 4.1.5-alpha2
+
+      - name: Auto install k8s using sealos
+        uses: labring/sealos-action@v0.0.2
+        with:
+          image: labring/kubernetes:v1.25.0 labring/helm:v3.8.2 labring/calico:v3.24.1
+          debug: true
+          type: run-k8s
+
+      - name: After k8s operation
+        run: |
+          mkdir -p $HOME/.kube
+          sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+          sudo chown $(id -u):$(id -g) $HOME/.kube/config
+          NODENAME=$(sudo -u root kubectl get nodes -ojsonpath='{.items[0].metadata.name}' )
+          NODEIP=$(sudo -u root kubectl get nodes -ojsonpath='{.items[0].status.addresses[0].address}' )
+          echo "NodeName=$NODENAME,NodeIP=$NODEIP"
+          sudo -u root kubectl taint node $NODENAME node-role.kubernetes.io/control-plane-
+          
+          sleep 60
+          sudo -u root crictl ps -a
+          sudo -u root cat /etc/hosts
+          sudo -u root systemctl status kubelet
+          sudo -u root kubectl get nodes 
+          sudo -u root kubectl get nodes 
+          sudo -u root kubectl get pods -A
+      - name: Prepare
+        id: prepare
+        run: |
+          TAG=deploy
+          echo tag_name=${TAG} >> $GITHUB_OUTPUT
+
+      - name: install CRD metering
+        working-directory: ./controllers/metering
+        run: |
+          sudo sealos run --env METERING_INTERVAL=1 ghcr.io/${{ github.repository_owner }}/sealos-metering-controller:${{ steps.prepare.outputs.tag_name }}-cluster
+          sudo -u root kubectl get configmap -A
+      - name: install CRD account
+        working-directory: ./controllers/account
+        run: |
+          make pre-deploy
+          sudo -u root kubectl create namespace account-system
+          sudo -u root kubectl create namespace sealos-system
+          sudo -u root kubectl create secret generic payment-secret -n account-system
+          sudo -u root kubectl apply -f ./deploy/manifests/deploy.yaml
+      - name: install CRD infra
+        working-directory: ./controllers/infra
+        run: |
+          make pre-deploy
+          sudo -u root kubectl apply -f ./deploy/manifests/deploy.yaml 
+          sleep 30
+      - name: Setup Golang with cache
+        uses: magnetikonline/action-golang-cache@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      -  name: e2e Test
+         working-directory: ./controllers/metering
+         run: |
+           sudo -u root kubectl get pod -A 
+           go test ./testdata/e2e/metering_test.go -v

controllers/account/config/rbac/role.yaml

@@ -17,6 +17,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - account.sealos.io
+  resources:
+  - accountbalances/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - account.sealos.io
   resources:

controllers/account/controllers/account_controller.go

@@ -59,6 +59,7 @@ type AccountReconciler struct {
 //+kubebuilder:rbac:groups=account.sealos.io,resources=accounts/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=account.sealos.io,resources=accounts/finalizers,verbs=update
 //+kubebuilder:rbac:groups=account.sealos.io,resources=accountbalances,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=account.sealos.io,resources=accountbalances/status,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete
 

controllers/account/deploy/manifests/deploy.yaml

@@ -289,6 +289,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - account.sealos.io
+  resources:
+  - accountbalances/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - account.sealos.io
   resources:

controllers/account/testdata/account/recharge.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	accountv1 "github.com/labring/sealos/controllers/account/api/v1"
 	"os"
 	"strconv"
 
@@ -46,9 +47,9 @@ func RechargeAccount(AccountName, namespace string, amount int64) error {
 	return nil
 }
 
-func GetAccount(namespace string, name string) (*userv1.Account, error) {
-	gvr := userv1.GroupVersion.WithResource("accounts")
-	var account userv1.Account
+func GetAccount(namespace string, name string) (*accountv1.Account, error) {
+	gvr := accountv1.GroupVersion.WithResource("accounts")
+	var account accountv1.Account
 	if err := baseapi.GetObject(namespace, name, gvr, &account); err != nil {
 		return nil, err
 	}

controllers/metering/Makefile

@@ -101,7 +101,6 @@ pre-deploy: manifests kustomize ## Deploy controller to the K8s cluster specifie
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default  > deploy/manifests/deploy.yaml
 
-
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -

controllers/metering/config/default/manager_auth_proxy_patch.yaml

@@ -28,8 +28,8 @@ spec:
           name: https
         resources:
           limits:
-            cpu: 500m
-            memory: 128Mi
+            cpu: 1000m
+            memory: 512Mi
           requests:
             cpu: 5m
             memory: 64Mi
@@ -38,6 +38,9 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        envFrom:
+          - configMapRef:
+              name: metering-manager-configmap
         env:
           - name: METERING_SYSTEM_NAMESPACE
             valueFrom:

controllers/metering/config/manager/manager.yaml

@@ -56,8 +56,8 @@ spec:
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
-            cpu: 500m
-            memory: 128Mi
+            cpu: 1000m
+            memory: 1280Mi
           requests:
             cpu: 10m
             memory: 64Mi

controllers/metering/config/rbac/role.yaml

@@ -5,6 +5,18 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - account.sealos.io
+  resources:
+  - accountbalances
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -137,15 +149,3 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-  - user.sealos.io
-  resources:
-  - accountbalances
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch

controllers/metering/controllers/metering_controller.go

@@ -66,7 +66,7 @@ type MeteringReconcile struct {
 //+kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=core,resources=resourcequotas,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=infra.sealos.io,resources=infras,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=user.sealos.io,resources=accountbalances,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=account.sealos.io,resources=accountbalances,verbs=get;list;watch;create;update;patch;delete
 
 // Reconcile metering belong to account，resourceQuota belong to metering
 func (r *MeteringReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

controllers/metering/deploy/Dockerfile

@@ -1,5 +1,6 @@
 FROM scratch
 USER 65532:65532
+ENV METERING_INTERVAL 60
 COPY registry ./registry
 COPY manifests ./metering-controller
-CMD ["kubectl apply -f metering-controller/deploy.yaml"]
+CMD ["echo $METERING_INTERVAL","kubectl apply -f metering-controller/deploy.yaml ","kubectl create configmap metering-manager-configmap --from-literal=METERING_INTERVAL=$METERING_INTERVAL -n metering-system"]

controllers/metering/deploy/README.md

@@ -0,0 +1,2 @@
+## how to deploy metering
+sealos run --env METERING_INTERVAL=60 docker.io/labring/sealos-metering-controller:dev

controllers/metering/deploy/manifests/deploy.yaml

@@ -424,6 +424,18 @@ metadata:
   creationTimestamp: null
   name: metering-manager-role
 rules:
+- apiGroups:
+  - account.sealos.io
+  resources:
+  - accountbalances
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -556,18 +568,6 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-  - user.sealos.io
-  resources:
-  - accountbalances
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -705,8 +705,8 @@ spec:
           protocol: TCP
         resources:
           limits:
-            cpu: 500m
-            memory: 128Mi
+            cpu: 1000m
+            memory: 512Mi
           requests:
             cpu: 5m
             memory: 64Mi
@@ -728,6 +728,9 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.namespace
+        envFrom:
+        - configMapRef:
+            name: metering-manager-configmap
         image: ghcr.io/labring/sealos-metering-controller:dev
         imagePullPolicy: Always
         livenessProbe:
@@ -745,8 +748,8 @@ spec:
           periodSeconds: 10
         resources:
           limits:
-            cpu: 500m
-            memory: 128Mi
+            cpu: 1000m
+            memory: 1280Mi
           requests:
             cpu: 10m
             memory: 64Mi

controllers/metering/testdata/api/AccountBalance.go

@@ -37,5 +37,5 @@ func EnsureAccountBalanceCreate(namespace string, name string, times int) (*acco
 		}
 		time.Sleep(time.Second)
 	}
-	return nil, fmt.Errorf("accountbalance get faile")
+	return nil, fmt.Errorf("accountbalance get fail")
 }

controllers/metering/testdata/api/metering.go

@@ -36,6 +36,7 @@ func EnsureMetering(namespace string, name string) {
 
 func EnsureMeteringCalculate(namespace string, name string, times int) (*meteringv1.Metering, error) {
 	EnsureMetering(namespace, name)
+	metering, _ := GetMetering(namespace, name)
 	time.Sleep(time.Second)
 	for i := 1; i <= times; i++ {
 		metering, err := GetMetering(namespace, name)
@@ -47,7 +48,7 @@ func EnsureMeteringCalculate(namespace string, name string, times int) (*meterin
 		}
 		time.Sleep(time.Second)
 	}
-	return nil, fmt.Errorf("metering calculate failed")
+	return metering, fmt.Errorf("metering calculate failed")
 }
 
 func DeleteMetering(namespace string, name string) error {

controllers/metering/testdata/e2e/metering_test.go

@@ -62,11 +62,12 @@ func TestMetering(t *testing.T) {
 		t.Run("extension should be ok", func(t *testing.T) {
 			// test extension will register cpu price to metering
 			t.Log("create metering  ")
+			time.Sleep(time.Second * 10)
 			baseapi.EnsureNamespace(TestNamespace)
-			time.Sleep(time.Second * 3)
+			time.Sleep(time.Second * 10)
 			metering, err := api.GetMetering(MeteringSystemNamespace, meteringv1.MeteringPrefix+TestNamespace)
 			if err != nil {
-				t.Fatalf("success get metering: %v", err)
+				t.Fatalf("fail get metering: %v", err)
 			}
 
 			if _, ok := metering.Spec.Resources["cpu"]; ok {
@@ -86,6 +87,7 @@ func TestMetering(t *testing.T) {
 		})
 
 		t.Run("pod controller should be ok", func(t *testing.T) {
+			time.Sleep(5 * time.Second)
 			baseapi.EnsureNamespace(TestNamespace)
 			time.Sleep(5 * time.Second)
 			t.Log("creat pod controller")
@@ -129,7 +131,7 @@ func TestMetering(t *testing.T) {
 
 		t.Run("metering used update and calculate should be ok", func(t *testing.T) {
 			api.EnsurePodController(MeteringSystemNamespace, meteringv1.PodResourcePricePrefix)
-			time.Sleep(time.Second * 5)
+			time.Sleep(time.Second * 10)
 			baseapi.EnsureNamespace(TestNamespace)
 			api.EnsurePod(TestNamespace, PodName)
 			time.Sleep(time.Second * 5)
@@ -167,6 +169,7 @@ func TestMetering(t *testing.T) {
 			accountBalance, err := api.EnsureAccountBalanceCreate(MeteringSystemNamespace, fmt.Sprintf("%s-%s-%v", accountv1.AccountBalancePrefix, metering.Spec.Owner, 1), 90)
 			if err != nil {
 				t.Log("ensure accountBalance is created again")
+				metering, err = api.GetMetering(MeteringSystemNamespace, meteringv1.MeteringPrefix+TestNamespace)
 				accountBalance, err = api.EnsureAccountBalanceCreate(MeteringSystemNamespace, fmt.Sprintf("%s-%s-%v", accountv1.AccountBalancePrefix, metering.Spec.Owner, 1), 90)
 				if err != nil {
 					t.Fatalf("failed to create accountBalance: %v", err)
@@ -217,7 +220,22 @@ func TestMetering(t *testing.T) {
 
 func clear() {
 	//time.Sleep(200 * time.Second)
-	err := baseapi.DeleteNamespace(TestNamespace)
+	execout, err := baseapi.Exec("sudo -u root kubectl get pod -n metering-system|grep metering |awk '{print $1}' |xargs kubectl logs -n metering-system")
+	log.Println(execout)
+	if err != nil {
+		log.Println(err)
+	}
+	execout, err = baseapi.Exec("sudo -u root kubectl get pod -n account-system|grep account |awk '{print $1}' |xargs kubectl logs -n account-system")
+	log.Println(execout)
+	if err != nil {
+		log.Println(err)
+	}
+	execout, err = baseapi.Exec("sudo -u root kubectl get accountbalance -A")
+	log.Println(execout)
+	if err != nil {
+		log.Println(err)
+	}
+	err = baseapi.DeleteNamespace(TestNamespace)
 	if err != nil {
 		log.Println(err)
 	}