From ef9d393890ffad3eacddf03b736324bb5ac0a4d0 Mon Sep 17 00:00:00 2001 From: Meteorite <88967335+xiaohan1202@users.noreply.github.com> Date: Thu, 20 Apr 2023 10:39:24 +0800 Subject: [PATCH] feat: add infra metering implementation with e2e test (#2797) * feat: add infra metering * add some features * changed unit --- controllers/cluster/common/common.go | 3 +- .../cluster/config/samples/cluster.yaml | 2 +- .../cluster/controllers/cluster_controller.go | 2 +- .../cluster/deploy/manifests/deploy.yaml | 18 +- controllers/cluster/go.mod | 2 +- controllers/cluster/go.sum | 2 +- controllers/infra/PROJECT | 9 + controllers/infra/api/v1/common.go | 14 + .../infra/api/v1/infraresource_types.go | 64 +++++ .../infra/api/v1/zz_generated.deepcopy.go | 98 +++++++ controllers/infra/common/common.go | 24 +- .../bases/infra.sealos.io_infraresources.yaml | 78 +++++ .../infra/config/crd/kustomization.yaml | 3 + .../cainjection_in_infraresources.yaml | 7 + .../patches/webhook_in_infraresources.yaml | 16 ++ .../rbac/infraresource_editor_role.yaml | 31 ++ .../rbac/infraresource_viewer_role.yaml | 27 ++ controllers/infra/config/rbac/role.yaml | 62 ++++ .../infra/config/samples/infra_aliyun.yaml | 26 +- .../samples/infra_v1_infraresource.yaml | 23 ++ .../infra/controllers/infra_controller.go | 7 +- .../controllers/infraresource_controller.go | 272 ++++++++++++++++++ controllers/infra/deploy/README.md | 4 + .../infra/deploy/manifests/deploy.yaml | 140 +++++++++ .../infra/drivers/get_instance_test.go | 2 +- controllers/infra/go.mod | 2 +- controllers/infra/go.sum | 2 +- controllers/infra/main.go | 11 + .../testdata/api/extension_resource_price.go | 50 ++++ controllers/infra/testdata/api/resource.go | 79 +++++ .../infra/testdata/e2e/infra_metering_test.go | 102 +++++++ controllers/infra/testdata/e2e/infra_test.go | 14 +- go.work.sum | 9 + 33 files changed, 1167 insertions(+), 38 deletions(-) create mode 100644 controllers/infra/api/v1/common.go create mode 100644 controllers/infra/api/v1/infraresource_types.go create mode 100644 controllers/infra/config/crd/bases/infra.sealos.io_infraresources.yaml create mode 100644 controllers/infra/config/crd/patches/cainjection_in_infraresources.yaml create mode 100644 controllers/infra/config/crd/patches/webhook_in_infraresources.yaml create mode 100644 controllers/infra/config/rbac/infraresource_editor_role.yaml create mode 100644 controllers/infra/config/rbac/infraresource_viewer_role.yaml create mode 100644 controllers/infra/config/samples/infra_v1_infraresource.yaml create mode 100644 controllers/infra/controllers/infraresource_controller.go create mode 100644 controllers/infra/testdata/api/extension_resource_price.go create mode 100644 controllers/infra/testdata/api/resource.go create mode 100644 controllers/infra/testdata/e2e/infra_metering_test.go diff --git a/controllers/cluster/common/common.go b/controllers/cluster/common/common.go index 6d03f4b83..dd828fe8a 100644 --- a/controllers/cluster/common/common.go +++ b/controllers/cluster/common/common.go @@ -1,5 +1,6 @@ package common const ( - Master0Label = "master0" + Master0Label = "master0" + SealosVersionAnnotation = "sealos.io/version" ) diff --git a/controllers/cluster/config/samples/cluster.yaml b/controllers/cluster/config/samples/cluster.yaml index dcc12b1a4..e12ae9cc9 100644 --- a/controllers/cluster/config/samples/cluster.yaml +++ b/controllers/cluster/config/samples/cluster.yaml @@ -24,7 +24,7 @@ kind: Cluster metadata: name: my-cluster annotations: - sealos.io/sealos/version: "4.1.4-rc1" + sealos.io/version: "4.1.4-rc1" spec: infra: infra-apply-test image: diff --git a/controllers/cluster/controllers/cluster_controller.go b/controllers/cluster/controllers/cluster_controller.go index bc0580723..6b9f61d2c 100644 --- a/controllers/cluster/controllers/cluster_controller.go +++ b/controllers/cluster/controllers/cluster_controller.go @@ -282,7 +282,7 @@ func getAllInstanceIP(infra *infrav1.Infra) []string { // get sealos version from cluster func getSealosVersion(cluster *v1.Cluster) string { - if v, ok := cluster.Annotations["sealos.io/version"]; ok && v != "" { + if v, ok := cluster.Annotations[common.SealosVersionAnnotation]; ok && v != "" { return v } return defaultSealosVersion diff --git a/controllers/cluster/deploy/manifests/deploy.yaml b/controllers/cluster/deploy/manifests/deploy.yaml index 9c2004b58..9c4adf77d 100644 --- a/controllers/cluster/deploy/manifests/deploy.yaml +++ b/controllers/cluster/deploy/manifests/deploy.yaml @@ -59,7 +59,6 @@ spec: items: properties: env: - description: overwrite env items: type: string type: array @@ -71,6 +70,23 @@ spec: items: type: string type: array + ssh: + properties: + passwd: + type: string + pk: + type: string + pkData: + type: string + pkName: + type: string + pkPasswd: + type: string + port: + type: integer + user: + type: string + type: object type: object type: array image: diff --git a/controllers/cluster/go.mod b/controllers/cluster/go.mod index c12830aed..d93deebb5 100644 --- a/controllers/cluster/go.mod +++ b/controllers/cluster/go.mod @@ -192,4 +192,4 @@ require ( replace ( github.com/labring/sealos => ./../.. github.com/labring/sealos/controllers/infra => ../infra -) +) \ No newline at end of file diff --git a/controllers/cluster/go.sum b/controllers/cluster/go.sum index b38cb0a25..6ef133e39 100644 --- a/controllers/cluster/go.sum +++ b/controllers/cluster/go.sum @@ -1113,4 +1113,4 @@ sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= \ No newline at end of file diff --git a/controllers/infra/PROJECT b/controllers/infra/PROJECT index 1cab0379b..a85236fb0 100644 --- a/controllers/infra/PROJECT +++ b/controllers/infra/PROJECT @@ -13,4 +13,13 @@ resources: kind: Infra path: github.com/labring/sealos/controllers/infra/api/v1 version: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: sealos.io + group: infra + kind: InfraResource + path: github.com/labring/sealos/controllers/infra/api/v1 + version: v1 version: "3" diff --git a/controllers/infra/api/v1/common.go b/controllers/infra/api/v1/common.go new file mode 100644 index 000000000..94f1706f6 --- /dev/null +++ b/controllers/infra/api/v1/common.go @@ -0,0 +1,14 @@ +package v1 + +import meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + +const ( + InfraResourcePricePrefix = "sealos-infra-controller" +) + +var DefaultInfraResourceGVK = []meteringcommonv1.GroupVersionKind{{ + Group: "infra.sealos.io", + Version: "v1", + Kind: "infra", +}, +} diff --git a/controllers/infra/api/v1/infraresource_types.go b/controllers/infra/api/v1/infraresource_types.go new file mode 100644 index 000000000..193ce649e --- /dev/null +++ b/controllers/infra/api/v1/infraresource_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 labring. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// InfraResourceSpec defines the desired state of InfraResource +type InfraResourceSpec struct { + ResourceName string `json:"resourceName,omitempty"` + Resources map[v1.ResourceName]meteringcommonv1.ResourcePrice `json:"resources,omitempty"` + + // update used resources every Interval minutes + //+kubebuilder:validation:Minimum=1 + Interval int `json:"interval,omitempty"` +} + +// InfraResourceStatus defines the observed state of InfraResource +type InfraResourceStatus struct { + LatestUpdateTime int64 `json:"latestUpdateTime,omitempty"` + SeqID int64 `json:"seqID,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// InfraResource is the Schema for the Infraresources API +type InfraResource struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec InfraResourceSpec `json:"spec,omitempty"` + Status InfraResourceStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// InfraResourceList contains a list of InfraResource +type InfraResourceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []InfraResource `json:"items"` +} + +func init() { + SchemeBuilder.Register(&InfraResource{}, &InfraResourceList{}) +} diff --git a/controllers/infra/api/v1/zz_generated.deepcopy.go b/controllers/infra/api/v1/zz_generated.deepcopy.go index 6a81a27fa..1cb3ce3d6 100644 --- a/controllers/infra/api/v1/zz_generated.deepcopy.go +++ b/controllers/infra/api/v1/zz_generated.deepcopy.go @@ -22,6 +22,8 @@ limitations under the License. package v1 import ( + apiv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -203,6 +205,102 @@ func (in *InfraList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InfraResource) DeepCopyInto(out *InfraResource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResource. +func (in *InfraResource) DeepCopy() *InfraResource { + if in == nil { + return nil + } + out := new(InfraResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *InfraResource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InfraResourceList) DeepCopyInto(out *InfraResourceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]InfraResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResourceList. +func (in *InfraResourceList) DeepCopy() *InfraResourceList { + if in == nil { + return nil + } + out := new(InfraResourceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *InfraResourceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InfraResourceSpec) DeepCopyInto(out *InfraResourceSpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make(map[corev1.ResourceName]apiv1.ResourcePrice, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResourceSpec. +func (in *InfraResourceSpec) DeepCopy() *InfraResourceSpec { + if in == nil { + return nil + } + out := new(InfraResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InfraResourceStatus) DeepCopyInto(out *InfraResourceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResourceStatus. +func (in *InfraResourceStatus) DeepCopy() *InfraResourceStatus { + if in == nil { + return nil + } + out := new(InfraResourceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InfraSpec) DeepCopyInto(out *InfraSpec) { *out = *in diff --git a/controllers/infra/common/common.go b/controllers/infra/common/common.go index 2e6c132d0..e825b57bc 100644 --- a/controllers/infra/common/common.go +++ b/controllers/infra/common/common.go @@ -15,11 +15,8 @@ const ( TrySleepTime = time.Second TRUELable = "true" SealosInfraFinalizer = "infra.sealos.io/finalizers" - InfraSecretPrefix = "infra-secret" - InfraVolumeIndex = "infra.sealos.io/volumes/index" VolumeInfraID = "infra.sealos.io/volumes/infraID" InstanceState = "instance-state-name" - DefaultRegion = "cn-north-1b" KeyPairUser = "sealos.io/keypair/user" ArchAmd64 = "amd64" ArchArm64 = "arm64" @@ -27,7 +24,28 @@ const ( AliyunKeyPairPrefix = "infra" DefaultNamespace = "default" MasterO = "master0" + CPUResourceName = "infra/CPU" + MemoryResourceName = "infra/Memory" + VolumeResourceName = "infra/Volume" + InfraSystemNamespace = "infra-system" + InfraDataAnnotation = "infra.sealos.io/data" ) var DefaultRootVolumeSize = int32(40) var DriverList = []string{"aliyun", "aws"} +var CPUMap = map[string]int64{ + "t2.medium": 2, + "t2.large": 2, + "t2.xlarge": 4, + "ecs.c7.large": 2, + "ecs.g7.large": 2, + "ecs.g7.xlarge": 4, +} +var MemoryMap = map[string]int64{ + "t2.medium": 4, + "t2.large": 8, + "t2.xlarge": 16, + "ecs.c7.large": 4, + "ecs.g7.large": 8, + "ecs.g7.xlarge": 16, +} diff --git a/controllers/infra/config/crd/bases/infra.sealos.io_infraresources.yaml b/controllers/infra/config/crd/bases/infra.sealos.io_infraresources.yaml new file mode 100644 index 000000000..89bff76b6 --- /dev/null +++ b/controllers/infra/config/crd/bases/infra.sealos.io_infraresources.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: infraresources.infra.sealos.io +spec: + group: infra.sealos.io + names: + kind: InfraResource + listKind: InfraResourceList + plural: infraresources + singular: infraresource + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: InfraResource is the Schema for the Infraresources API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InfraResourceSpec defines the desired state of InfraResource + properties: + interval: + description: update used resources every Interval minutes + minimum: 1 + type: integer + resourceName: + type: string + resources: + additionalProperties: + properties: + describe: + type: string + price: + format: int64 + type: integer + unit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - price + - unit + type: object + type: object + type: object + status: + description: InfraResourceStatus defines the observed state of InfraResource + properties: + latestUpdateTime: + format: int64 + type: integer + seqID: + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/controllers/infra/config/crd/kustomization.yaml b/controllers/infra/config/crd/kustomization.yaml index 1a4c3b8b5..83ba2c9fc 100644 --- a/controllers/infra/config/crd/kustomization.yaml +++ b/controllers/infra/config/crd/kustomization.yaml @@ -17,17 +17,20 @@ # It should be run by config/default resources: - bases/infra.sealos.io_infras.yaml +- bases/infra.sealos.io_infraresources.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_infras.yaml +#- patches/webhook_in_infraresources.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_infras.yaml +#- patches/cainjection_in_infraresources.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/controllers/infra/config/crd/patches/cainjection_in_infraresources.yaml b/controllers/infra/config/crd/patches/cainjection_in_infraresources.yaml new file mode 100644 index 000000000..e7791737a --- /dev/null +++ b/controllers/infra/config/crd/patches/cainjection_in_infraresources.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: infraresources.infra.sealos.io diff --git a/controllers/infra/config/crd/patches/webhook_in_infraresources.yaml b/controllers/infra/config/crd/patches/webhook_in_infraresources.yaml new file mode 100644 index 000000000..353c5449f --- /dev/null +++ b/controllers/infra/config/crd/patches/webhook_in_infraresources.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: infraresources.infra.sealos.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/controllers/infra/config/rbac/infraresource_editor_role.yaml b/controllers/infra/config/rbac/infraresource_editor_role.yaml new file mode 100644 index 000000000..2cea364af --- /dev/null +++ b/controllers/infra/config/rbac/infraresource_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit infraresources. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: infraresource-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: controller + app.kubernetes.io/part-of: controller + app.kubernetes.io/managed-by: kustomize + name: infraresource-editor-role +rules: +- apiGroups: + - infra.sealos.io + resources: + - infraresources + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infra.sealos.io + resources: + - infraresources/status + verbs: + - get diff --git a/controllers/infra/config/rbac/infraresource_viewer_role.yaml b/controllers/infra/config/rbac/infraresource_viewer_role.yaml new file mode 100644 index 000000000..69aa1ebc8 --- /dev/null +++ b/controllers/infra/config/rbac/infraresource_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view infraresources. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: infraresource-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: controller + app.kubernetes.io/part-of: controller + app.kubernetes.io/managed-by: kustomize + name: infraresource-viewer-role +rules: +- apiGroups: + - infra.sealos.io + resources: + - infraresources + verbs: + - get + - list + - watch +- apiGroups: + - infra.sealos.io + resources: + - infraresources/status + verbs: + - get diff --git a/controllers/infra/config/rbac/role.yaml b/controllers/infra/config/rbac/role.yaml index 14df63482..2eeb15476 100644 --- a/controllers/infra/config/rbac/role.yaml +++ b/controllers/infra/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -17,6 +29,32 @@ rules: - patch - update - watch +- apiGroups: + - infra.sealos.io + resources: + - infraresources + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infra.sealos.io + resources: + - infraresources/finalizers + verbs: + - update +- apiGroups: + - infra.sealos.io + resources: + - infraresources/status + verbs: + - get + - patch + - update - apiGroups: - infra.sealos.io resources: @@ -43,3 +81,27 @@ rules: - get - patch - update +- apiGroups: + - metering.common.sealos.io + resources: + - extensionresourceprices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - metering.common.sealos.io + resources: + - resources + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/controllers/infra/config/samples/infra_aliyun.yaml b/controllers/infra/config/samples/infra_aliyun.yaml index b0284be2b..3eb1c5ee4 100644 --- a/controllers/infra/config/samples/infra_aliyun.yaml +++ b/controllers/infra/config/samples/infra_aliyun.yaml @@ -20,24 +20,18 @@ spec: platform: aliyun hosts: - roles: [ master ] - count: 2 - flavor: ecs.s6-c1m1.small - image: "centos_7_9_x64_20G_alibase_20230109.vhd" - disks: - - capacity: 33 - volumeType: "cloud_efficiency" - type: "root" - - capacity: 34 - volumeType: "cloud_efficiency" - type: "data" - # - capacity: 35 - # volumeType: gp3 - # type: "data" - - roles: [ node ] count: 1 - flavor: ecs.s6-c1m1.small + flavor: ecs.c7.large image: "centos_7_9_x64_20G_alibase_20230109.vhd" disks: - capacity: 40 - volumeType: "cloud_efficiency" + volumeType: "cloud_essd" + type: "root" + - roles: [ node ] + count: 1 + flavor: ecs.c7.large + image: "centos_7_9_x64_20G_alibase_20230109.vhd" + disks: + - capacity: 40 + volumeType: "cloud_essd" type: "root" \ No newline at end of file diff --git a/controllers/infra/config/samples/infra_v1_infraresource.yaml b/controllers/infra/config/samples/infra_v1_infraresource.yaml new file mode 100644 index 000000000..d101b20f6 --- /dev/null +++ b/controllers/infra/config/samples/infra_v1_infraresource.yaml @@ -0,0 +1,23 @@ +apiVersion: infra.sealos.io/v1 +kind: InfraResource +metadata: + name: infraresource-sample + namespace: infra-system +spec: + resourceName: infra + interval: 60 + resources: + infra/CPU: + unit: "10" + price: 112 + describe: "cost of per cpu per hour" + + infra/Memory: + unit: "10Gi" + price: 55 + describe: "cost of per memory per hour" + + infra/Volume: + unit: "100Gi" + price: 35 + describe: "cost of per 1G volume per hour" diff --git a/controllers/infra/controllers/infra_controller.go b/controllers/infra/controllers/infra_controller.go index a3b993840..c92f04769 100644 --- a/controllers/infra/controllers/infra_controller.go +++ b/controllers/infra/controllers/infra_controller.go @@ -117,7 +117,6 @@ func (r *InfraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } r.recorder.Eventf(infra, corev1.EventTypeNormal, "start to reconcile instance", "%s/%s", infra.Namespace, infra.Name) - if err := r.applier.ReconcileInstance(infra, r.driver[infra.Spec.Platform]); err != nil { r.recorder.Eventf(infra, corev1.EventTypeWarning, "reconcile infra failed", "%v", err) return err @@ -197,9 +196,9 @@ func (r *InfraReconciler) SetupWithManager(mgr ctrl.Manager, opts InfraReconcile var err error for _, driverName := range common.DriverList { r.driver[driverName], err = drivers.NewDriver(driverName) - } - if err != nil { - return fmt.Errorf("new driver error: %v", err) + if err != nil { + return fmt.Errorf("new %s driver error: %v", driverName, err) + } } return ctrl.NewControllerManagedBy(mgr). For(&infrav1.Infra{}, builder.WithPredicates( diff --git a/controllers/infra/controllers/infraresource_controller.go b/controllers/infra/controllers/infraresource_controller.go new file mode 100644 index 000000000..f61505800 --- /dev/null +++ b/controllers/infra/controllers/infraresource_controller.go @@ -0,0 +1,272 @@ +/* +Copyright 2022 labring. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "math" + "time" + + meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + + "k8s.io/apimachinery/pkg/api/resource" + + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + v1 "k8s.io/api/core/v1" + + "github.com/labring/sealos/controllers/infra/common" + + "golang.org/x/sync/errgroup" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + infrav1 "github.com/labring/sealos/controllers/infra/api/v1" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InfraResourceReconciler reconciles a InfraResource object +type InfraResourceReconciler struct { + client.Client + Scheme *runtime.Scheme + logr.Logger +} + +type InfraData struct { + Flavor string + Volume []VolumeData + Count int +} + +type VolumeData struct { + Capacity int + VolumeType string +} + +//+kubebuilder:rbac:groups=infra.sealos.io,resources=infraresources,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=infra.sealos.io,resources=infraresources/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=infra.sealos.io,resources=infraresources/finalizers,verbs=update +//+kubebuilder:rbac:groups=metering.common.sealos.io,resources=resources,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=metering.common.sealos.io,resources=extensionresourceprices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;create;update;patch;delete + +func (r *InfraResourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + infraController := &infrav1.InfraResource{} + err := r.Get(ctx, req.NamespacedName, infraController) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Register the resource to metering-quota + err = r.CreateOrUpdateExtensionResourcesPrice(ctx, infraController) + if err != nil { + r.Logger.Error(err, "CreateOrUpdateExtensionResourcesPrice failed") + return ctrl.Result{Requeue: true, RequeueAfter: time.Second}, client.IgnoreNotFound(err) + } + + // update resource used every infraController.Spec.Interval Minutes + if time.Now().Unix()-infraController.Status.LatestUpdateTime >= int64(time.Minute.Minutes())*int64(infraController.Spec.Interval) { + if err := r.UpdateResourceUsed(ctx, infraController); err != nil { + r.Logger.Error(err, "UpdateResourceUsed failed") + return ctrl.Result{Requeue: true}, client.IgnoreNotFound(err) + } + } + return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(infraController.Spec.Interval) * time.Second}, nil +} + +// CreateOrUpdateExtensionResourcesPrice need to create a ExtensionResourcesPrice to make metering-quota know this resource +func (r *InfraResourceReconciler) CreateOrUpdateExtensionResourcesPrice(ctx context.Context, obj client.Object) error { + infraController := obj.(*infrav1.InfraResource) + extensionResourcesPrice := &meteringcommonv1.ExtensionResourcePrice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: common.InfraSystemNamespace, + Name: meteringcommonv1.ExtensionResourcePricePrefix + infrav1.InfraResourcePricePrefix, + }, + } + r.Logger.Info("create or update extensionResourcePrice", "infraController name", infraController.Name, "infraController Resources", infraController.Spec.Resources) + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, extensionResourcesPrice, func() error { + extensionResourcesPrice.Spec.Resources = infraController.Spec.Resources + extensionResourcesPrice.Spec.ResourceName = infraController.Spec.ResourceName + extensionResourcesPrice.Spec.GroupVersionKinds = infrav1.DefaultInfraResourceGVK + return controllerutil.SetControllerReference(infraController, extensionResourcesPrice, r.Scheme) + }); err != nil { + return fmt.Errorf("sync ExtensionResourcesPrice failed: %v", err) + } + r.Logger.V(1).Info("sync extensionResourcesPrice", "extensionResourcesPrice.Spec", extensionResourcesPrice.Spec) + return nil +} + +// UpdateResourceUsed generate a resourceUsed per infra per interval +func (r *InfraResourceReconciler) UpdateResourceUsed(ctx context.Context, obj client.Object) error { + infraController := obj.(*infrav1.InfraResource) + var infraList infrav1.InfraList + eg, _ := errgroup.WithContext(context.Background()) + + // List all infras + err := r.List(ctx, &infraList) + if err != nil { + return err + } + // sync resource used for each infra + for i := range infraList.Items { + infra := infraList.Items[i] + // if infra is not running, skip it + if !r.checkInfraStatusRunning(infra) { + continue + } + eg.Go(func() error { + // sync resource + return r.syncInfraResource(ctx, infra, infraController) + }) + + if err := eg.Wait(); err != nil { + return err + } + } + // update infraController status + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := r.Get(ctx, types.NamespacedName{Namespace: infraController.Namespace, Name: infraController.Name}, infraController); err != nil { + return client.IgnoreNotFound(err) + } + infraController.Status.LatestUpdateTime = time.Now().Unix() + infraController.Status.SeqID++ + return r.Status().Update(ctx, infraController) + }); err != nil { + r.Logger.Error(err, "UpdateResourceUsed failed") + return fmt.Errorf("infra controller update err:%v", err) + } + r.Logger.Info("infra controller calculate resource success") + return nil +} + +// checkInfraStatus check infra status +func (r *InfraResourceReconciler) checkInfraStatusRunning(infra infrav1.Infra) bool { + if infra.Status.Status == infrav1.Running.String() { + return true + } + r.Logger.Info("infra status is not running", "infra name", infra.Name, "infra namespace", infra.Namespace, "infra status", infra.Status.Status) + return false +} + +// syncInfraResource sync resource used for each infra +func (r *InfraResourceReconciler) syncInfraResource(ctx context.Context, infra infrav1.Infra, infraController *infrav1.InfraResource) error { + quantityMap := make(map[v1.ResourceName]int64) + for _, host := range infra.Spec.Hosts { + cnt := int64(host.Count) + flavor := host.Flavor + quantityMap[common.CPUResourceName] += common.CPUMap[flavor] * cnt + quantityMap[common.MemoryResourceName] += common.MemoryMap[flavor] * cnt + for _, disk := range host.Disks { + quantityMap[common.VolumeResourceName] += int64(disk.Capacity) + } + quantityMap[common.VolumeResourceName] = quantityMap[common.VolumeResourceName] * int64(host.Count) + } + r.Logger.Info("infra resources", "cpu quantity", quantityMap[common.CPUResourceName], "memory quantity", quantityMap[common.MemoryResourceName], "volume quantity", quantityMap[common.VolumeResourceName]) + + infraResource := meteringcommonv1.Resource{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetResourceName(&infra, infraController.Status.SeqID), + Namespace: common.InfraSystemNamespace, + }, + } + infraData, err := r.syncInfraData(infra) + if err != nil { + return fmt.Errorf("sync infra data failed: %v", err) + } + infraResource.Annotations = map[string]string{ + common.InfraDataAnnotation: infraData, + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, &infraResource, func() error { + if infraResource.Spec.Resources == nil { + infraResource.Spec.Resources = make(map[v1.ResourceName]meteringcommonv1.ResourceInfo) + } + for k, v := range infraController.Spec.Resources { + var quantity *resource.Quantity + var cost int64 + if k == common.CPUResourceName { + quantity = resource.NewQuantity(quantityMap[k], resource.BinarySI) + cost = int64(math.Ceil(float64(quantityMap[k]) * float64(v.Price) / 10)) + } else if k == common.MemoryResourceName { + quantity = resource.NewQuantity(quantityMap[k]<<30, resource.BinarySI) + cost = int64(math.Ceil(float64(quantityMap[k]) * float64(v.Price) / 10)) + } else { + quantity = resource.NewQuantity(quantityMap[k]<<30, resource.BinarySI) + cost = int64(math.Ceil(float64(quantityMap[k]) * float64(v.Price) / 100)) + } + infraResource.Spec.Resources[k] = meteringcommonv1.ResourceInfo{ + Used: quantity, + Timestamp: time.Now().Unix(), + Namespace: infra.Namespace, + Cost: cost, + } + } + return nil + }); err != nil { + return err + } + + return nil +} + +func (r *InfraResourceReconciler) syncInfraData(infra infrav1.Infra) (string, error) { + var infraData []InfraData + var volumeData []VolumeData + for _, host := range infra.Spec.Hosts { + for _, disk := range host.Disks { + volumeData = append(volumeData, VolumeData{ + Capacity: disk.Capacity, + VolumeType: disk.VolumeType, + }) + } + infraData = append(infraData, InfraData{ + Flavor: host.Flavor, + Volume: volumeData, + Count: host.Count, + }) + } + infraDataStr, err := json.Marshal(infraData) + if err != nil { + return "", fmt.Errorf("marshal infra data err:%v", err) + } + return string(infraDataStr), nil +} + +// GetResourceName get resource name +func GetResourceName(infra *infrav1.Infra, seqID int64) string { + return fmt.Sprintf("%s-%s-%v", infra.Namespace, infra.Name, seqID) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *InfraResourceReconciler) SetupWithManager(mgr ctrl.Manager) error { + const controllerName = "InfraResource-controller" + r.Logger = ctrl.Log.WithName(controllerName) + r.Logger.V(1).Info("reconcile InfraResource-controller") + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.InfraResource{}, builder.WithPredicates( + predicate.Or(predicate.GenerationChangedPredicate{}))). + Complete(r) +} diff --git a/controllers/infra/deploy/README.md b/controllers/infra/deploy/README.md index 4a185c1fd..31ff012b7 100644 --- a/controllers/infra/deploy/README.md +++ b/controllers/infra/deploy/README.md @@ -12,6 +12,10 @@ data: AWS_ACCESS_KEY_ID: xxxx AWS_SECRET_ACCESS_KEY: xxxxx AWS_DEFAULT_REGION: xxxx + ALIYUN_ACCESS_KEY_ID: xxxx + ALIYUN_ACCESS_KEY_SECRET: xxxx + ALIYUN_REGION_ID: xxxx + ALIYUN_RESOURCE_GROUP_ID: xxxx ``` diff --git a/controllers/infra/deploy/manifests/deploy.yaml b/controllers/infra/deploy/manifests/deploy.yaml index 581ac4cd5..1e7b3bf03 100644 --- a/controllers/infra/deploy/manifests/deploy.yaml +++ b/controllers/infra/deploy/manifests/deploy.yaml @@ -7,6 +7,84 @@ metadata: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: infraresources.infra.sealos.io +spec: + group: infra.sealos.io + names: + kind: InfraResource + listKind: InfraResourceList + plural: infraresources + singular: infraresource + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: InfraResource is the Schema for the Infraresources API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InfraResourceSpec defines the desired state of InfraResource + properties: + interval: + description: update used resources every Interval minutes + minimum: 1 + type: integer + resourceName: + type: string + resources: + additionalProperties: + properties: + describe: + type: string + price: + format: int64 + type: integer + unit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - price + - unit + type: object + type: object + type: object + status: + description: InfraResourceStatus defines the observed state of InfraResource + properties: + latestUpdateTime: + format: int64 + type: integer + seqID: + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.9.0 @@ -260,6 +338,18 @@ metadata: creationTimestamp: null name: infra-manager-role rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -272,6 +362,32 @@ rules: - patch - update - watch +- apiGroups: + - infra.sealos.io + resources: + - infraresources + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infra.sealos.io + resources: + - infraresources/finalizers + verbs: + - update +- apiGroups: + - infra.sealos.io + resources: + - infraresources/status + verbs: + - get + - patch + - update - apiGroups: - infra.sealos.io resources: @@ -298,6 +414,30 @@ rules: - get - patch - update +- apiGroups: + - metering.common.sealos.io + resources: + - extensionresourceprices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - metering.common.sealos.io + resources: + - resources + verbs: + - create + - delete + - get + - list + - patch + - update + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/controllers/infra/drivers/get_instance_test.go b/controllers/infra/drivers/get_instance_test.go index 3460ccc70..e9fbb358c 100644 --- a/controllers/infra/drivers/get_instance_test.go +++ b/controllers/infra/drivers/get_instance_test.go @@ -33,7 +33,7 @@ func TestDriver_GetInstances(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "sealos-infra", Namespace: "sealos-infra-ns", - UID: "0abafc31-735b-4a9c-923f-493af2ed1b25", + UID: "b5919c2f-969b-47fd-b9a1-d8d36c28049a", }, Spec: v1.InfraSpec{ AvailabilityZone: "cn-hangzhou-i", diff --git a/controllers/infra/go.mod b/controllers/infra/go.mod index d9a8a978f..1971c04b2 100644 --- a/controllers/infra/go.mod +++ b/controllers/infra/go.mod @@ -97,4 +97,4 @@ require ( replace ( github.com/labring/sealos => ../../ github.com/labring/sealos/controllers/cluster => ../../controllers/cluster -) +) \ No newline at end of file diff --git a/controllers/infra/go.sum b/controllers/infra/go.sum index 5ee15112c..d15135a74 100644 --- a/controllers/infra/go.sum +++ b/controllers/infra/go.sum @@ -961,4 +961,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ih sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= \ No newline at end of file diff --git a/controllers/infra/main.go b/controllers/infra/main.go index f9e67af63..f7dba254d 100644 --- a/controllers/infra/main.go +++ b/controllers/infra/main.go @@ -20,6 +20,7 @@ import ( "flag" "os" + meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -45,6 +46,9 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(infrav1.AddToScheme(scheme)) + + utilruntime.Must(meteringcommonv1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme } @@ -100,6 +104,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Infra") os.Exit(1) } + if err = (&controllers.InfraResourceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "InfraResource") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/controllers/infra/testdata/api/extension_resource_price.go b/controllers/infra/testdata/api/extension_resource_price.go new file mode 100644 index 000000000..8ce512b31 --- /dev/null +++ b/controllers/infra/testdata/api/extension_resource_price.go @@ -0,0 +1,50 @@ +package api + +import ( + meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + baseapi "github.com/labring/sealos/test/testdata/api" +) + +const ExtensionResourcePriceYaml = ` +apiVersion: metering.common.sealos.io/v1 +kind: ExtensionResourcePrice +metadata: + name: extensionresourceprice-sealos-infra-controller + namespace: infra-system +spec: + resourceName: infra + resources: + infra/CPU: + describe: cost of per cpu per hour + price: 670 + unit: '1' + infra/Memory: + describe: cost of per memory per hour + price: 330 + unit: 1G + infra/Volume: + describe: cost of per 1G volume per hour + price: 21 + unit: 1G + +` + +func GetExtensionResourcePrice(namespace string, name string) (*meteringcommonv1.ExtensionResourcePrice, error) { + gvr := meteringcommonv1.GroupVersion.WithResource("extensionresourceprices") + var extensionResourcesPrice meteringcommonv1.ExtensionResourcePrice + if err := baseapi.GetObject(namespace, name, gvr, &extensionResourcesPrice); err != nil { + return nil, err + } + return &extensionResourcesPrice, nil +} + +func DeleteExtensionResourcePrice(namespace string, name string) error { + _, err := baseapi.KubeDeleteFromTemplate(ExtensionResourcePriceYaml, map[string]string{ + "namespace": namespace, + "name": name, + }) + if err != nil { + return err + } + return nil +} diff --git a/controllers/infra/testdata/api/resource.go b/controllers/infra/testdata/api/resource.go new file mode 100644 index 000000000..0a3635042 --- /dev/null +++ b/controllers/infra/testdata/api/resource.go @@ -0,0 +1,79 @@ +package api + +import ( + "fmt" + "time" + + meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + baseapi "github.com/labring/sealos/test/testdata/api" +) + +const ResourceYaml = ` +apiVersion: infra.sealos.io/v1 +kind: InfraResource +metadata: + name: ${name} + namespace: ${namespace} +spec: + resourceName: infra + interval: 60 + resources: + infra/CPU: + unit: "10" + price: 112 + describe: "cost of per cpu per hour" + + infra/Memory: + unit: "10Gi" + price: 55 + describe: "cost of per memory per hour" + + infra/Volume: + unit: "100Gi" + price: 35 + describe: "cost of per 1G volume per hour" +` + +func GetResource(namespace string, name string) (*meteringcommonv1.Resource, error) { + gvr := meteringcommonv1.GroupVersion.WithResource("resources") + var resource meteringcommonv1.Resource + if err := baseapi.GetObject(namespace, name, gvr, &resource); err != nil { + return nil, err + } + return &resource, nil +} + +func SyncResource(namespace string, name string) error { + _, err := baseapi.KubeApplyFromTemplate(ResourceYaml, map[string]string{ + "namespace": namespace, + "name": name, + }) + if err != nil { + return err + } + return nil +} + +func EnsureResourceCreate(namespace string, name string, times int) (*meteringcommonv1.Resource, error) { + time.Sleep(time.Second) + for i := 1; i <= times; i++ { + resource, err := GetResource(namespace, name) + if err != nil { + time.Sleep(time.Second) + continue + } + return resource, nil + } + return nil, fmt.Errorf("resource create failed") +} + +func DeleteResource(namespace string, name string) error { + _, err := baseapi.KubeDeleteFromTemplate(ResourceYaml, map[string]string{ + "namespace": namespace, + "name": name, + }) + if err != nil { + return err + } + return nil +} diff --git a/controllers/infra/testdata/e2e/infra_metering_test.go b/controllers/infra/testdata/e2e/infra_metering_test.go new file mode 100644 index 000000000..a85c83517 --- /dev/null +++ b/controllers/infra/testdata/e2e/infra_metering_test.go @@ -0,0 +1,102 @@ +package e2e + +import ( + "fmt" + "log" + "testing" + "time" + + infrav1 "github.com/labring/sealos/controllers/infra/api/v1" + v1 "k8s.io/api/core/v1" + + "github.com/labring/sealos/controllers/infra/common" + "github.com/labring/sealos/controllers/infra/controllers" + + meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1" + "github.com/labring/sealos/controllers/infra/testdata/api" + + baseapi "github.com/labring/sealos/test/testdata/api" +) + +func TestInfraMetering(t *testing.T) { + baseapi.EnsureNamespace(TestNamespace) + t.Run("infra controller test", func(t *testing.T) { + time.Sleep(5 * time.Second) + baseapi.EnsureNamespace(TestNamespace) + baseapi.EnsureNamespace(InfraSystemNamespace) + t.Log("create infra and wait it to run") + if err := api.CreateInfra(TestNamespace, InfraName); err != nil { + t.Fatalf("create infra failed: %v", err) + } + + if err := api.WaitInfraRunning(TestNamespace, InfraName, 100); err != nil { + t.Fatalf("infra failed to run: %v", err) + } + t.Log("infra succeeded to run") + + t.Log("sync resource") + if err := api.SyncResource(InfraSystemNamespace, InfraResourceName); err != nil { + t.Fatalf("failed to sync resource: %v", err) + } + + time.Sleep(5 * time.Second) + t.Log("ensure extension resource is created") + _, err := api.GetExtensionResourcePrice(InfraSystemNamespace, meteringcommonv1.GetExtensionResourcePriceName(infrav1.InfraResourcePricePrefix)) + if err != nil { + t.Fatalf("failed to get extension resource: %v", err) + } + + t.Log("ensure resource CR is created") + infra, _ := api.GetInfra(TestNamespace, InfraName) + resource, err := api.EnsureResourceCreate(InfraSystemNamespace, controllers.GetResourceName(infra, 1), 90) + if err != nil { + resource, err = api.EnsureResourceCreate(InfraSystemNamespace, controllers.GetResourceName(infra, 2), 90) + if err != nil { + t.Fatalf(err.Error()) + } + } + //check used resource + if err = checkResourceUsed(resource, common.CPUResourceName); err != nil { + t.Fatalf(err.Error()) + } + if err = checkResourceUsed(resource, common.MemoryResourceName); err != nil { + t.Fatalf(err.Error()) + } + if err = checkResourceUsed(resource, common.VolumeResourceName); err != nil { + t.Fatalf(err.Error()) + } + t.Log(resource) + }) + t.Cleanup(cleanInfraMetering) +} + +func checkResourceUsed(resource *meteringcommonv1.Resource, resourceName v1.ResourceName) error { + if _, ok := resource.Spec.Resources[resourceName]; !ok { + return fmt.Errorf("%s resource not found", resourceName) + } + if resource.Spec.Resources[resourceName].Used.Value() != 0 { + fmt.Printf("resource %s has been used %v\n", resourceName, resource.Spec.Resources[resourceName].Used.Value()) + } + return nil +} + +func cleanInfraMetering() { + fmt.Println("clean infra metering") + //execout, err := baseapi.Exec("kubectl get resource -A -oyaml") + //log.Println(execout) + //if err != nil { + // log.Println(err) + //} + + if err := api.DeleteInfra(TestNamespace, InfraName); err != nil { + log.Println(err) + } + + if err := api.DeleteResource(InfraSystemNamespace, InfraResourceName); err != nil { + log.Println(err) + } + + if err := baseapi.DeleteNamespace(TestNamespace); err != nil { + log.Println(err) + } +} diff --git a/controllers/infra/testdata/e2e/infra_test.go b/controllers/infra/testdata/e2e/infra_test.go index a7b6fe016..7f2c5c755 100644 --- a/controllers/infra/testdata/e2e/infra_test.go +++ b/controllers/infra/testdata/e2e/infra_test.go @@ -12,8 +12,10 @@ import ( ) const ( - TestNamespace = "ns-infra-test" - InfraName = "infra-test" + TestNamespace = "ns-infra-test" + InfraName = "infra-test" + InfraSystemNamespace = "infra-system" + InfraResourceName = "infra-resource" ) func TestInfra(t *testing.T) { @@ -50,14 +52,14 @@ func TestInfra(t *testing.T) { } t.Log("cluster succeeded to run") - t.Cleanup(clean) + t.Cleanup(cleanInfra) cnt++ }) } - t.Cleanup(cleanAll) + t.Cleanup(cleanAllInfra) } -func clean() { +func cleanInfra() { if err := api.DeleteCluster(TestNamespace, InfraName); err != nil { log.Println(err) } @@ -67,7 +69,7 @@ func clean() { } } -func cleanAll() { +func cleanAllInfra() { if err := baseapi.DeleteNamespace(TestNamespace); err != nil { log.Println(err) } diff --git a/go.work.sum b/go.work.sum index d0f6ac8de..cc4f9b809 100644 --- a/go.work.sum +++ b/go.work.sum @@ -489,6 +489,7 @@ github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.198/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= @@ -600,6 +601,7 @@ github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGF github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -628,7 +630,11 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/intel/goresctrl v0.3.0/go.mod h1:fdz3mD85cmP9sHD8JUlrNWAxvwM86CrbmVXltEKd7zk= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= @@ -674,6 +680,7 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221014010322-58c91d646d86/go. github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/opencontainers/selinux v1.10.2/go.mod h1:cARutUbaUrlRClyvxOICCgKixCs6L05aUsohzA3EkHQ= github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df/go.mod h1:TRYHe4CH9U6nkDjxjBNM5klrLbJBrRbpJE5SaRwUBsQ= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -699,6 +706,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.7/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=