opensource/sealos
3
0
Fork 0
mirror of https://github.com/labring/sealos.git synced 2026-10-30 18:09:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add infra metering implementation with e2e test (#2797)

Browse Source 
* feat: add infra metering

* add some features

* changed unit
This commit is contained in:
Meteorite
2023-04-20 10:39:24 +08:00
committed by GitHub
parent aa72e131e7
commit ef9d393890
33 changed files with 1167 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers/cluster/common/common.go
+2 -1
View File
@@ -1,5 +1,6 @@
package common
const (
Master0Label = "master0"
Master0Label = "master0"
SealosVersionAnnotation = "sealos.io/version"
)
controllers/cluster/config/samples/cluster.yaml
+1 -1
View File
@@ -24,7 +24,7 @@ kind: Cluster
metadata:
name: my-cluster
annotations:
sealos.io/sealos/version: "4.1.4-rc1"
sealos.io/version: "4.1.4-rc1"
spec:
infra: infra-apply-test
image:
controllers/cluster/controllers/cluster_controller.go
+1 -1
View File
@@ -282,7 +282,7 @@ func getAllInstanceIP(infra *infrav1.Infra) []string {
// get sealos version from cluster
func getSealosVersion(cluster *v1.Cluster) string {
if v, ok := cluster.Annotations["sealos.io/version"]; ok && v != "" {
if v, ok := cluster.Annotations[common.SealosVersionAnnotation]; ok && v != "" {
return v
}
return defaultSealosVersion
controllers/cluster/deploy/manifests/deploy.yaml
+17 -1
View File
@@ -59,7 +59,6 @@ spec:
items:
properties:
env:
description: overwrite env
items:
type: string
type: array
@@ -71,6 +70,23 @@ spec:
items:
type: string
type: array
ssh:
properties:
passwd:
type: string
pk:
type: string
pkData:
type: string
pkName:
type: string
pkPasswd:
type: string
port:
type: integer
user:
type: string
type: object
type: object
type: array
image:
controllers/cluster/go.mod
+1 -1
View File
@@ -192,4 +192,4 @@ require (
replace (
github.com/labring/sealos => ./../..
github.com/labring/sealos/controllers/infra => ../infra
)
)
controllers/cluster/go.sum
+1 -1
View File
@@ -1113,4 +1113,4 @@ sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
controllers/infra/PROJECT
+9
View File
@@ -13,4 +13,13 @@ resources:
kind: Infra
path: github.com/labring/sealos/controllers/infra/api/v1
version: v1
- api:
crdVersion: v1
namespaced: true
controller: true
domain: sealos.io
group: infra
kind: InfraResource
path: github.com/labring/sealos/controllers/infra/api/v1
version: v1
version: "3"
controllers/infra/api/v1/common.go
+14
View File
@@ -0,0 +1,14 @@
package v1
import meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
const (
InfraResourcePricePrefix = "sealos-infra-controller"
)
var DefaultInfraResourceGVK = []meteringcommonv1.GroupVersionKind{{
Group: "infra.sealos.io",
Version: "v1",
Kind: "infra",
},
}
controllers/infra/api/v1/infraresource_types.go
+64
View File
@@ -0,0 +1,64 @@
/*
Copyright 2022 labring.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import (
meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// InfraResourceSpec defines the desired state of InfraResource
type InfraResourceSpec struct {
ResourceName string `json:"resourceName,omitempty"`
Resources map[v1.ResourceName]meteringcommonv1.ResourcePrice `json:"resources,omitempty"`
// update used resources every Interval minutes
//+kubebuilder:validation:Minimum=1
Interval int `json:"interval,omitempty"`
}
// InfraResourceStatus defines the observed state of InfraResource
type InfraResourceStatus struct {
LatestUpdateTime int64 `json:"latestUpdateTime,omitempty"`
SeqID int64 `json:"seqID,omitempty"`
}
//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
// InfraResource is the Schema for the Infraresources API
type InfraResource struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec InfraResourceSpec `json:"spec,omitempty"`
Status InfraResourceStatus `json:"status,omitempty"`
}
//+kubebuilder:object:root=true
// InfraResourceList contains a list of InfraResource
type InfraResourceList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []InfraResource `json:"items"`
}
func init() {
SchemeBuilder.Register(&InfraResource{}, &InfraResourceList{})
}
controllers/infra/api/v1/zz_generated.deepcopy.go
+98
View File
@@ -22,6 +22,8 @@ limitations under the License.
package v1
import (
apiv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@@ -203,6 +205,102 @@ func (in *InfraList) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InfraResource) DeepCopyInto(out *InfraResource) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
out.Status = in.Status
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResource.
func (in *InfraResource) DeepCopy() *InfraResource {
if in == nil {
return nil
}
out := new(InfraResource)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *InfraResource) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InfraResourceList) DeepCopyInto(out *InfraResourceList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]InfraResource, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResourceList.
func (in *InfraResourceList) DeepCopy() *InfraResourceList {
if in == nil {
return nil
}
out := new(InfraResourceList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *InfraResourceList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InfraResourceSpec) DeepCopyInto(out *InfraResourceSpec) {
*out = *in
if in.Resources != nil {
in, out := &in.Resources, &out.Resources
*out = make(map[corev1.ResourceName]apiv1.ResourcePrice, len(*in))
for key, val := range *in {
(*out)[key] = *val.DeepCopy()
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResourceSpec.
func (in *InfraResourceSpec) DeepCopy() *InfraResourceSpec {
if in == nil {
return nil
}
out := new(InfraResourceSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InfraResourceStatus) DeepCopyInto(out *InfraResourceStatus) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraResourceStatus.
func (in *InfraResourceStatus) DeepCopy() *InfraResourceStatus {
if in == nil {
return nil
}
out := new(InfraResourceStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InfraSpec) DeepCopyInto(out *InfraSpec) {
*out = *in
controllers/infra/common/common.go
+21 -3
View File
@@ -15,11 +15,8 @@ const (
TrySleepTime = time.Second
TRUELable = "true"
SealosInfraFinalizer = "infra.sealos.io/finalizers"
InfraSecretPrefix = "infra-secret"
InfraVolumeIndex = "infra.sealos.io/volumes/index"
VolumeInfraID = "infra.sealos.io/volumes/infraID"
InstanceState = "instance-state-name"
DefaultRegion = "cn-north-1b"
KeyPairUser = "sealos.io/keypair/user"
ArchAmd64 = "amd64"
ArchArm64 = "arm64"
@@ -27,7 +24,28 @@ const (
AliyunKeyPairPrefix = "infra"
DefaultNamespace = "default"
MasterO = "master0"
CPUResourceName = "infra/CPU"
MemoryResourceName = "infra/Memory"
VolumeResourceName = "infra/Volume"
InfraSystemNamespace = "infra-system"
InfraDataAnnotation = "infra.sealos.io/data"
)
var DefaultRootVolumeSize = int32(40)
var DriverList = []string{"aliyun", "aws"}
var CPUMap = map[string]int64{
"t2.medium": 2,
"t2.large": 2,
"t2.xlarge": 4,
"ecs.c7.large": 2,
"ecs.g7.large": 2,
"ecs.g7.xlarge": 4,
}
var MemoryMap = map[string]int64{
"t2.medium": 4,
"t2.large": 8,
"t2.xlarge": 16,
"ecs.c7.large": 4,
"ecs.g7.large": 8,
"ecs.g7.xlarge": 16,
}
controllers/infra/config/crd/bases/infra.sealos.io_infraresources.yaml
+78
View File
@@ -0,0 +1,78 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
creationTimestamp: null
name: infraresources.infra.sealos.io
spec:
group: infra.sealos.io
names:
kind: InfraResource
listKind: InfraResourceList
plural: infraresources
singular: infraresource
scope: Namespaced
versions:
- name: v1
schema:
openAPIV3Schema:
description: InfraResource is the Schema for the Infraresources API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: InfraResourceSpec defines the desired state of InfraResource
properties:
interval:
description: update used resources every Interval minutes
minimum: 1
type: integer
resourceName:
type: string
resources:
additionalProperties:
properties:
describe:
type: string
price:
format: int64
type: integer
unit:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
required:
- price
- unit
type: object
type: object
type: object
status:
description: InfraResourceStatus defines the observed state of InfraResource
properties:
latestUpdateTime:
format: int64
type: integer
seqID:
format: int64
type: integer
type: object
type: object
served: true
storage: true
subresources:
status: {}
controllers/infra/config/crd/kustomization.yaml
+3
View File
@@ -17,17 +17,20 @@
# It should be run by config/default
resources:
- bases/infra.sealos.io_infras.yaml
- bases/infra.sealos.io_infraresources.yaml
#+kubebuilder:scaffold:crdkustomizeresource
patchesStrategicMerge:
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
# patches here are for enabling the conversion webhook for each CRD
#- patches/webhook_in_infras.yaml
#- patches/webhook_in_infraresources.yaml
#+kubebuilder:scaffold:crdkustomizewebhookpatch
# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
# patches here are for enabling the CA injection for each CRD
#- patches/cainjection_in_infras.yaml
#- patches/cainjection_in_infraresources.yaml
#+kubebuilder:scaffold:crdkustomizecainjectionpatch
# the following config is for teaching kustomize how to do kustomization for CRDs.
controllers/infra/config/crd/patches/cainjection_in_infraresources.yaml
+7
View File
@@ -0,0 +1,7 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
name: infraresources.infra.sealos.io
controllers/infra/config/crd/patches/webhook_in_infraresources.yaml
+16
View File
@@ -0,0 +1,16 @@
# The following patch enables a conversion webhook for the CRD
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: infraresources.infra.sealos.io
spec:
conversion:
strategy: Webhook
webhook:
clientConfig:
service:
namespace: system
name: webhook-service
path: /convert
conversionReviewVersions:
- v1
controllers/infra/config/rbac/infraresource_editor_role.yaml
+31
View File
@@ -0,0 +1,31 @@
# permissions for end users to edit infraresources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: clusterrole
app.kubernetes.io/instance: infraresource-editor-role
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: controller
app.kubernetes.io/part-of: controller
app.kubernetes.io/managed-by: kustomize
name: infraresource-editor-role
rules:
- apiGroups:
- infra.sealos.io
resources:
- infraresources
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- infra.sealos.io
resources:
- infraresources/status
verbs:
- get
controllers/infra/config/rbac/infraresource_viewer_role.yaml
+27
View File
@@ -0,0 +1,27 @@
# permissions for end users to view infraresources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: clusterrole
app.kubernetes.io/instance: infraresource-viewer-role
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: controller
app.kubernetes.io/part-of: controller
app.kubernetes.io/managed-by: kustomize
name: infraresource-viewer-role
rules:
- apiGroups:
- infra.sealos.io
resources:
- infraresources
verbs:
- get
- list
- watch
- apiGroups:
- infra.sealos.io
resources:
- infraresources/status
verbs:
- get
controllers/infra/config/rbac/role.yaml
+62
View File
@@ -5,6 +5,18 @@ metadata:
creationTimestamp: null
name: manager-role
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
@@ -17,6 +29,32 @@ rules:
- patch
- update
- watch
- apiGroups:
- infra.sealos.io
resources:
- infraresources
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- infra.sealos.io
resources:
- infraresources/finalizers
verbs:
- update
- apiGroups:
- infra.sealos.io
resources:
- infraresources/status
verbs:
- get
- patch
- update
- apiGroups:
- infra.sealos.io
resources:
@@ -43,3 +81,27 @@ rules:
- get
- patch
- update
- apiGroups:
- metering.common.sealos.io
resources:
- extensionresourceprices
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- metering.common.sealos.io
resources:
- resources
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
controllers/infra/config/samples/infra_aliyun.yaml
+10 -16
View File
@@ -20,24 +20,18 @@ spec:
platform: aliyun
hosts:
- roles: [ master ]
count: 2
flavor: ecs.s6-c1m1.small
image: "centos_7_9_x64_20G_alibase_20230109.vhd"
disks:
- capacity: 33
volumeType: "cloud_efficiency"
type: "root"
- capacity: 34
volumeType: "cloud_efficiency"
type: "data"
# - capacity: 35
# volumeType: gp3
# type: "data"
- roles: [ node ]
count: 1
flavor: ecs.s6-c1m1.small
flavor: ecs.c7.large
image: "centos_7_9_x64_20G_alibase_20230109.vhd"
disks:
- capacity: 40
volumeType: "cloud_efficiency"
volumeType: "cloud_essd"
type: "root"
- roles: [ node ]
count: 1
flavor: ecs.c7.large
image: "centos_7_9_x64_20G_alibase_20230109.vhd"
disks:
- capacity: 40
volumeType: "cloud_essd"
type: "root"
controllers/infra/config/samples/infra_v1_infraresource.yaml
+23
View File
@@ -0,0 +1,23 @@
apiVersion: infra.sealos.io/v1
kind: InfraResource
metadata:
name: infraresource-sample
namespace: infra-system
spec:
resourceName: infra
interval: 60
resources:
infra/CPU:
unit: "10"
price: 112
describe: "cost of per cpu per hour"
infra/Memory:
unit: "10Gi"
price: 55
describe: "cost of per memory per hour"
infra/Volume:
unit: "100Gi"
price: 35
describe: "cost of per 1G volume per hour"
controllers/infra/controllers/infra_controller.go
+3 -4
View File
@@ -117,7 +117,6 @@ func (r *InfraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
}
r.recorder.Eventf(infra, corev1.EventTypeNormal, "start to reconcile instance", "%s/%s", infra.Namespace, infra.Name)
if err := r.applier.ReconcileInstance(infra, r.driver[infra.Spec.Platform]); err != nil {
r.recorder.Eventf(infra, corev1.EventTypeWarning, "reconcile infra failed", "%v", err)
return err
@@ -197,9 +196,9 @@ func (r *InfraReconciler) SetupWithManager(mgr ctrl.Manager, opts InfraReconcile
var err error
for _, driverName := range common.DriverList {
r.driver[driverName], err = drivers.NewDriver(driverName)
}
if err != nil {
return fmt.Errorf("new driver error: %v", err)
if err != nil {
return fmt.Errorf("new %s driver error: %v", driverName, err)
}
}
return ctrl.NewControllerManagedBy(mgr).
For(&infrav1.Infra{}, builder.WithPredicates(
controllers/infra/controllers/infraresource_controller.go
+272
View File
@@ -0,0 +1,272 @@
/*
Copyright 2022 labring.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
"encoding/json"
"fmt"
"math"
"time"
meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
"k8s.io/apimachinery/pkg/api/resource"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/predicate"
v1 "k8s.io/api/core/v1"
"github.com/labring/sealos/controllers/infra/common"
"golang.org/x/sync/errgroup"
"github.com/go-logr/logr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
infrav1 "github.com/labring/sealos/controllers/infra/api/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// InfraResourceReconciler reconciles a InfraResource object
type InfraResourceReconciler struct {
client.Client
Scheme *runtime.Scheme
logr.Logger
}
type InfraData struct {
Flavor string
Volume []VolumeData
Count int
}
type VolumeData struct {
Capacity int
VolumeType string
}
//+kubebuilder:rbac:groups=infra.sealos.io,resources=infraresources,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=infra.sealos.io,resources=infraresources/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=infra.sealos.io,resources=infraresources/finalizers,verbs=update
//+kubebuilder:rbac:groups=metering.common.sealos.io,resources=resources,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=metering.common.sealos.io,resources=extensionresourceprices,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;create;update;patch;delete
func (r *InfraResourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
infraController := &infrav1.InfraResource{}
err := r.Get(ctx, req.NamespacedName, infraController)
if err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Register the resource to metering-quota
err = r.CreateOrUpdateExtensionResourcesPrice(ctx, infraController)
if err != nil {
r.Logger.Error(err, "CreateOrUpdateExtensionResourcesPrice failed")
return ctrl.Result{Requeue: true, RequeueAfter: time.Second}, client.IgnoreNotFound(err)
}
// update resource used every infraController.Spec.Interval Minutes
if time.Now().Unix()-infraController.Status.LatestUpdateTime >= int64(time.Minute.Minutes())*int64(infraController.Spec.Interval) {
if err := r.UpdateResourceUsed(ctx, infraController); err != nil {
r.Logger.Error(err, "UpdateResourceUsed failed")
return ctrl.Result{Requeue: true}, client.IgnoreNotFound(err)
}
}
return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(infraController.Spec.Interval) * time.Second}, nil
}
// CreateOrUpdateExtensionResourcesPrice need to create a ExtensionResourcesPrice to make metering-quota know this resource
func (r *InfraResourceReconciler) CreateOrUpdateExtensionResourcesPrice(ctx context.Context, obj client.Object) error {
infraController := obj.(*infrav1.InfraResource)
extensionResourcesPrice := &meteringcommonv1.ExtensionResourcePrice{
ObjectMeta: metav1.ObjectMeta{
Namespace: common.InfraSystemNamespace,
Name: meteringcommonv1.ExtensionResourcePricePrefix + infrav1.InfraResourcePricePrefix,
},
}
r.Logger.Info("create or update extensionResourcePrice", "infraController name", infraController.Name, "infraController Resources", infraController.Spec.Resources)
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, extensionResourcesPrice, func() error {
extensionResourcesPrice.Spec.Resources = infraController.Spec.Resources
extensionResourcesPrice.Spec.ResourceName = infraController.Spec.ResourceName
extensionResourcesPrice.Spec.GroupVersionKinds = infrav1.DefaultInfraResourceGVK
return controllerutil.SetControllerReference(infraController, extensionResourcesPrice, r.Scheme)
}); err != nil {
return fmt.Errorf("sync ExtensionResourcesPrice failed: %v", err)
}
r.Logger.V(1).Info("sync extensionResourcesPrice", "extensionResourcesPrice.Spec", extensionResourcesPrice.Spec)
return nil
}
// UpdateResourceUsed generate a resourceUsed per infra per interval
func (r *InfraResourceReconciler) UpdateResourceUsed(ctx context.Context, obj client.Object) error {
infraController := obj.(*infrav1.InfraResource)
var infraList infrav1.InfraList
eg, _ := errgroup.WithContext(context.Background())
// List all infras
err := r.List(ctx, &infraList)
if err != nil {
return err
}
// sync resource used for each infra
for i := range infraList.Items {
infra := infraList.Items[i]
// if infra is not running, skip it
if !r.checkInfraStatusRunning(infra) {
continue
}
eg.Go(func() error {
// sync resource
return r.syncInfraResource(ctx, infra, infraController)
})
if err := eg.Wait(); err != nil {
return err
}
}
// update infraController status
if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
if err := r.Get(ctx, types.NamespacedName{Namespace: infraController.Namespace, Name: infraController.Name}, infraController); err != nil {
return client.IgnoreNotFound(err)
}
infraController.Status.LatestUpdateTime = time.Now().Unix()
infraController.Status.SeqID++
return r.Status().Update(ctx, infraController)
}); err != nil {
r.Logger.Error(err, "UpdateResourceUsed failed")
return fmt.Errorf("infra controller update err:%v", err)
}
r.Logger.Info("infra controller calculate resource success")
return nil
}
// checkInfraStatus check infra status
func (r *InfraResourceReconciler) checkInfraStatusRunning(infra infrav1.Infra) bool {
if infra.Status.Status == infrav1.Running.String() {
return true
}
r.Logger.Info("infra status is not running", "infra name", infra.Name, "infra namespace", infra.Namespace, "infra status", infra.Status.Status)
return false
}
// syncInfraResource sync resource used for each infra
func (r *InfraResourceReconciler) syncInfraResource(ctx context.Context, infra infrav1.Infra, infraController *infrav1.InfraResource) error {
quantityMap := make(map[v1.ResourceName]int64)
for _, host := range infra.Spec.Hosts {
cnt := int64(host.Count)
flavor := host.Flavor
quantityMap[common.CPUResourceName] += common.CPUMap[flavor] * cnt
quantityMap[common.MemoryResourceName] += common.MemoryMap[flavor] * cnt
for _, disk := range host.Disks {
quantityMap[common.VolumeResourceName] += int64(disk.Capacity)
}
quantityMap[common.VolumeResourceName] = quantityMap[common.VolumeResourceName] * int64(host.Count)
}
r.Logger.Info("infra resources", "cpu quantity", quantityMap[common.CPUResourceName], "memory quantity", quantityMap[common.MemoryResourceName], "volume quantity", quantityMap[common.VolumeResourceName])
infraResource := meteringcommonv1.Resource{
ObjectMeta: metav1.ObjectMeta{
Name: GetResourceName(&infra, infraController.Status.SeqID),
Namespace: common.InfraSystemNamespace,
},
}
infraData, err := r.syncInfraData(infra)
if err != nil {
return fmt.Errorf("sync infra data failed: %v", err)
}
infraResource.Annotations = map[string]string{
common.InfraDataAnnotation: infraData,
}
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, &infraResource, func() error {
if infraResource.Spec.Resources == nil {
infraResource.Spec.Resources = make(map[v1.ResourceName]meteringcommonv1.ResourceInfo)
}
for k, v := range infraController.Spec.Resources {
var quantity *resource.Quantity
var cost int64
if k == common.CPUResourceName {
quantity = resource.NewQuantity(quantityMap[k], resource.BinarySI)
cost = int64(math.Ceil(float64(quantityMap[k]) * float64(v.Price) / 10))
} else if k == common.MemoryResourceName {
quantity = resource.NewQuantity(quantityMap[k]<<30, resource.BinarySI)
cost = int64(math.Ceil(float64(quantityMap[k]) * float64(v.Price) / 10))
} else {
quantity = resource.NewQuantity(quantityMap[k]<<30, resource.BinarySI)
cost = int64(math.Ceil(float64(quantityMap[k]) * float64(v.Price) / 100))
}
infraResource.Spec.Resources[k] = meteringcommonv1.ResourceInfo{
Used: quantity,
Timestamp: time.Now().Unix(),
Namespace: infra.Namespace,
Cost: cost,
}
}
return nil
}); err != nil {
return err
}
return nil
}
func (r *InfraResourceReconciler) syncInfraData(infra infrav1.Infra) (string, error) {
var infraData []InfraData
var volumeData []VolumeData
for _, host := range infra.Spec.Hosts {
for _, disk := range host.Disks {
volumeData = append(volumeData, VolumeData{
Capacity: disk.Capacity,
VolumeType: disk.VolumeType,
})
}
infraData = append(infraData, InfraData{
Flavor: host.Flavor,
Volume: volumeData,
Count: host.Count,
})
}
infraDataStr, err := json.Marshal(infraData)
if err != nil {
return "", fmt.Errorf("marshal infra data err:%v", err)
}
return string(infraDataStr), nil
}
// GetResourceName get resource name
func GetResourceName(infra *infrav1.Infra, seqID int64) string {
return fmt.Sprintf("%s-%s-%v", infra.Namespace, infra.Name, seqID)
}
// SetupWithManager sets up the controller with the Manager.
func (r *InfraResourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
const controllerName = "InfraResource-controller"
r.Logger = ctrl.Log.WithName(controllerName)
r.Logger.V(1).Info("reconcile InfraResource-controller")
return ctrl.NewControllerManagedBy(mgr).
For(&infrav1.InfraResource{}, builder.WithPredicates(
predicate.Or(predicate.GenerationChangedPredicate{}))).
Complete(r)
}
controllers/infra/deploy/README.md
+4
View File
@@ -12,6 +12,10 @@ data:
AWS_ACCESS_KEY_ID: xxxx
AWS_SECRET_ACCESS_KEY: xxxxx
AWS_DEFAULT_REGION: xxxx
ALIYUN_ACCESS_KEY_ID: xxxx
ALIYUN_ACCESS_KEY_SECRET: xxxx
ALIYUN_REGION_ID: xxxx
ALIYUN_RESOURCE_GROUP_ID: xxxx
```
controllers/infra/deploy/manifests/deploy.yaml
+140
View File
@@ -7,6 +7,84 @@ metadata:
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
creationTimestamp: null
name: infraresources.infra.sealos.io
spec:
group: infra.sealos.io
names:
kind: InfraResource
listKind: InfraResourceList
plural: infraresources
singular: infraresource
scope: Namespaced
versions:
- name: v1
schema:
openAPIV3Schema:
description: InfraResource is the Schema for the Infraresources API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: InfraResourceSpec defines the desired state of InfraResource
properties:
interval:
description: update used resources every Interval minutes
minimum: 1
type: integer
resourceName:
type: string
resources:
additionalProperties:
properties:
describe:
type: string
price:
format: int64
type: integer
unit:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
required:
- price
- unit
type: object
type: object
type: object
status:
description: InfraResourceStatus defines the observed state of InfraResource
properties:
latestUpdateTime:
format: int64
type: integer
seqID:
format: int64
type: integer
type: object
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
@@ -260,6 +338,18 @@ metadata:
creationTimestamp: null
name: infra-manager-role
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
@@ -272,6 +362,32 @@ rules:
- patch
- update
- watch
- apiGroups:
- infra.sealos.io
resources:
- infraresources
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- infra.sealos.io
resources:
- infraresources/finalizers
verbs:
- update
- apiGroups:
- infra.sealos.io
resources:
- infraresources/status
verbs:
- get
- patch
- update
- apiGroups:
- infra.sealos.io
resources:
@@ -298,6 +414,30 @@ rules:
- get
- patch
- update
- apiGroups:
- metering.common.sealos.io
resources:
- extensionresourceprices
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- metering.common.sealos.io
resources:
- resources
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
controllers/infra/drivers/get_instance_test.go
+1 -1
View File
@@ -33,7 +33,7 @@ func TestDriver_GetInstances(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "sealos-infra",
Namespace: "sealos-infra-ns",
UID: "0abafc31-735b-4a9c-923f-493af2ed1b25",
UID: "b5919c2f-969b-47fd-b9a1-d8d36c28049a",
},
Spec: v1.InfraSpec{
AvailabilityZone: "cn-hangzhou-i",
controllers/infra/go.mod
+1 -1
View File
@@ -97,4 +97,4 @@ require (
replace (
github.com/labring/sealos => ../../
github.com/labring/sealos/controllers/cluster => ../../controllers/cluster
)
)
controllers/infra/go.sum
+1 -1
View File
@@ -961,4 +961,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ih
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
controllers/infra/main.go
+11
View File
@@ -20,6 +20,7 @@ import (
"flag"
"os"
meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -45,6 +46,9 @@ func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(infrav1.AddToScheme(scheme))
utilruntime.Must(meteringcommonv1.AddToScheme(scheme))
//+kubebuilder:scaffold:scheme
}
@@ -100,6 +104,13 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "Infra")
os.Exit(1)
}
if err = (&controllers.InfraResourceReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "InfraResource")
os.Exit(1)
}
//+kubebuilder:scaffold:builder
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
controllers/infra/testdata/api/extension_resource_price.go
Vendored
+50
View File
@@ -0,0 +1,50 @@
package api
import (
meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
baseapi "github.com/labring/sealos/test/testdata/api"
)
const ExtensionResourcePriceYaml = `
apiVersion: metering.common.sealos.io/v1
kind: ExtensionResourcePrice
metadata:
name: extensionresourceprice-sealos-infra-controller
namespace: infra-system
spec:
resourceName: infra
resources:
infra/CPU:
describe: cost of per cpu per hour
price: 670
unit: '1'
infra/Memory:
describe: cost of per memory per hour
price: 330
unit: 1G
infra/Volume:
describe: cost of per 1G volume per hour
price: 21
unit: 1G
`
func GetExtensionResourcePrice(namespace string, name string) (*meteringcommonv1.ExtensionResourcePrice, error) {
gvr := meteringcommonv1.GroupVersion.WithResource("extensionresourceprices")
var extensionResourcesPrice meteringcommonv1.ExtensionResourcePrice
if err := baseapi.GetObject(namespace, name, gvr, &extensionResourcesPrice); err != nil {
return nil, err
}
return &extensionResourcesPrice, nil
}
func DeleteExtensionResourcePrice(namespace string, name string) error {
_, err := baseapi.KubeDeleteFromTemplate(ExtensionResourcePriceYaml, map[string]string{
"namespace": namespace,
"name": name,
})
if err != nil {
return err
}
return nil
}
controllers/infra/testdata/api/resource.go
Vendored
+79
View File
@@ -0,0 +1,79 @@
package api
import (
"fmt"
"time"
meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
baseapi "github.com/labring/sealos/test/testdata/api"
)
const ResourceYaml = `
apiVersion: infra.sealos.io/v1
kind: InfraResource
metadata:
name: ${name}
namespace: ${namespace}
spec:
resourceName: infra
interval: 60
resources:
infra/CPU:
unit: "10"
price: 112
describe: "cost of per cpu per hour"
infra/Memory:
unit: "10Gi"
price: 55
describe: "cost of per memory per hour"
infra/Volume:
unit: "100Gi"
price: 35
describe: "cost of per 1G volume per hour"
`
func GetResource(namespace string, name string) (*meteringcommonv1.Resource, error) {
gvr := meteringcommonv1.GroupVersion.WithResource("resources")
var resource meteringcommonv1.Resource
if err := baseapi.GetObject(namespace, name, gvr, &resource); err != nil {
return nil, err
}
return &resource, nil
}
func SyncResource(namespace string, name string) error {
_, err := baseapi.KubeApplyFromTemplate(ResourceYaml, map[string]string{
"namespace": namespace,
"name": name,
})
if err != nil {
return err
}
return nil
}
func EnsureResourceCreate(namespace string, name string, times int) (*meteringcommonv1.Resource, error) {
time.Sleep(time.Second)
for i := 1; i <= times; i++ {
resource, err := GetResource(namespace, name)
if err != nil {
time.Sleep(time.Second)
continue
}
return resource, nil
}
return nil, fmt.Errorf("resource create failed")
}
func DeleteResource(namespace string, name string) error {
_, err := baseapi.KubeDeleteFromTemplate(ResourceYaml, map[string]string{
"namespace": namespace,
"name": name,
})
if err != nil {
return err
}
return nil
}
controllers/infra/testdata/e2e/infra_metering_test.go
Vendored
+102
View File
@@ -0,0 +1,102 @@
package e2e
import (
"fmt"
"log"
"testing"
"time"
infrav1 "github.com/labring/sealos/controllers/infra/api/v1"
v1 "k8s.io/api/core/v1"
"github.com/labring/sealos/controllers/infra/common"
"github.com/labring/sealos/controllers/infra/controllers"
meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
"github.com/labring/sealos/controllers/infra/testdata/api"
baseapi "github.com/labring/sealos/test/testdata/api"
)
func TestInfraMetering(t *testing.T) {
baseapi.EnsureNamespace(TestNamespace)
t.Run("infra controller test", func(t *testing.T) {
time.Sleep(5 * time.Second)
baseapi.EnsureNamespace(TestNamespace)
baseapi.EnsureNamespace(InfraSystemNamespace)
t.Log("create infra and wait it to run")
if err := api.CreateInfra(TestNamespace, InfraName); err != nil {
t.Fatalf("create infra failed: %v", err)
}
if err := api.WaitInfraRunning(TestNamespace, InfraName, 100); err != nil {
t.Fatalf("infra failed to run: %v", err)
}
t.Log("infra succeeded to run")
t.Log("sync resource")
if err := api.SyncResource(InfraSystemNamespace, InfraResourceName); err != nil {
t.Fatalf("failed to sync resource: %v", err)
}
time.Sleep(5 * time.Second)
t.Log("ensure extension resource is created")
_, err := api.GetExtensionResourcePrice(InfraSystemNamespace, meteringcommonv1.GetExtensionResourcePriceName(infrav1.InfraResourcePricePrefix))
if err != nil {
t.Fatalf("failed to get extension resource: %v", err)
}
t.Log("ensure resource CR is created")
infra, _ := api.GetInfra(TestNamespace, InfraName)
resource, err := api.EnsureResourceCreate(InfraSystemNamespace, controllers.GetResourceName(infra, 1), 90)
if err != nil {
resource, err = api.EnsureResourceCreate(InfraSystemNamespace, controllers.GetResourceName(infra, 2), 90)
if err != nil {
t.Fatalf(err.Error())
}
}
//check used resource
if err = checkResourceUsed(resource, common.CPUResourceName); err != nil {
t.Fatalf(err.Error())
}
if err = checkResourceUsed(resource, common.MemoryResourceName); err != nil {
t.Fatalf(err.Error())
}
if err = checkResourceUsed(resource, common.VolumeResourceName); err != nil {
t.Fatalf(err.Error())
}
t.Log(resource)
})
t.Cleanup(cleanInfraMetering)
}
func checkResourceUsed(resource *meteringcommonv1.Resource, resourceName v1.ResourceName) error {
if _, ok := resource.Spec.Resources[resourceName]; !ok {
return fmt.Errorf("%s resource not found", resourceName)
}
if resource.Spec.Resources[resourceName].Used.Value() != 0 {
fmt.Printf("resource %s has been used %v\n", resourceName, resource.Spec.Resources[resourceName].Used.Value())
}
return nil
}
func cleanInfraMetering() {
fmt.Println("clean infra metering")
//execout, err := baseapi.Exec("kubectl get resource -A -oyaml")
//log.Println(execout)
//if err != nil {
// log.Println(err)
//}
if err := api.DeleteInfra(TestNamespace, InfraName); err != nil {
log.Println(err)
}
if err := api.DeleteResource(InfraSystemNamespace, InfraResourceName); err != nil {
log.Println(err)
}
if err := baseapi.DeleteNamespace(TestNamespace); err != nil {
log.Println(err)
}
}
controllers/infra/testdata/e2e/infra_test.go
Vendored
+8 -6
View File
@@ -12,8 +12,10 @@ import (
)
const (
TestNamespace = "ns-infra-test"
InfraName = "infra-test"
TestNamespace = "ns-infra-test"
InfraName = "infra-test"
InfraSystemNamespace = "infra-system"
InfraResourceName = "infra-resource"
)
func TestInfra(t *testing.T) {
@@ -50,14 +52,14 @@ func TestInfra(t *testing.T) {
}
t.Log("cluster succeeded to run")
t.Cleanup(clean)
t.Cleanup(cleanInfra)
cnt++
})
}
t.Cleanup(cleanAll)
t.Cleanup(cleanAllInfra)
}
func clean() {
func cleanInfra() {
if err := api.DeleteCluster(TestNamespace, InfraName); err != nil {
log.Println(err)
}
@@ -67,7 +69,7 @@ func clean() {
}
}
func cleanAll() {
func cleanAllInfra() {
if err := baseapi.DeleteNamespace(TestNamespace); err != nil {
log.Println(err)
}
go.work.sum
+9
View File
@@ -489,6 +489,7 @@ github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c=
github.com/aliyun/alibaba-cloud-sdk-go v1.62.198/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
@@ -600,6 +601,7 @@ github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGF
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
@@ -628,7 +630,11 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
github.com/intel/goresctrl v0.3.0/go.mod h1:fdz3mD85cmP9sHD8JUlrNWAxvwM86CrbmVXltEKd7zk=
github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
@@ -674,6 +680,7 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221014010322-58c91d646d86/go.
github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo=
github.com/opencontainers/selinux v1.10.2/go.mod h1:cARutUbaUrlRClyvxOICCgKixCs6L05aUsohzA3EkHQ=
github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df/go.mod h1:TRYHe4CH9U6nkDjxjBNM5klrLbJBrRbpJE5SaRwUBsQ=
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
@@ -699,6 +706,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.22.7/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=