Delete useless code (#4340)

* delete monitor-metering useless code

* delete monitor-metering ci workflow
This commit is contained in:
Jiahui
2023-11-23 16:21:39 +08:00
committed by GitHub
parent 520c69ded6
commit 4efa9d802e
10 changed files with 0 additions and 493 deletions
-2
View File
@@ -97,7 +97,6 @@ jobs:
- { name: license, path: license }
- { name: job-init, path: job/init }
- { name: resources, path: resources }
- { name: resources-metering, path: resources/metering }
- { name: node, path: node }
- { name: admission, path: admission }
steps:
@@ -199,7 +198,6 @@ jobs:
- { name: license, path: license }
- { name: job-init, path: job/init }
- { name: resources, path: resources }
- { name: resources-metering, path: resources/metering }
- { name: node, path: node }
- { name: admission, path: admission }
steps:
-5
View File
@@ -24,9 +24,7 @@ import (
type Interface interface {
//InitDB() error
GetMeteringOwnerTimeResult(queryTime time.Time, queryCategories, queryProperties []string) (*MeteringOwnerTimeResult, error)
GetBillingLastUpdateTime(owner string, _type accountv1.Type) (bool, time.Time, error)
//TODO will delete this
GetBillingHistoryNamespaceList(ns *accountv1.NamespaceBillingHistorySpec, owner string) ([]string, error)
GetBillingHistoryNamespaces(startTime, endTime *time.Time, billType int, owner string) ([]string, error)
SaveBillings(billing ...*resources.Billing) error
@@ -38,8 +36,6 @@ type Interface interface {
InitDefaultPropertyTypeLS() error
SavePropertyTypes(types []resources.PropertyType) error
GetBillingCount(accountType accountv1.Type, startTime, endTime time.Time) (count, amount int64, err error)
//TODO delete
GenerateMeteringData(startTime, endTime time.Time, prices map[string]resources.Price) error
GenerateBillingData(startTime, endTime time.Time, prols *resources.PropertyTypeLS, namespaces []string, owner string) (orderID []string, amount int64, err error)
InsertMonitor(ctx context.Context, monitors ...*resources.Monitor) error
DropMonitorCollectionsOlderThan(days int) error
@@ -51,7 +47,6 @@ type Creator interface {
CreateBillingIfNotExist() error
//suffix by day, eg: monitor_20200101
CreateMonitorTimeSeriesIfNotExist(collTime time.Time) error
CreateMeteringTimeSeriesIfNotExist() error
}
type MeteringOwnerTimeResult struct {
-63
View File
@@ -235,64 +235,6 @@ func (m *MongoDB) SaveBillings(billing ...*resources.Billing) error {
return err
}
func (m *MongoDB) GetMeteringOwnerTimeResult(queryTime time.Time, queryCategories, queryProperties []string) (*MeteringOwnerTimeResult, error) {
matchValue := bson.M{
"time": queryTime,
"category": bson.M{"$in": queryCategories},
}
if len(queryProperties) > 0 {
matchValue["property"] = bson.M{"$in": queryProperties}
}
pipeline := bson.A{
bson.D{{Key: "$match", Value: matchValue}},
bson.D{{Key: "$group", Value: bson.M{
"_id": bson.M{"property": "$property"},
"propertyTotal": bson.M{"$sum": "$amount"},
}}},
bson.D{{Key: "$project", Value: bson.M{
"_id": 0,
"property": "$_id.property",
"propertyTotal": 1,
}}},
bson.D{{Key: "$group", Value: bson.M{
"_id": nil,
"amountTotal": bson.M{"$sum": "$propertyTotal"},
"costs": bson.M{"$push": bson.M{"k": "$property", "v": "$propertyTotal"}},
}}},
bson.D{{Key: "$addFields", Value: bson.M{
//"owner": queryOwner,
"time": queryTime,
"amount": "$amountTotal",
"costs": bson.M{"$arrayToObject": "$costs"},
}}},
}
/*
db.metering.aggregate([
{ $match:
{ time: queryTime, category:
{ $in: ["ns-gxqoxr8s"] }, property: { $in: ["cpu", "memory", "storage"] } } },
{ $group: { _id: { property: "$property" }, propertyTotal: { $sum: "$amount" } } },
{ $project: { _id: 0, property: "$_id.property", propertyTotal: 1 } },
{ $group: { _id: null, amountTotal: { $sum: "$propertyTotal" }, costs: { $push: { k: "$property", v: "$propertyTotal" } } } },
{ $addFields: { orderId: "111111111", own: queryOwn, time: queryTime, type: 0, amount: "$amountTotal", costs: { $arrayToObject: "$costs" } } },
{ $out: "results1" }]);
*/
cursor, err := m.getMeteringCollection().Aggregate(context.Background(), pipeline)
if err != nil {
return nil, err
}
defer cursor.Close(context.Background())
if cursor.Next(context.Background()) {
var result MeteringOwnerTimeResult
if err = cursor.Decode(&result); err != nil {
return nil, err
}
return &result, nil
}
return nil, nil
}
// InsertMonitor insert monitor data to mongodb collection monitor + time (eg: monitor_20200101)
// The monitor data is saved daily 2020-12-01 00:00:00 - 2020-12-01 23:59:59 => monitor_20201201
func (m *MongoDB) InsertMonitor(ctx context.Context, monitors ...*resources.Monitor) error {
@@ -990,11 +932,6 @@ func (m *MongoDB) CreateMonitorTimeSeriesIfNotExist(collTime time.Time) error {
return m.CreateTimeSeriesIfNotExist(m.DBName, m.getMonitorCollectionName(collTime))
}
// CreateMeteringTimeSeriesIfNotExist creates the time series table for metering
func (m *MongoDB) CreateMeteringTimeSeriesIfNotExist() error {
return m.CreateTimeSeriesIfNotExist(m.DBName, m.MeteringConn)
}
func (m *MongoDB) CreateTimeSeriesIfNotExist(dbName, collectionName string) error {
// Check if the collection already exists
if exist, err := m.collectionExist(dbName, collectionName); exist || err != nil {
-24
View File
@@ -33,30 +33,6 @@ import (
accountv1 "github.com/labring/sealos/controllers/account/api/v1"
)
func TestMongoDB_GetMeteringOwnerTimeResult(t *testing.T) {
dbCTX := context.Background()
//"mongodb://192.168.64.21:27017/"
m, err := NewMongoDB(dbCTX, os.Getenv("MONGODB_URI"))
if err != nil {
t.Errorf("failed to connect mongo: error = %v", err)
}
defer func() {
if err = m.Disconnect(dbCTX); err != nil {
t.Errorf("failed to disconnect mongo: error = %v", err)
}
}()
now := time.Now()
queryTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local).UTC().Add(-time.Hour)
got, err := m.GetMeteringOwnerTimeResult(queryTime, []string{"ns-0nfm8mkr"}, nil)
if err != nil {
t.Errorf("failed to get metering owner time result: error = %v", err)
}
t.Logf("all properties got: %v", got)
}
func TestMongoDB_QueryBillingRecords(t *testing.T) {
dbCTX := context.Background()
-22
View File
@@ -1,22 +0,0 @@
# Copyright © 2023 sealos.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM gcr.io/distroless/static:nonroot
ARG TARGETARCH
WORKDIR /
USER 65532:65532
COPY bin/controller-resources-metering-$TARGETARCH /manager
ENTRYPOINT ["/manager", "start"]
-50
View File
@@ -1,50 +0,0 @@
# Image URL to use all building/pushing image targets
IMG ?= ghcr.io/labring/sealos-resources-controller:latest
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
ENVTEST_K8S_VERSION = 1.23
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
else
GOBIN=$(shell go env GOBIN)
endif
# Setting SHELL to bash allows bash commands to be executed by recipes.
# This is a requirement for 'setup-envtest.sh' in the test target.
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec
.PHONY: all
all: build
##@ Build
.PHONY: build
build: ## Build manager binary.
CGO_ENABLED=0 GOOS=linux go build $(shell [ -n "${CRYPTOKEY}" ] && echo "-ldflags '-X github.com/labring/sealos/controllers/pkg/crypto.encryptionKey=${CRYPTOKEY} -X github.com/labring/sealos/controllers/pkg/database.cryptoKey=${CRYPTOKEY}'") -o bin/manager main.go
.PHONY: run
run: ## Run a controller from your host.
go run ./main.go
.PHONY: docker-build
docker-build: test ## Build docker image with the manager.
docker build -t ${IMG} .
.PHONY: docker-push
docker-push: ## Push docker image with the manager.
docker push ${IMG}
##@ Deployment
ifndef ignore-not-found
ignore-not-found = false
endif
## Location to install dependencies to
LOCALBIN ?= $(shell pwd)/bin
$(LOCALBIN):
mkdir -p $(LOCALBIN)
@@ -1,12 +0,0 @@
FROM scratch
USER 65532:65532
COPY registry registry
COPY manifests manifests
ENV DEFAULT_NAMESPACE resources-system
ENV MONGO_URI "mongodb://mongo:27017/resources"
CMD ["kubectl apply -f manifests/deploy.yaml -n $DEFAULT_NAMESPACE && ( kubectl create -f manifests/mongo-secret.yaml -n $DEFAULT_NAMESPACE || true )"]
@@ -1,66 +0,0 @@
# Copyright © 2023 sealos.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: controller-manager
name: resources-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
control-plane: metering-manager
name: resources-metering-manager
namespace: resources-system
spec:
replicas: 1
selector:
matchLabels:
control-plane: metering-manager
template:
metadata:
labels:
control-plane: metering-manager
spec:
containers:
- image: ghcr.io/labring/sealos-resources-metering-controller:latest
name: resource-metering
command:
- "/manager"
- "start"
- "--show-path"
- "--debug"
resources:
limits:
cpu: 500m
memory: 128Mi
requests:
cpu: 5m
memory: 64Mi
securityContext:
runAsNonRoot: true
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
env:
- name: MONGO_URI
valueFrom:
secretKeyRef:
name: mongo-secret
key: MONGO_URI
imagePullPolicy: Always
@@ -1,7 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: mongo-secret
namespace: {{ .DEFAULT_NAMESPACE }}
stringData:
MONGO_URI: "{{ .MONGO_URI }}"
-242
View File
@@ -1,242 +0,0 @@
// Copyright © 2023 sealos.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/spf13/cobra"
"github.com/labring/sealos/controllers/pkg/database"
"github.com/labring/sealos/controllers/pkg/resources"
"github.com/labring/sealos/controllers/pkg/utils/env"
"github.com/labring/sealos/controllers/pkg/utils/flags"
"github.com/labring/sealos/controllers/pkg/utils/logger"
"github.com/labring/sealos/controllers/pkg/utils/retry"
)
type Config struct {
// mongodb connect url
MongoConnectURI string
MongoUsername string
MongoPassword string
RetentionDay int64
PermanentRetention bool
// interval of metering resources
Interval time.Duration
}
var config *Config
func ResourcesMetering() {
ctx, cancel := context.WithCancel(context.Background())
go func() {
// 信号监听
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
<-sigCh
logger.Info("Receiving signal, canceling...")
cancel()
}()
// 在启动时执行任务
logger.Info("start to execute metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05"))
if err := PreApply(); err != nil {
logger.Error(err.Error())
os.Exit(1)
}
logger.Info("success first metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05"))
go func() {
for {
// start execution beginning of every hour
durationToNextHour := time.Until(time.Now().Truncate(time.Hour).Add(time.Hour))
select {
case <-time.After(durationToNextHour):
go func() {
logger.Debug("Start to execute metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05"))
if err := retry.Retry(5, time.Minute, executeTask); err != nil {
logger.Error("Failed to execute task: %v", err)
return
}
logger.Debug("Success metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05"))
}()
case <-ctx.Done():
return
}
}
}()
if !config.PermanentRetention {
ticker := time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
if err := DropMonitorCollectionOlder(); err != nil {
logger.Error("Failed to drop monitor collection older than %d days: %v", config.RetentionDay, err)
}
}
}()
}
<-ctx.Done()
logger.Info("program exit")
}
func DropMonitorCollectionOlder() error {
dbCtx := context.Background()
dbClient, err := database.NewMongoDB(dbCtx, config.MongoConnectURI)
if err != nil {
return fmt.Errorf("connect mongo client failed: %v", err)
}
defer func() {
err := dbClient.Disconnect(dbCtx)
if err != nil {
logger.Error("disconnect mongo client failed: %v", err)
}
}()
return dbClient.DropMonitorCollectionsOlderThan(int(config.RetentionDay))
}
func PreApply() error {
switch {
case config.MongoConnectURI == "":
return fmt.Errorf("mongo connect url is empty")
}
dbCtx := context.Background()
dbClient, err := database.NewMongoDB(dbCtx, config.MongoConnectURI)
if err != nil {
return fmt.Errorf("connect mongo client failed: %v", err)
}
defer func() {
err := dbClient.Disconnect(dbCtx)
if err != nil {
logger.Error("disconnect mongo client failed: %v", err)
}
}()
if err = dbClient.CreateMeteringTimeSeriesIfNotExist(); err != nil {
logger.Warn("create metering time series failed: %v", err)
}
if err = CreateMonitorTimeSeries(dbClient, time.Now().UTC()); err != nil {
logger.Warn("create monitor time series failed: %v", err)
}
return executeTask()
}
func executeTask() error {
//opts := options.Client().ApplyURI("mongodb://192.168.64.17:27017")
dbCtx := context.Background()
dbClient, err := database.NewMongoDB(dbCtx, config.MongoConnectURI)
if err != nil {
return fmt.Errorf("connect mongo client failed: %v", err)
}
defer func() {
err := dbClient.Disconnect(dbCtx)
if err != nil {
logger.Warn("disconnect mongo client failed: %v", err)
}
}()
prices, err := dbClient.GetAllPricesMap()
if err != nil {
logger.Error("failed to get all prices map: %v", err)
}
//prices is empty, use default price
if len(prices) == 0 || err != nil {
prices = resources.DefaultPrices
}
now := time.Now().UTC()
startTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour()-1, 0, 0, 0, time.UTC)
endTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.UTC)
if err := dbClient.GenerateMeteringData(startTime, endTime, prices); err != nil {
return fmt.Errorf("failed to generate metering data: %v", err)
}
// create tomorrow monitor time series
if err := CreateMonitorTimeSeries(dbClient, now.Add(24*time.Hour)); err != nil {
logger.Debug("failed to create monitor time series: %v", err)
}
return nil
}
func CreateMonitorTimeSeries(dbClient database.Interface, collTime time.Time) error {
return dbClient.CreateMonitorTimeSeriesIfNotExist(collTime)
}
func main() {
Execute()
}
var (
debug bool
showPath bool
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "metering",
Short: "program for sealos resources metering.",
// Uncomment the following line if your bare application
// has an action associated with it:
// Run: func(cmd *cobra.Command, args []string) { },
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func newStartCmd() *cobra.Command {
//type startFlag struct {
//}
//var flag startFlag
startCmd := &cobra.Command{
Use: "start",
Short: "start metering",
Long: `start metering`,
Run: func(cmd *cobra.Command, args []string) {
flags.PrintFlags(cmd.Flags())
ResourcesMetering()
},
}
return startCmd
}
func init() {
cobra.OnInitialize(func() {
logger.CfgConsoleLogger(debug, showPath)
})
rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debug logger")
rootCmd.PersistentFlags().BoolVar(&showPath, "show-path", false, "enable show code path")
rootCmd.AddCommand(newStartCmd())
config = &Config{
MongoConnectURI: os.Getenv(database.MongoURI),
MongoUsername: os.Getenv(database.MongoUsername),
MongoPassword: os.Getenv(database.MongoPassword),
RetentionDay: env.GetInt64EnvWithDefault(database.RetentionDay, database.DefaultRetentionDay),
PermanentRetention: os.Getenv(database.PermanentRetention) == "true",
}
}