opensource/sealos
3
0
Fork 0
mirror of https://github.com/labring/sealos.git synced 2026-10-29 17:25:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

add quota monitor (#5167)

Browse Source 
* add quota monitor
This commit is contained in:
谢金虎
2024-10-28 14:57:11 +08:00
committed by GitHub
parent b300a0c945
commit d1424a7710
4 changed files with 227 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/exceptionmonitor/api/api.go
+7
View File
@@ -64,10 +64,12 @@ var (
DiskMonitor bool
CPUMemMonitor bool
BackupMonitor bool
QuotaMonitor bool
DatabaseDiskMonitorThreshold float64
DatabaseExceptionMonitorThreshold float64
DatabaseCPUMonitorThreshold float64
DatabaseMemMonitorThreshold float64
QuotaThreshold float64
APPID string
APPSECRET string
DatabaseStatusMessageIDMap = make(map[string]string)
@@ -75,6 +77,7 @@ var (
DatabaseCPUMessageIDMap = make(map[string]string)
DatabaseMemMessageIDMap = make(map[string]string)
DatabaseBackupMessageIDMap = make(map[string]string)
QuotaMessageIDMap = make(map[string]string)
)
func GetENV() error {
@@ -91,10 +94,12 @@ func GetENV() error {
DiskMonitor, _ = strconv.ParseBool(getEnvWithCheck("DiskMonitor", &missingEnvVars))
CPUMemMonitor, _ = strconv.ParseBool(getEnvWithCheck("CPUMemMonitor", &missingEnvVars))
BackupMonitor, _ = strconv.ParseBool(getEnvWithCheck("BackupMonitor", &missingEnvVars))
QuotaMonitor, _ = strconv.ParseBool(getEnvWithCheck("QuotaMonitor", &missingEnvVars))
DatabaseDiskMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseDiskMonitorThreshold", &missingEnvVars), 64)
DatabaseExceptionMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseExceptionMonitorThreshold", &missingEnvVars), 64)
DatabaseCPUMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseCPUMonitorThreshold", &missingEnvVars), 64)
DatabaseMemMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseMemMonitorThreshold", &missingEnvVars), 64)
QuotaThreshold, _ = strconv.ParseFloat(getEnvWithCheck("QuotaThreshold", &missingEnvVars), 64)
if clusterNS != "" {
ClusterNS = strings.Split(clusterNS, ",")
@@ -112,6 +117,8 @@ func GetENV() error {
"FeishuWebhookURLImportant",
//Backup
"FeishuWebhookURLBackup",
//Quota
"FeishuWebhookURLQuota",
}, FeishuWebhookURLMap, &missingEnvVars)
// Get ClusterRegionMap
service/exceptionmonitor/helper/monitor/quota_monitor.go
+127
View File
@@ -0,0 +1,127 @@
package monitor
import (
"context"
"fmt"
"log"
"strings"
"time"
"github.com/labring/sealos/service/exceptionmonitor/api"
"github.com/labring/sealos/service/exceptionmonitor/helper/notification"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func QuotaMonitor() {
for api.QuotaMonitor {
if err := checkQuota(); err != nil {
log.Printf("Failed to check qouta: %v", err)
}
time.Sleep(3 * time.Hour)
}
}
func checkQuota() error {
namespaceList, _ := api.ClientSet.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
fmt.Println(len(namespaceList.Items))
for _, ns := range namespaceList.Items {
if !strings.Contains(ns.Name, "ns-") {
continue
}
quotaList, err := api.ClientSet.CoreV1().ResourceQuotas(ns.Name).List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Printf("Failed to check quota: %v", err)
}
//user debt
if len(quotaList.Items) != 1 || quotaList.Items[0].Name != "quota-"+ns.Name {
continue
}
nsQuota := notification.NameSpaceQuota{
NameSpace: ns.Name,
}
notificationInfo := notification.Info{
ExceptionType: "Quota",
PerformanceType: "Quota",
NotificationType: "exception",
}
send := processQuota(quotaList, &nsQuota)
if send {
message := notification.GetQuotaMessage(&nsQuota)
if err := notification.SendFeishuNotification(notificationInfo, message, api.FeishuWebhookURLMap["FeishuWebhookURLQuota"]); err != nil {
log.Printf("Error sending exception notification:%v", err)
}
}
}
return nil
}
func processQuota(quotaList *v1.ResourceQuotaList, nsQuota *notification.NameSpaceQuota) bool {
send := false
for resourceName, hardQuantity := range quotaList.Items[0].Status.Hard {
usedQuantity, exists := quotaList.Items[0].Status.Used[resourceName]
if !exists {
usedQuantity = *resource.NewQuantity(0, hardQuantity.Format)
}
hardValue, err := quantityToFloat64(hardQuantity, string(resourceName))
if err != nil {
log.Printf("Analytic hard constraint error : %s , %v\n", resourceName, err)
continue
}
usedValue, err := quantityToFloat64(usedQuantity, string(resourceName))
if err != nil {
fmt.Printf("Analytic usage error : %s , %v\n", resourceName, err)
continue
}
// usage
utilization := (usedValue / hardValue) * 100
if utilization > api.QuotaThreshold {
send = true
switch resourceName {
case "limits.cpu":
nsQuota.CPULimit = hardQuantity.String()
nsQuota.CPUUsage = fmt.Sprintf("%.2f%%", utilization)
case "limits.memory":
nsQuota.MemLimit = hardQuantity.String()
nsQuota.MemUsage = fmt.Sprintf("%.2f%%", utilization)
case "limits.nvidia.com/gpu":
nsQuota.GPULimit = hardQuantity.String()
nsQuota.GPUUsage = fmt.Sprintf("%.2f%%", utilization)
case "limits.ephemeral-storage":
nsQuota.EphemeralStorageLimit = hardQuantity.String()
nsQuota.EphemeralStorageUsage = fmt.Sprintf("%.2f%%", utilization)
case "objectstorage/size":
nsQuota.ObjectStorageLimit = hardQuantity.String()
nsQuota.ObjectStorageUsage = fmt.Sprintf("%.2f%%", utilization)
case "services.nodeports":
nsQuota.NodePortLimit = hardQuantity.String()
nsQuota.NodePortUsage = fmt.Sprintf("%.2f%%", utilization)
case "requests.storage":
nsQuota.StorageLimit = hardQuantity.String()
nsQuota.StorageUsage = fmt.Sprintf("%.2f%%", utilization)
}
}
}
return send
}
func quantityToFloat64(q resource.Quantity, resourceName string) (float64, error) {
switch resourceName {
case "limits.cpu", "requests.cpu", "cpu",
"limits.nvidia.com/gpu", "requests.nvidia.com/gpu":
return float64(q.MilliValue()), nil
case "limits.memory", "requests.memory", "memory",
"limits.ephemeral-storage", "requests.ephemeral-storage",
"requests.storage", "objectstorage/size":
return float64(q.Value()), nil
case "services.nodeports":
return float64(q.Value()), nil
default:
return q.AsApproximateFloat64(), nil
}
}
service/exceptionmonitor/helper/notification/feishu.go
+92 -4
View File
@@ -35,6 +35,24 @@ type Info struct {
ExceptionType string
}
type NameSpaceQuota struct {
NameSpace string
CPULimit string
MemLimit string
GPULimit string
EphemeralStorageLimit string
ObjectStorageLimit string
NodePortLimit string
StorageLimit string
CPUUsage string
MemUsage string
GPUUsage string
EphemeralStorageUsage string
ObjectStorageUsage string
NodePortUsage string
StorageUsage string
}
func InitFeishuClient() {
feiShuClient = lark.NewClient(api.APPID, api.APPSECRET)
}
@@ -201,6 +219,8 @@ func getMessageIDMap(performanceType string) map[string]string {
return api.DatabaseCPUMessageIDMap
case "Backup":
return api.DatabaseBackupMessageIDMap
case "Quota":
return api.QuotaMessageIDMap
default:
return api.DatabaseStatusMessageIDMap
}
@@ -248,7 +268,7 @@ func createFeishuNotification(notification Info, message, feishuWebHook string,
respStr := larkcore.Prettify(resp)
messageID := extractAndPrintMessageID(respStr)
if notification.DatabaseClusterName == "Backup" {
if notification.PerformanceType == "Backup" || notification.PerformanceType == "Quota" {
return nil
}
if messageID == "" {
@@ -297,7 +317,75 @@ func createCard(headerTemplate, headerTitle string, elements []map[string]string
return card
}
func createElements(namespace, backupName, status, startTime, reason string, includeReason bool) []map[string]string {
func GetQuotaMessage(nsQuota *NameSpaceQuota) string {
var card map[string]interface{}
elements := createQuotaElements(nsQuota)
card = createCard("red", "Quota阀值通知", elements)
databaseMessage, err := marshalCard(card)
if err != nil {
fmt.Println(err)
return ""
}
return databaseMessage
}
func createQuotaElements(nsQuota *NameSpaceQuota) []map[string]string {
elements := []map[string]string{
{"label": "集群环境", "value": api.ClusterName},
{"label": "命名空间", "value": nsQuota.NameSpace},
}
addNonEmptyFieldsToElements(nsQuota, &elements)
return elements
}
func addNonEmptyFieldsToElements(nsQuota *NameSpaceQuota, elements *[]map[string]string) {
fields := map[string]string{
"CPULimit": "CPU总量",
"CPUUsage": "CPU使用率",
"MemLimit": "内存总量",
"MemUsage": "内存使用率",
"GPULimit": "GPU总量",
"GPUUsage": "GPU使用率",
"EphemeralStorageLimit": "临时存储总量",
"EphemeralStorageUsage": "临时存储使用率",
"ObjectStorageLimit": "对象存储总量",
"ObjectStorageUsage": "对象存储使用率",
"NodePortLimit": "节点端口总量",
"NodePortUsage": "节点端口使用率",
"StorageLimit": "存储总量",
"StorageUsage": "存储使用率",
}
quotaValues := map[string]string{
"CPULimit": nsQuota.CPULimit,
"CPUUsage": nsQuota.CPUUsage,
"MemLimit": nsQuota.MemLimit,
"MemUsage": nsQuota.MemUsage,
"GPULimit": nsQuota.GPULimit,
"GPUUsage": nsQuota.GPUUsage,
"EphemeralStorageLimit": nsQuota.EphemeralStorageLimit,
"EphemeralStorageUsage": nsQuota.EphemeralStorageUsage,
"ObjectStorageLimit": nsQuota.ObjectStorageLimit,
"ObjectStorageUsage": nsQuota.ObjectStorageUsage,
"NodePortLimit": nsQuota.NodePortLimit,
"NodePortUsage": nsQuota.NodePortUsage,
"StorageLimit": nsQuota.StorageLimit,
"StorageUsage": nsQuota.StorageUsage,
}
for field, label := range fields {
value := quotaValues[field]
if value != "" {
*elements = append(*elements, map[string]string{
"label": label,
"value": value,
})
}
}
}
func createBackupElements(namespace, backupName, status, startTime, reason string, includeReason bool) []map[string]string {
elements := []map[string]string{
{"label": "集群环境", "value": api.ClusterName},
{"label": "命名空间", "value": namespace},
@@ -322,10 +410,10 @@ func marshalCard(card map[string]interface{}) (string, error) {
func GetBackupMessage(notificationType, namespace, backupName, status, startTime, reason string) string {
var card map[string]interface{}
if notificationType == ExceptionType {
elements := createElements(namespace, backupName, status, startTime, reason, true)
elements := createBackupElements(namespace, backupName, status, startTime, reason, true)
card = createCard("red", "备份异常通知", elements)
} else {
elements := createElements(namespace, backupName, status, startTime, "", false)
elements := createBackupElements(namespace, backupName, status, startTime, "", false)
card = createCard("blue", "备份恢复通知", elements)
}
service/exceptionmonitor/main.go
+1
View File
@@ -18,6 +18,7 @@ func main() {
go monitor.DatabaseExceptionMonitor()
go monitor.DatabasePerformanceMonitor()
go monitor.DatabaseBackupMonitor()
go monitor.QuotaMonitor()
select {}
}