diff --git a/service/exceptionmonitor/api/api.go b/service/exceptionmonitor/api/api.go index 0a13961c5..1bc492559 100644 --- a/service/exceptionmonitor/api/api.go +++ b/service/exceptionmonitor/api/api.go @@ -64,10 +64,12 @@ var ( DiskMonitor bool CPUMemMonitor bool BackupMonitor bool + QuotaMonitor bool DatabaseDiskMonitorThreshold float64 DatabaseExceptionMonitorThreshold float64 DatabaseCPUMonitorThreshold float64 DatabaseMemMonitorThreshold float64 + QuotaThreshold float64 APPID string APPSECRET string DatabaseStatusMessageIDMap = make(map[string]string) @@ -75,6 +77,7 @@ var ( DatabaseCPUMessageIDMap = make(map[string]string) DatabaseMemMessageIDMap = make(map[string]string) DatabaseBackupMessageIDMap = make(map[string]string) + QuotaMessageIDMap = make(map[string]string) ) func GetENV() error { @@ -91,10 +94,12 @@ func GetENV() error { DiskMonitor, _ = strconv.ParseBool(getEnvWithCheck("DiskMonitor", &missingEnvVars)) CPUMemMonitor, _ = strconv.ParseBool(getEnvWithCheck("CPUMemMonitor", &missingEnvVars)) BackupMonitor, _ = strconv.ParseBool(getEnvWithCheck("BackupMonitor", &missingEnvVars)) + QuotaMonitor, _ = strconv.ParseBool(getEnvWithCheck("QuotaMonitor", &missingEnvVars)) DatabaseDiskMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseDiskMonitorThreshold", &missingEnvVars), 64) DatabaseExceptionMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseExceptionMonitorThreshold", &missingEnvVars), 64) DatabaseCPUMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseCPUMonitorThreshold", &missingEnvVars), 64) DatabaseMemMonitorThreshold, _ = strconv.ParseFloat(getEnvWithCheck("DatabaseMemMonitorThreshold", &missingEnvVars), 64) + QuotaThreshold, _ = strconv.ParseFloat(getEnvWithCheck("QuotaThreshold", &missingEnvVars), 64) if clusterNS != "" { ClusterNS = strings.Split(clusterNS, ",") @@ -112,6 +117,8 @@ func GetENV() error { "FeishuWebhookURLImportant", //Backup "FeishuWebhookURLBackup", + //Quota + "FeishuWebhookURLQuota", }, FeishuWebhookURLMap, &missingEnvVars) // Get ClusterRegionMap diff --git a/service/exceptionmonitor/helper/monitor/quota_monitor.go b/service/exceptionmonitor/helper/monitor/quota_monitor.go new file mode 100644 index 000000000..87a201381 --- /dev/null +++ b/service/exceptionmonitor/helper/monitor/quota_monitor.go @@ -0,0 +1,127 @@ +package monitor + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "github.com/labring/sealos/service/exceptionmonitor/api" + "github.com/labring/sealos/service/exceptionmonitor/helper/notification" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func QuotaMonitor() { + for api.QuotaMonitor { + if err := checkQuota(); err != nil { + log.Printf("Failed to check qouta: %v", err) + } + time.Sleep(3 * time.Hour) + } +} + +func checkQuota() error { + namespaceList, _ := api.ClientSet.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{}) + + fmt.Println(len(namespaceList.Items)) + for _, ns := range namespaceList.Items { + if !strings.Contains(ns.Name, "ns-") { + continue + } + quotaList, err := api.ClientSet.CoreV1().ResourceQuotas(ns.Name).List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Printf("Failed to check quota: %v", err) + } + //user debt + if len(quotaList.Items) != 1 || quotaList.Items[0].Name != "quota-"+ns.Name { + continue + } + nsQuota := notification.NameSpaceQuota{ + NameSpace: ns.Name, + } + notificationInfo := notification.Info{ + ExceptionType: "Quota", + PerformanceType: "Quota", + NotificationType: "exception", + } + send := processQuota(quotaList, &nsQuota) + if send { + message := notification.GetQuotaMessage(&nsQuota) + if err := notification.SendFeishuNotification(notificationInfo, message, api.FeishuWebhookURLMap["FeishuWebhookURLQuota"]); err != nil { + log.Printf("Error sending exception notification:%v", err) + } + } + } + return nil +} + +func processQuota(quotaList *v1.ResourceQuotaList, nsQuota *notification.NameSpaceQuota) bool { + send := false + for resourceName, hardQuantity := range quotaList.Items[0].Status.Hard { + usedQuantity, exists := quotaList.Items[0].Status.Used[resourceName] + if !exists { + usedQuantity = *resource.NewQuantity(0, hardQuantity.Format) + } + + hardValue, err := quantityToFloat64(hardQuantity, string(resourceName)) + if err != nil { + log.Printf("Analytic hard constraint error : %s , %v\n", resourceName, err) + continue + } + + usedValue, err := quantityToFloat64(usedQuantity, string(resourceName)) + if err != nil { + fmt.Printf("Analytic usage error : %s , %v\n", resourceName, err) + continue + } + + // usage + utilization := (usedValue / hardValue) * 100 + if utilization > api.QuotaThreshold { + send = true + switch resourceName { + case "limits.cpu": + nsQuota.CPULimit = hardQuantity.String() + nsQuota.CPUUsage = fmt.Sprintf("%.2f%%", utilization) + case "limits.memory": + nsQuota.MemLimit = hardQuantity.String() + nsQuota.MemUsage = fmt.Sprintf("%.2f%%", utilization) + case "limits.nvidia.com/gpu": + nsQuota.GPULimit = hardQuantity.String() + nsQuota.GPUUsage = fmt.Sprintf("%.2f%%", utilization) + case "limits.ephemeral-storage": + nsQuota.EphemeralStorageLimit = hardQuantity.String() + nsQuota.EphemeralStorageUsage = fmt.Sprintf("%.2f%%", utilization) + case "objectstorage/size": + nsQuota.ObjectStorageLimit = hardQuantity.String() + nsQuota.ObjectStorageUsage = fmt.Sprintf("%.2f%%", utilization) + case "services.nodeports": + nsQuota.NodePortLimit = hardQuantity.String() + nsQuota.NodePortUsage = fmt.Sprintf("%.2f%%", utilization) + case "requests.storage": + nsQuota.StorageLimit = hardQuantity.String() + nsQuota.StorageUsage = fmt.Sprintf("%.2f%%", utilization) + } + } + } + return send +} + +func quantityToFloat64(q resource.Quantity, resourceName string) (float64, error) { + switch resourceName { + case "limits.cpu", "requests.cpu", "cpu", + "limits.nvidia.com/gpu", "requests.nvidia.com/gpu": + return float64(q.MilliValue()), nil + case "limits.memory", "requests.memory", "memory", + "limits.ephemeral-storage", "requests.ephemeral-storage", + "requests.storage", "objectstorage/size": + return float64(q.Value()), nil + case "services.nodeports": + return float64(q.Value()), nil + default: + return q.AsApproximateFloat64(), nil + } +} diff --git a/service/exceptionmonitor/helper/notification/feishu.go b/service/exceptionmonitor/helper/notification/feishu.go index 90d419af8..312547f98 100644 --- a/service/exceptionmonitor/helper/notification/feishu.go +++ b/service/exceptionmonitor/helper/notification/feishu.go @@ -35,6 +35,24 @@ type Info struct { ExceptionType string } +type NameSpaceQuota struct { + NameSpace string + CPULimit string + MemLimit string + GPULimit string + EphemeralStorageLimit string + ObjectStorageLimit string + NodePortLimit string + StorageLimit string + CPUUsage string + MemUsage string + GPUUsage string + EphemeralStorageUsage string + ObjectStorageUsage string + NodePortUsage string + StorageUsage string +} + func InitFeishuClient() { feiShuClient = lark.NewClient(api.APPID, api.APPSECRET) } @@ -201,6 +219,8 @@ func getMessageIDMap(performanceType string) map[string]string { return api.DatabaseCPUMessageIDMap case "Backup": return api.DatabaseBackupMessageIDMap + case "Quota": + return api.QuotaMessageIDMap default: return api.DatabaseStatusMessageIDMap } @@ -248,7 +268,7 @@ func createFeishuNotification(notification Info, message, feishuWebHook string, respStr := larkcore.Prettify(resp) messageID := extractAndPrintMessageID(respStr) - if notification.DatabaseClusterName == "Backup" { + if notification.PerformanceType == "Backup" || notification.PerformanceType == "Quota" { return nil } if messageID == "" { @@ -297,7 +317,75 @@ func createCard(headerTemplate, headerTitle string, elements []map[string]string return card } -func createElements(namespace, backupName, status, startTime, reason string, includeReason bool) []map[string]string { +func GetQuotaMessage(nsQuota *NameSpaceQuota) string { + var card map[string]interface{} + elements := createQuotaElements(nsQuota) + card = createCard("red", "Quota阀值通知", elements) + + databaseMessage, err := marshalCard(card) + if err != nil { + fmt.Println(err) + return "" + } + return databaseMessage +} + +func createQuotaElements(nsQuota *NameSpaceQuota) []map[string]string { + elements := []map[string]string{ + {"label": "集群环境", "value": api.ClusterName}, + {"label": "命名空间", "value": nsQuota.NameSpace}, + } + addNonEmptyFieldsToElements(nsQuota, &elements) + return elements +} + +func addNonEmptyFieldsToElements(nsQuota *NameSpaceQuota, elements *[]map[string]string) { + fields := map[string]string{ + "CPULimit": "CPU总量", + "CPUUsage": "CPU使用率", + "MemLimit": "内存总量", + "MemUsage": "内存使用率", + "GPULimit": "GPU总量", + "GPUUsage": "GPU使用率", + "EphemeralStorageLimit": "临时存储总量", + "EphemeralStorageUsage": "临时存储使用率", + "ObjectStorageLimit": "对象存储总量", + "ObjectStorageUsage": "对象存储使用率", + "NodePortLimit": "节点端口总量", + "NodePortUsage": "节点端口使用率", + "StorageLimit": "存储总量", + "StorageUsage": "存储使用率", + } + + quotaValues := map[string]string{ + "CPULimit": nsQuota.CPULimit, + "CPUUsage": nsQuota.CPUUsage, + "MemLimit": nsQuota.MemLimit, + "MemUsage": nsQuota.MemUsage, + "GPULimit": nsQuota.GPULimit, + "GPUUsage": nsQuota.GPUUsage, + "EphemeralStorageLimit": nsQuota.EphemeralStorageLimit, + "EphemeralStorageUsage": nsQuota.EphemeralStorageUsage, + "ObjectStorageLimit": nsQuota.ObjectStorageLimit, + "ObjectStorageUsage": nsQuota.ObjectStorageUsage, + "NodePortLimit": nsQuota.NodePortLimit, + "NodePortUsage": nsQuota.NodePortUsage, + "StorageLimit": nsQuota.StorageLimit, + "StorageUsage": nsQuota.StorageUsage, + } + + for field, label := range fields { + value := quotaValues[field] + if value != "" { + *elements = append(*elements, map[string]string{ + "label": label, + "value": value, + }) + } + } +} + +func createBackupElements(namespace, backupName, status, startTime, reason string, includeReason bool) []map[string]string { elements := []map[string]string{ {"label": "集群环境", "value": api.ClusterName}, {"label": "命名空间", "value": namespace}, @@ -322,10 +410,10 @@ func marshalCard(card map[string]interface{}) (string, error) { func GetBackupMessage(notificationType, namespace, backupName, status, startTime, reason string) string { var card map[string]interface{} if notificationType == ExceptionType { - elements := createElements(namespace, backupName, status, startTime, reason, true) + elements := createBackupElements(namespace, backupName, status, startTime, reason, true) card = createCard("red", "备份异常通知", elements) } else { - elements := createElements(namespace, backupName, status, startTime, "", false) + elements := createBackupElements(namespace, backupName, status, startTime, "", false) card = createCard("blue", "备份恢复通知", elements) } diff --git a/service/exceptionmonitor/main.go b/service/exceptionmonitor/main.go index 943d49fdc..fe518bc00 100644 --- a/service/exceptionmonitor/main.go +++ b/service/exceptionmonitor/main.go @@ -18,6 +18,7 @@ func main() { go monitor.DatabaseExceptionMonitor() go monitor.DatabasePerformanceMonitor() go monitor.DatabaseBackupMonitor() + go monitor.QuotaMonitor() select {} }