feat(monitor): support disable_notify_recovery for commonalert (#24377)

pkg/apis/monitor/commalert.go

@@ -70,6 +70,9 @@ type CommonAlertCreateBaseInput struct {
 	// 角色 id 或者 name
 	Roles []string `json:"roles"`
 
+	// 为 true 时不发送恢复通知（OK）
+	DisableNotifyRecovery *bool `json:"disable_notify_recovery"`
+
 	// 静默期
 	SilentPeriod string `json:"silent_period"`
 	// 报警类型
@@ -155,6 +158,8 @@ type CommonAlertUpdateInput struct {
 	Channel []string `json:"channel"`
 	// 通知接受者
 	Recipients []string `json:"recipients"`
+	// 为 true 时不发送恢复通知（OK）
+	DisableNotifyRecovery *bool `json:"disable_notify_recovery"`
 	// 静默期
 	SilentPeriod string `json:"silent_period"`
 	// systemalert policy may need update through operator
@@ -167,13 +172,14 @@ type CommonAlertDetails struct {
 	AlertDetails
 	Period string `json:"period"`
 	// 报警连续持续周期数
-	AlertDuration int64    `json:"alert_duration"`
-	Level         string   `json:"level"`
-	NotifierId    string   `json:"notifier_id"`
-	Channel       []string `json:"channel"`
-	Recipients    []string `json:"recipients"`
-	RobotIds      []string `json:"robot_ids"`
-	RoleIds       []string `json:"role_ids"`
+	AlertDuration         int64    `json:"alert_duration"`
+	Level                 string   `json:"level"`
+	NotifierId            string   `json:"notifier_id"`
+	Channel               []string `json:"channel"`
+	Recipients            []string `json:"recipients"`
+	RobotIds              []string `json:"robot_ids"`
+	RoleIds               []string `json:"role_ids"`
+	DisableNotifyRecovery bool     `json:"disable_notify_recovery"`
 	// 静默期
 	SilentPeriod string `json:"silent_period"`
 	Status       string `json:"status"`

pkg/mcclient/options/monitor/commonalert.go

@@ -59,6 +59,8 @@ type CommonAlertUpdateOptions struct {
 	Comparator string `help:"Alarm policy threshold comparison method" json:"comparator" `
 	Threshold  string `help:"Alarm policy threshold" json:"threshold"`
 	Reason     string `help:"Alarm policy reason" json:"reason"`
+	// 为 true 时不发送恢复通知
+	DisableNotifyRecovery *bool `help:"when true, do not send recovery notifications" json:"disable_notify_recovery"`
 }
 
 func (o *CommonAlertUpdateOptions) Params() (jsonutils.JSONObject, error) {

pkg/monitor/alerting/notifiers/dingding.go

@@ -94,7 +94,7 @@ func (dd *DingDingNotifier) Notify(evalCtx *alerting.EvalContext, d jsonutils.JS
 			errs = append(errs, errors.Wrap(err, "notify alerting matches"))
 		}
 	}
-	if evalCtx.HasRecoveredMatches() {
+	if evalCtx.HasRecoveredMatches() && !evalCtx.Rule.DisableNotifyRecovery {
 		if err := dd.notify(evalCtx, evalCtx.GetRecoveredMatches(), true, d); err != nil {
 			errs = append(errs, errors.Wrap(err, "notify recovered matches"))
 		}

pkg/monitor/alerting/notifiers/onecloud.go

@@ -325,7 +325,7 @@ func (oc *OneCloudNotifier) notifyByContextLang(ctx context.Context, evalCtx *al
 			errs = append(errs, errors.Wrapf(err, "notify alerting matches"))
 		}
 	}
-	if evalCtx.HasRecoveredMatches() {
+	if evalCtx.HasRecoveredMatches() && !evalCtx.Rule.DisableNotifyRecovery {
 		if err := oc.notifyMatchesByContextLang(ctx, evalCtx, evalCtx.GetRecoveredMatches(), uids, true); err != nil {
 			errs = append(errs, errors.Wrapf(err, "notify recovered matches"))
 		}

pkg/monitor/alerting/rule.go

@@ -49,14 +49,15 @@ type Rule struct {
 	Name      string
 	Message   string
 	// 使用 TriggeredMessages 存储触发的条件，替代 Message
-	TriggeredMessages   []string
-	LastStateChange     time.Time
-	For                 time.Duration
-	NoDataState         monitor.NoDataOption
-	ExecutionErrorState monitor.ExecutionErrorOption
-	State               monitor.AlertStateType
-	Conditions          []Condition
-	Notifications       []string
+	TriggeredMessages     []string
+	LastStateChange       time.Time
+	For                   time.Duration
+	NoDataState           monitor.NoDataOption
+	ExecutionErrorState   monitor.ExecutionErrorOption
+	State                 monitor.AlertStateType
+	Conditions            []Condition
+	Notifications         []string
+	DisableNotifyRecovery bool
 	// AlertRuleTags       []*models.AlertRuleTag
 	Level           string
 	Reason          string
@@ -126,6 +127,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 	model.RuleDescription = make([]*monitor.AlertRecordRule, 0)
 
 	model.Frequency = ruleDef.Frequency
+	model.DisableNotifyRecovery = ruleDef.DisableNotifyRecovery
 	// frequency cannot be zero since that would not execute the alert rule.
 	// so we fallback to 60 seconds if `Frequency` is missing
 	if model.Frequency == 0 {

pkg/monitor/models/alert.go

@@ -124,6 +124,8 @@ type SAlert struct {
 	Message   string                `charset:"utf8" list:"user" create:"optional" update:"user"`
 	UsedBy    string                `charset:"ascii" create:"optional" list:"user"`
 
+	DisableNotifyRecovery bool `nullable:"false" default:"false" list:"user" create:"optional" update:"user"`
+
 	// Silenced       bool
 	ExecutionError string `charset:"utf8" list:"user"`
 

pkg/monitor/models/commonalert.go

@@ -358,6 +358,9 @@ func (alert *SCommonAlert) CustomizeCreate(
 	if err := data.Unmarshal(input); err != nil {
 		return err
 	}
+	if input.DisableNotifyRecovery != nil {
+		alert.DisableNotifyRecovery = *input.DisableNotifyRecovery
+	}
 
 	return alert.customizeCreateNotis(ctx, userCred, query, data)
 }
@@ -795,6 +798,7 @@ func (man *SCommonAlertManager) FetchCustomizeColumns(
 	}
 	for i := range rows {
 		alert := objs[i].(*SCommonAlert)
+		rows[i].DisableNotifyRecovery = alert.DisableNotifyRecovery
 		if alertNotis, ok := alertNotificationMap[alertIds[i]]; ok {
 			channel := sets.String{}
 			for j, alertNoti := range alertNotis {
@@ -1307,6 +1311,14 @@ func (alert *SCommonAlert) PostUpdate(
 	query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	updateInput := new(monitor.CommonAlertUpdateInput)
 	data.Unmarshal(updateInput)
+	if updateInput.DisableNotifyRecovery != nil {
+		if _, err := db.Update(alert, func() error {
+			alert.DisableNotifyRecovery = *updateInput.DisableNotifyRecovery
+			return nil
+		}); err != nil {
+			log.Errorf("update disable_notify_recovery error: %v", err)
+		}
+	}
 	if len(updateInput.Channel) != 0 {
 		if err := alert.UpdateNotification(ctx, userCred, query, data); err != nil {
 			log.Errorf("update notification error: %v", err)
@@ -1524,6 +1536,10 @@ func (alert *SCommonAlert) PerformConfig(ctx context.Context, userCred mcclient.
 		if len(reason) != 0 {
 			alert.Reason = reason
 		}
+		disableNotifyRecovery, gErr := data.Bool("disable_notify_recovery")
+		if gErr == nil {
+			alert.DisableNotifyRecovery = disableNotifyRecovery
+		}
 		return nil
 	})
 	PerformConfigLog(alert, userCred)