diff --git a/pkg/apis/monitor/alertrecord.go b/pkg/apis/monitor/alertrecord.go index f9e6f0da53..e8b08319b2 100644 --- a/pkg/apis/monitor/alertrecord.go +++ b/pkg/apis/monitor/alertrecord.go @@ -2,6 +2,11 @@ package monitor import "yunion.io/x/onecloud/pkg/apis" +const ( + SEND_STATE_OK = "ok" + SEND_STATE_SILENT = "silent" +) + type AlertRecordListInput struct { apis.Meta @@ -31,6 +36,7 @@ type AlertRecordCreateInput struct { // 报警级别 Level string `json:"level"` State string `json:"state"` + SendState string `json:"send_state"` ResType string `json:"res_type"` EvalData []*EvalMatch `json:"eval_data"` AlertRule AlertRecordRule diff --git a/pkg/apis/monitor/commalert.go b/pkg/apis/monitor/commalert.go index 858dcccfe6..0757d65754 100644 --- a/pkg/apis/monitor/commalert.go +++ b/pkg/apis/monitor/commalert.go @@ -42,6 +42,8 @@ type CommonAlertCreateInput struct { Channel []string `json:"channel"` // 通知接受者 Recipients []string `json:"recipients"` + // 静默期 + SilentPeriod string `json:"silent_period"` // 报警类型 AlertType string `json:"alert_type"` @@ -98,6 +100,8 @@ type CommonAlertUpdateInput struct { Channel []string `json:"channel"` // 通知接受者 Recipients []string `json:"recipients"` + // 静默期 + SilentPeriod string `json:"silent_period"` // systemalert policy may need update through operator ForceUpdate bool `json:"force_update"` GetPointStr bool `json:"get_point_str"` diff --git a/pkg/apis/monitor/notification.go b/pkg/apis/monitor/notification.go index 13d931b534..5842df4673 100644 --- a/pkg/apis/monitor/notification.go +++ b/pkg/apis/monitor/notification.go @@ -50,7 +50,7 @@ type NotificationCreateInput struct { SendReminder *bool `json:"send_reminder"` // 是否禁用报警恢复提醒 DisableResolveMessage *bool `json:"disable_resolve_message"` - // 发送频率 + // 发送频率 单位:s Frequency time.Duration `json:"frequency"` // 通知配置 Settings jsonutils.JSONObject `json:"settings"` diff --git a/pkg/monitor/alerting/notifier.go b/pkg/monitor/alerting/notifier.go index 03b126738e..13689f284f 100644 --- a/pkg/monitor/alerting/notifier.go +++ b/pkg/monitor/alerting/notifier.go @@ -144,8 +144,8 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon }) } } - if shouldNotify { - n.createAlertRecordWhenNotify(evalCtx) + if shouldNotify || evalCtx.Rule.State == monitor.AlertStateAlerting { + n.createAlertRecordWhenNotify(evalCtx, shouldNotify) } if !shouldNotify && evalCtx.shouldUpdateAlertState() && evalCtx.NoDataFound { n.detachAlertResourceWhenNodata(evalCtx) @@ -154,7 +154,7 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon return result, nil } -func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext) { +func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, shouldNotify bool) { var matches []*monitor.EvalMatch if evalCtx.Firing { matches = evalCtx.EvalMatches @@ -168,9 +168,13 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext) AlertId: evalCtx.Rule.Id, Level: evalCtx.Rule.Level, State: string(evalCtx.Rule.State), + SendState: monitor.SEND_STATE_OK, EvalData: matches, AlertRule: newAlertRecordRule(evalCtx), } + if !shouldNotify { + recordCreateInput.SendState = monitor.SEND_STATE_SILENT + } recordCreateInput.ResType = recordCreateInput.AlertRule.ResType createData := recordCreateInput.JSON(recordCreateInput) alert, _ := models.CommonAlertManager.GetAlert(evalCtx.Rule.Id) diff --git a/pkg/monitor/alerting/notifiers/base.go b/pkg/monitor/alerting/notifiers/base.go index 4e28db2964..c0e7547201 100644 --- a/pkg/monitor/alerting/notifiers/base.go +++ b/pkg/monitor/alerting/notifiers/base.go @@ -66,6 +66,9 @@ func (n *NotifierBase) ShouldNotify(_ context.Context, evalCtx *alerting.EvalCon } if newState == monitor.AlertStateAlerting { + if prevState == monitor.AlertStateOK { + return true + } send, err := state.ShouldSendNotification() if err != nil { log.Errorf("Alertnotification ShouldSendNotification exec err:%v", err) diff --git a/pkg/monitor/models/alertrecord.go b/pkg/monitor/models/alertrecord.go index 19e8bfc35f..6c83a92f8c 100644 --- a/pkg/monitor/models/alertrecord.go +++ b/pkg/monitor/models/alertrecord.go @@ -38,6 +38,7 @@ type SAlertRecord struct { AlertId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"` Level string `charset:"ascii" width:"36" nullable:"false" default:"normal" list:"user" update:"user"` State string `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"` + SendState string `width:"36" charset:"ascii" default:"ok" list:"user" update:"user"` EvalData jsonutils.JSONObject `list:"user" update:"user"` AlertRule jsonutils.JSONObject `list:"user" update:"user"` ResType string `width:"36" list:"user" update:"user"` diff --git a/pkg/monitor/models/commonalert.go b/pkg/monitor/models/commonalert.go index 0a8adaa0d3..e135799d37 100644 --- a/pkg/monitor/models/commonalert.go +++ b/pkg/monitor/models/commonalert.go @@ -113,6 +113,11 @@ func (man *SCommonAlertManager) ValidateCreateData( if _, err := time.ParseDuration(data.Period); err != nil { return data, httperrors.NewInputParameterError("Invalid period format: %s", data.Period) } + if data.SilentPeriod != "" { + if _, err := time.ParseDuration(data.SilentPeriod); err != nil { + return data, httperrors.NewInputParameterError("Invalid silent_period format: %s", data.SilentPeriod) + } + } // 默认的系统配置Recipients=commonalert-default if data.AlertType != monitor.CommonAlertSystemAlertType && len(data.Recipients) == 0 { return data, merrors.NewArgIsEmptyErr("recipients") @@ -253,10 +258,10 @@ func (alert *SCommonAlert) customizeCreateNotis(ctx context.Context, userCred mc } //user_by 弃用 if input.AlertType == monitor.CommonAlertSystemAlertType { - return alert.createAlertNoti(ctx, userCred, input.Name, "webconsole", []string{}, true) + return alert.createAlertNoti(ctx, userCred, input.Name, "webconsole", []string{}, input.SilentPeriod, true) } for _, channel := range input.Channel { - err := alert.createAlertNoti(ctx, userCred, input.Name, channel, input.Recipients, false) + err := alert.createAlertNoti(ctx, userCred, input.Name, channel, input.Recipients, input.SilentPeriod, false) if err != nil { return errors.Wrap(err, fmt.Sprintf("create notify[channel is %s]error", channel)) } @@ -265,8 +270,8 @@ func (alert *SCommonAlert) customizeCreateNotis(ctx context.Context, userCred mc } func (alert *SCommonAlert) createAlertNoti(ctx context.Context, userCred mcclient.TokenCredential, - notiName, channel string, userIds []string, isSysNoti bool) error { - noti, err := NotificationManager.CreateOneCloudNotification(ctx, userCred, notiName, channel, userIds) + notiName, channel string, userIds []string, silentPeriod string, isSysNoti bool) error { + noti, err := NotificationManager.CreateOneCloudNotification(ctx, userCred, notiName, channel, userIds, silentPeriod) if err != nil { return errors.Wrap(err, "create notification") } @@ -746,6 +751,11 @@ func (alert *SCommonAlert) ValidateUpdateData( data.Set("frequency", jsonutils.NewInt(freqSpec)) } } + if silentPeriod, _ := data.GetString("silent_period"); len(silentPeriod) > 0 { + if _, err := time.ParseDuration(silentPeriod); err != nil { + return data, httperrors.NewInputParameterError("Invalid silent_period format: %s", silentPeriod) + } + } //if recipients, _ := data.GetArray("recipients"); len(recipients) > 0 { // channelStr, _ := data.GetString("channel") // channel, _ := data.GetArray("channel") diff --git a/pkg/monitor/models/nodealert.go b/pkg/monitor/models/nodealert.go index 6a966b31b1..55ae2ae9da 100644 --- a/pkg/monitor/models/nodealert.go +++ b/pkg/monitor/models/nodealert.go @@ -83,7 +83,7 @@ func (v1man *SV1AlertManager) CreateNotification( channel string, recipients string) (*SNotification, error) { userIds := strings.Split(recipients, ",") - return NotificationManager.CreateOneCloudNotification(ctx, userCred, alertName, channel, userIds) + return NotificationManager.CreateOneCloudNotification(ctx, userCred, alertName, channel, userIds, "") } func (man *SNodeAlertManager) ValidateCreateData( diff --git a/pkg/monitor/models/notification.go b/pkg/monitor/models/notification.go index b4c3d4d13f..2bb17089d5 100644 --- a/pkg/monitor/models/notification.go +++ b/pkg/monitor/models/notification.go @@ -165,7 +165,7 @@ func (man *SNotificationManager) CreateOneCloudNotification( userCred mcclient.TokenCredential, alertName string, channel string, - userIds []string) (*SNotification, error) { + userIds []string, silentPeriod string) (*SNotification, error) { settings := &monitor.NotificationSettingOneCloud{ Channel: channel, UserIds: userIds, @@ -179,6 +179,10 @@ func (man *SNotificationManager) CreateOneCloudNotification( Type: monitor.AlertNotificationTypeOneCloud, Settings: jsonutils.Marshal(settings), } + if silentPeriod != "" { + duration, _ := time.ParseDuration(silentPeriod) + input.Frequency = duration / time.Second + } obj, err := db.DoCreate(man, ctx, userCred, nil, input.JSON(input), userCred) if err != nil { return nil, errors.Wrapf(err, "create notification input: %s", input.JSON(input))