From 7b30dcab082633fdb6190b18ec2a4f6735d5fd7a Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 30 Jul 2024 16:15:55 +0800 Subject: [PATCH] feat(monitor): multiple alerts of resource (#20924) --- pkg/apis/monitor/monitor_resource_alert.go | 7 +- .../modules/monitor/mod_monitor_resource.go | 2 +- .../options/monitor/monitor_resource_alert.go | 1 + pkg/monitor/alerting/conditions/query.go | 4 +- pkg/monitor/alerting/eval_context.go | 5 - pkg/monitor/alerting/notifier.go | 88 +++++----- pkg/monitor/alerting/rule.go | 68 +------- pkg/monitor/models/alertrecord.go | 7 +- pkg/monitor/models/commonalert.go | 164 ++++++++++++------ pkg/monitor/models/monitor_resource.go | 97 +++++++---- pkg/monitor/models/monitor_resource_alert.go | 46 +++-- 11 files changed, 268 insertions(+), 221 deletions(-) diff --git a/pkg/apis/monitor/monitor_resource_alert.go b/pkg/apis/monitor/monitor_resource_alert.go index a5f187fae8..b0ee49f55d 100644 --- a/pkg/apis/monitor/monitor_resource_alert.go +++ b/pkg/apis/monitor/monitor_resource_alert.go @@ -28,18 +28,23 @@ type MonitorResourceJointListInput struct { MonitorResourceId string `json:"monitor_resource_id"` AlertId string `json:"alert_id"` JointId []int64 `json:"joint_id"` - Alerting bool `json:"alertinng"` + Alerting bool `json:"alerting"` + AlertState string `json:"alert_state"` SendState string `json:"send_state"` ResType string `json:"res_type"` + Metric string `json:"metric"` ResName string `json:"res_name"` AlertName string `json:"alert_name"` Level string `json:"level"` + // 查询所有状态 + AllState bool `json:"all_state"` } type MonitorResourceJointCreateInput struct { apis.Meta MonitorResourceId string `json:"monitor_resource_id"` AlertId string `json:"alert_id"` + Metric string `json:"metric"` AlertRecordId string `width:"36" charset:"ascii" list:"user" update:"user"` AlertState string `width:"18" charset:"ascii" list:"user" update:"user"` diff --git a/pkg/mcclient/modules/monitor/mod_monitor_resource.go b/pkg/mcclient/modules/monitor/mod_monitor_resource.go index b1de1d344c..2c7e6d4da7 100644 --- a/pkg/mcclient/modules/monitor/mod_monitor_resource.go +++ b/pkg/mcclient/modules/monitor/mod_monitor_resource.go @@ -51,7 +51,7 @@ func NewMonitorResourceManager() *SMonitorResourceManager { func newAlertResourceAlertManager() *SMonitorResourceAlertManager { man := modules.NewJointMonitorV2Manager("monitorresourcealert", "monitorresourcealerts", - []string{"monitor_resource_id", "alert_id", "res_name", "res_type", "alert_name", "alert_state", "send_state", "level", + []string{"monitor_resource_id", "alert_id", "res_name", "res_type", "metric", "alert_name", "alert_state", "send_state", "level", "trigger_time", "data"}, []string{}, MonitorResourceManager, CommonAlerts) diff --git a/pkg/mcclient/options/monitor/monitor_resource_alert.go b/pkg/mcclient/options/monitor/monitor_resource_alert.go index cb0427e35d..9615adacf7 100644 --- a/pkg/mcclient/options/monitor/monitor_resource_alert.go +++ b/pkg/mcclient/options/monitor/monitor_resource_alert.go @@ -26,6 +26,7 @@ type MonitorResourceAlertListOptions struct { AlertId string `help:"ID of alert" json:"alert_id"` Alerting bool `help:"search alerting resource" json:"alerting"` SendState string `json:"send_state"` + AllState bool `help:"Show all state" json:"all_state"` } func (o *MonitorResourceAlertListOptions) GetMasterOpt() string { diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index e3903dee76..6f245b568d 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -296,7 +296,7 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series moni evalMatch.Unit = alertDetails.FieldDescription.Unit evalMatch.Tags = c.filterTags(series.Tags, *alertDetails) evalMatch.Value = value - evalMatch.ValueStr = alerting.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit, + evalMatch.ValueStr = models.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit, alertDetails.FieldOpt) if alertDetails.GetPointStr { evalMatch.ValueStr = c.jointPointStr(series, evalMatch.ValueStr, valStrArr) @@ -305,7 +305,7 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series moni //c.newRuleDescription(context, alertDetails) //evalMatch.Condition = c.GenerateFormatCond(meta, queryKeyInfo).String() msg := fmt.Sprintf("%s.%s %s %s", alertDetails.Measurement, alertDetails.Field, - alertDetails.Comparator, alerting.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, "")) + alertDetails.Comparator, models.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, "")) if len(context.Rule.Message) == 0 { context.Rule.Message = msg } diff --git a/pkg/monitor/alerting/eval_context.go b/pkg/monitor/alerting/eval_context.go index c4b170addb..0580c385a8 100644 --- a/pkg/monitor/alerting/eval_context.go +++ b/pkg/monitor/alerting/eval_context.go @@ -43,7 +43,6 @@ type EvalContext struct { StartTime time.Time EndTime time.Time Rule *Rule - //RuleDescription *RuleDescription NoDataFound bool PrevAlertState monitor.AlertStateType @@ -52,10 +51,6 @@ type EvalContext struct { UserCred mcclient.TokenCredential } -type RuleDescription struct { - monitor.AlertRecordRule -} - // NewEvalContext is the EvalContext constructor. func NewEvalContext(alertCtx context.Context, userCred mcclient.TokenCredential, rule *Rule) *EvalContext { return &EvalContext{ diff --git a/pkg/monitor/alerting/notifier.go b/pkg/monitor/alerting/notifier.go index cb8e454182..db97702f5b 100644 --- a/pkg/monitor/alerting/notifier.go +++ b/pkg/monitor/alerting/notifier.go @@ -16,7 +16,6 @@ package alerting import ( "database/sql" - "fmt" "time" "yunion.io/x/jsonutils" @@ -39,11 +38,13 @@ func newNotificationService() *notificationService { } func (n *notificationService) SendIfNeeded(evalCtx *EvalContext) error { - notifierStates, err := n.getNeededNotifiers(evalCtx.Rule.Notifications, evalCtx) + notifierStates, shouldNotify, err := n.getNeededNotifiers(evalCtx.Rule.Notifications, evalCtx) if err != nil { return errors.Wrap(err, "failed to get alert notifiers") } + n.syncResources(evalCtx, shouldNotify) + if len(notifierStates) == 0 { return nil } @@ -98,10 +99,10 @@ func (n *notificationService) sendNotifications(evalCtx *EvalContext, states not return nil } -func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalContext) (notifierStateSlice, error) { +func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalContext) (notifierStateSlice, bool, error) { notis, err := models.NotificationManager.GetNotificationsWithDefault(nIds) if err != nil { - return nil, errors.Wrapf(err, "GetNotificationsWithDefault with %v", nIds) + return nil, false, errors.Wrapf(err, "GetNotificationsWithDefault with %v", nIds) } var result notifierStateSlice @@ -143,6 +144,11 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon }) } } + + return result, shouldNotify, nil +} + +func (n *notificationService) syncResources(evalCtx *EvalContext, shouldNotify bool) { if shouldNotify || evalCtx.Rule.State == monitor.AlertStateAlerting { go func() { if err := n.createAlertRecordWhenNotify(evalCtx, shouldNotify); err != nil { @@ -156,7 +162,13 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon }() } - return result, nil + if len(evalCtx.AlertOkEvalMatches) > 0 { + go func() { + if err := n.syncMonitorResourceAlerts(evalCtx); err != nil { + log.Errorf("syncMonitorResourceAlerts error: %v", err) + } + }() + } } func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, shouldNotify bool) error { @@ -166,7 +178,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, } else { matches = evalCtx.AlertOkEvalMatches } - n.dealNeedShieldEvalMatchs(evalCtx, matches) + n.dealNeedShieldEvalMatches(evalCtx, matches) recordCreateInput := monitor.AlertRecordCreateInput{ StandaloneResourceCreateInput: apis.StandaloneResourceCreateInput{ GenerateName: evalCtx.Rule.Name, @@ -176,7 +188,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, State: string(evalCtx.Rule.State), SendState: monitor.SEND_STATE_OK, EvalData: matches, - AlertRule: newAlertRecordRules(evalCtx), + AlertRule: evalCtx.Rule.RuleDescription, } if !shouldNotify { recordCreateInput.SendState = monitor.SEND_STATE_SILENT @@ -187,8 +199,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, } createData := recordCreateInput.JSON(recordCreateInput) alert, _ := models.CommonAlertManager.GetAlert(evalCtx.Rule.Id) - record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), - createData, evalCtx.UserCred) + record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), createData, evalCtx.UserCred) if err != nil { return errors.Wrapf(err, "db.DoCreate") } @@ -206,7 +217,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, return nil } -func (n *notificationService) dealNeedShieldEvalMatchs(evalCtx *EvalContext, match []*monitor.EvalMatch) { +func (n *notificationService) dealNeedShieldEvalMatches(evalCtx *EvalContext, match []*monitor.EvalMatch) { input := monitor.AlertRecordShieldListInput{ ResType: evalCtx.Rule.RuleDescription[0].ResType, AlertId: evalCtx.Rule.Id, @@ -238,6 +249,32 @@ func (n *notificationService) detachAlertResourceWhenNodata(evalCtx *EvalContext } } +func (n *notificationService) syncMonitorResourceAlerts(evalCtx *EvalContext) error { + if len(evalCtx.AlertOkEvalMatches) == 0 { + log.Infof("alert_ok_eval_matches is empty, skip syncMonitorResourceAlerts") + return nil + } + // only sync resource not need notify + matches := make([]monitor.EvalMatch, len(evalCtx.AlertOkEvalMatches)) + for i := range evalCtx.AlertOkEvalMatches { + matches[i] = *evalCtx.AlertOkEvalMatches[i] + } + alertRule := evalCtx.Rule.RuleDescription + input := &models.UpdateMonitorResourceAlertInput{ + AlertId: evalCtx.Rule.Id, + Matches: matches, + ResType: alertRule[0].ResType, + AlertState: string(monitor.AlertStateOK), + SendState: monitor.SEND_STATE_SILENT, + TriggerTime: time.Now(), + AlertRecordId: "", + } + if err := models.MonitorResourceManager.UpdateMonitorResourceAttachJoint(evalCtx.Ctx, evalCtx.UserCred, input); err != nil { + return errors.Wrap(err, "UpdateMonitorResourceAttachJoint") + } + return nil +} + type NotifierPlugin struct { Type string Factory NotifierFactory @@ -271,34 +308,3 @@ func InitNotifier(config NotificationConfig) (Notifier, error) { } return plug.(Notifier), nil } - -func newAlertRecordRules(ctx *EvalContext) []*monitor.AlertRecordRule { - rules := make([]*monitor.AlertRecordRule, 0) - for i := range ctx.Rule.RuleDescription { - rule := newAlertRecordRule(ctx, i) - rules = append(rules, rule) - } - return rules -} - -func newAlertRecordRule(evalCtx *EvalContext, idx int) *monitor.AlertRecordRule { - alertRule := monitor.AlertRecordRule{} - if len(evalCtx.Rule.RuleDescription) != 0 { - alertRule = evalCtx.Rule.RuleDescription[idx].AlertRecordRule - } - if evalCtx.Rule.Frequency < 60 { - alertRule.Period = fmt.Sprintf("%ds", evalCtx.Rule.Frequency) - } else { - alertRule.Period = fmt.Sprintf("%dm", evalCtx.Rule.Frequency/60) - } - - alertRule.AlertDuration = int64(evalCtx.Rule.For) / evalCtx.Rule.Frequency - if alertRule.AlertDuration == 0 { - alertRule.AlertDuration = 1 - } - - if evalCtx.Rule.SilentPeriod != 0 { - alertRule.SilentPeriod = fmt.Sprintf("%dm", evalCtx.Rule.SilentPeriod/60) - } - return &alertRule -} diff --git a/pkg/monitor/alerting/rule.go b/pkg/monitor/alerting/rule.go index 70b138308d..8660293ad4 100644 --- a/pkg/monitor/alerting/rule.go +++ b/pkg/monitor/alerting/rule.go @@ -16,14 +16,12 @@ package alerting import ( "context" - "fmt" "regexp" "strconv" "time" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis/monitor" "yunion.io/x/onecloud/pkg/mcclient" @@ -61,7 +59,7 @@ type Rule struct { Notifications []string // AlertRuleTags []*models.AlertRuleTag Level string - RuleDescription []*RuleDescription + RuleDescription []*monitor.AlertRecordRule StateChanges int @@ -124,7 +122,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { model.NoDataState = monitor.NoDataOption(ruleDef.NoDataState) model.ExecutionErrorState = monitor.ExecutionErrorOption(ruleDef.ExecutionErrorState) model.StateChanges = ruleDef.StateChanges - model.RuleDescription = make([]*RuleDescription, 0) + model.RuleDescription = make([]*monitor.AlertRecordRule, 0) model.Frequency = ruleDef.Frequency // frequency cannot be zero since that would not execute the alert rule. @@ -135,7 +133,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { model.CustomizeConfig = ruleDef.CustomizeConfig settings, err := ruleDef.GetSettings() if err != nil { - return nil, err + return nil, errors.Wrap(err, "get settings") } model.Level = ruleDef.Level @@ -158,7 +156,6 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { return nil, errors.Wrap(err, "GetCommonAlert error") } for index, condition := range settings.Conditions { - alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(index, &settings.Conditions[index]) condType := condition.Type factory, exist := conditionFactories[condType] if !exist { @@ -168,7 +165,8 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { if err != nil { return nil, errors.Wrapf(err, "construct query condition %s", jsonutils.Marshal(condition)) } - newRuleDescription(model, alertDetails) + ruleDesc := alert.GetAlertRule(settings, index, model.SilentPeriod) + model.RuleDescription = append(model.RuleDescription, ruleDesc) model.Conditions = append(model.Conditions, queryCond) } @@ -178,62 +176,6 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { return model, nil } -func newRuleDescription(rule *Rule, alertDetails *monitor.CommonAlertMetricDetails) { - ruleDes := RuleDescription{ - AlertRecordRule: monitor.AlertRecordRule{ - ResType: alertDetails.ResType, - Metric: fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field), - Measurement: alertDetails.Measurement, - Database: alertDetails.DB, - MeasurementDesc: alertDetails.MeasurementDisplayName, - Field: alertDetails.Field, - FieldDesc: alertDetails.FieldDescription.DisplayName, - Comparator: alertDetails.Comparator, - Threshold: RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""), - ConditionType: alertDetails.ConditionType, - Reducer: alertDetails.Reduce, - }, - } - if len(ruleDes.ResType) == 0 { - if alertDetails.DB == monitor.METRIC_DATABASE_TELE { - ruleDes.ResType = monitor.METRIC_RES_TYPE_HOST - } - } - rule.RuleDescription = append(rule.RuleDescription, &ruleDes) -} - -var fileSize = []string{"bps", "Bps", "byte"} - -func RationalizeValueFromUnit(value float64, unit string, opt string) string { - if utils.IsInStringArray(unit, fileSize) { - if unit == "byte" { - return (FormatFileSize(value, unit, float64(1024))) - } - return FormatFileSize(value, unit, float64(1000)) - } - if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt { - return fmt.Sprintf("%0.4f%s", value*100, unit) - } - return fmt.Sprintf("%0.4f%s", value, unit) -} - -// 单位转换 保留四位小数 -func FormatFileSize(fileSize float64, unit string, unitsize float64) (size string) { - if fileSize < unitsize { - return fmt.Sprintf("%.4f%s", fileSize, unit) - } else if fileSize < (unitsize * unitsize) { - return fmt.Sprintf("%.4fK%s", float64(fileSize)/float64(unitsize), unit) - } else if fileSize < (unitsize * unitsize * unitsize) { - return fmt.Sprintf("%.4fM%s", float64(fileSize)/float64(unitsize*unitsize), unit) - } else if fileSize < (unitsize * unitsize * unitsize * unitsize) { - return fmt.Sprintf("%.4fG%s", float64(fileSize)/float64(unitsize*unitsize*unitsize), unit) - } else if fileSize < (unitsize * unitsize * unitsize * unitsize * unitsize) { - return fmt.Sprintf("%.4fT%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize), unit) - } else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024) - return fmt.Sprintf("%.4fE%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize*unitsize), unit) - } -} - type AlertRuleTester struct{} func NewAlertRuleTester() models.AlertTestRunner { diff --git a/pkg/monitor/models/alertrecord.go b/pkg/monitor/models/alertrecord.go index 0c73f37f0f..e4966f0b9a 100644 --- a/pkg/monitor/models/alertrecord.go +++ b/pkg/monitor/models/alertrecord.go @@ -332,9 +332,9 @@ getNewMatchTag: func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { record.SStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerId, query, data) - err := MonitorResourceManager.UpdateMonitorResourceAttachJoint(ctx, userCred, record) + err := MonitorResourceManager.UpdateMonitorResourceAttachJointByRecord(ctx, userCred, record) if err != nil { - log.Errorf("UpdateMonitorResourceAttachJoint error: %v", err) + log.Errorf("UpdateMonitorResourceAttachJointByRecord error: %v", err) } if err := GetAlertResourceManager().ReconcileFromRecord(ctx, userCred, ownerId, record); err != nil { log.Errorf("Reconcile from alert record error: %v", err) @@ -357,7 +357,6 @@ func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Con records := make([]SAlertRecord, 0) query := manager.Query() query = query.LE("created_at", timeutils.MysqlTime(time.Now().Add(-time.Hour*24*30))) - log.Errorf("query:%s", query.String()) err := db.FetchModelObjects(manager, query, &records) if err != nil { log.Errorf("fetch records ofthirty days ago err:%v", err) @@ -366,7 +365,7 @@ func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Con for i, _ := range records { err := db.DeleteModel(ctx, userCred, &records[i]) if err != nil { - log.Errorf("delete expire record:%s err:%v", records[i].GetId(), err) + log.Errorf("delete expire record: %s err: %v", records[i].GetId(), err) } } } diff --git a/pkg/monitor/models/commonalert.go b/pkg/monitor/models/commonalert.go index 88cd9cc8f3..31ea259e28 100644 --- a/pkg/monitor/models/commonalert.go +++ b/pkg/monitor/models/commonalert.go @@ -473,7 +473,7 @@ func (alert *SCommonAlert) PostCreate(ctx context.Context, log.Errorln(errors.Wrap(err, "Alert PerformSetScope")) } CommonAlertManager.SetSubscriptionAlert(alert) - alert.StartUpdateMonitorAlertJointTask(ctx, userCred) + //alert.StartUpdateMonitorAlertJointTask(ctx, userCred) } func (man *SCommonAlertManager) ListItemFilter( @@ -1112,7 +1112,7 @@ func (alert *SCommonAlert) PostUpdate( alert.setMetaName(ctx, userCred, updateInput.MetaName) } CommonAlertManager.SetSubscriptionAlert(alert) - alert.StartUpdateMonitorAlertJointTask(ctx, userCred) + //alert.StartUpdateMonitorAlertJointTask(ctx, userCred) } func (alert *SCommonAlert) UpdateNotification(ctx context.Context, userCred mcclient.TokenCredential, @@ -1322,7 +1322,7 @@ func (alert *SCommonAlert) PerformEnable(ctx context.Context, userCred mcclient. if err != nil { return nil, errors.Wrap(err, "EnabledPerformEnable") } - alert.StartUpdateMonitorAlertJointTask(ctx, userCred) + //alert.StartUpdateMonitorAlertJointTask(ctx, userCred) return nil, nil } @@ -1406,61 +1406,25 @@ func (alert *SCommonAlert) UpdateMonitorResourceJoint(ctx context.Context, userC if err != nil { return errors.Wrapf(err, "TestRunAlert %s", alert.GetName()) } - resourceIds := make([]string, 0) - for _, em := range ret.EvalMatches { - resourceKeyId := monitor.MEASUREMENT_TAG_ID[resType] - resourceId := em.Tags[resourceKeyId] - if len(resourceId) == 0 { - continue + if len(ret.AlertOKEvalMatches) > 0 { + matches := make([]monitor.EvalMatch, len(ret.AlertOKEvalMatches)) + for i := range ret.AlertOKEvalMatches { + matches[i] = *ret.AlertOKEvalMatches[i] } - resourceIds = append(resourceIds, resourceId) - } - deleteJointIds := make([]int64, 0) - joints, _ := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{AlertId: alert.GetId()}) -jointLoop: - for _, joint := range joints { - for i, resId := range resourceIds { - if resId == joint.MonitorResourceId { - resourceIds = append(resourceIds[0:i], resourceIds[i+1:]...) - continue jointLoop - } + input := &UpdateMonitorResourceAlertInput{ + AlertId: alert.GetId(), + Matches: matches, + ResType: resType, + AlertState: string(monitor.AlertStateOK), + SendState: monitor.SEND_STATE_SILENT, + TriggerTime: time.Now(), + AlertRecordId: "", } - // 排除近期有报警状态的情况：system.uptime - if joint.AlertState == monitor.MONITOR_RESOURCE_ALERT_STATUS_ALERTING && time.Now().Sub(joint.TriggerTime). - Minutes() < 30 { - continue + if err := MonitorResourceManager.UpdateMonitorResourceAttachJoint(ctx, userCred, input); err != nil { + return errors.Wrap(err, "UpdateMonitorResourceAttachJoint") } - deleteJointIds = append(deleteJointIds, joint.RowId) - } - - if len(resourceIds) == 0 && len(deleteJointIds) == 0 { return nil } - // sync joints should be deleted - if len(deleteJointIds) > 0 { - err := MonitorResourceAlertManager.DetachJoint(ctx, userCred, monitor.MonitorResourceJointListInput{JointId: deleteJointIds}) - if err != nil { - return errors.Wrapf(err, "DetachJoint by alert %s(%s)", alert.GetName(), alert.GetId()) - } - } - - if len(resourceIds) > 0 { - monitorResources, _ := MonitorResourceManager.GetMonitorResources(monitor.MonitorResourceListInput{ResId: resourceIds}) - errs := make([]error, 0) - for _, monRes := range monitorResources { - resDesc := fmt.Sprintf("%s/%s/%s", monRes.ResType, monRes.GetName(), monRes.ResId) - if _, err := monRes.AttachAlert(ctx, userCred, alert.GetId()); err != nil { - errs = append(errs, errors.Wrapf(err, "AttachAlert %s to %s", alert.GetName(), resDesc)) - } - if err := monRes.UpdateAlertState(); err != nil { - errs = append(errs, errors.Wrapf(err, "UpdateAlertState for monitor resource %s", resDesc)) - } - } - - if len(errs) != 0 { - return errors.NewAggregate(errs) - } - } return nil } @@ -1505,6 +1469,100 @@ func (alert *SCommonAlert) UpdateResType() error { return nil } +func (alert *SCommonAlert) GetSilentPeriod() (int64, error) { + notis, err := alert.GetNotifications() + if err != nil { + return 0, errors.Wrap(err, "GetNotifications") + } + for _, n := range notis { + noti, _ := n.GetNotification() + if noti.Frequency != 0 { + return noti.Frequency, nil + } + } + return 0, nil +} + +func (alert *SCommonAlert) GetAlertRules(silentPeriod int64) ([]*monitor.AlertRecordRule, error) { + rules := make([]*monitor.AlertRecordRule, 0) + settings, err := alert.GetSettings() + if err != nil { + return nil, errors.Wrapf(err, "get alert %s settings", alert.GetId()) + } + for index := range settings.Conditions { + rule := alert.GetAlertRule(settings, index, silentPeriod) + rules = append(rules, rule) + } + return rules, nil +} + +func (alert *SCommonAlert) GetAlertRule(settings *monitor.AlertSetting, index int, silentPeriod int64) *monitor.AlertRecordRule { + alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(index, &settings.Conditions[index]) + rule := &monitor.AlertRecordRule{ + ResType: alertDetails.ResType, + Metric: fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field), + Measurement: alertDetails.Measurement, + Database: alertDetails.DB, + MeasurementDesc: alertDetails.MeasurementDisplayName, + Field: alertDetails.Field, + FieldDesc: alertDetails.FieldDescription.DisplayName, + Comparator: alertDetails.Comparator, + Threshold: RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""), + ConditionType: alertDetails.ConditionType, + Reducer: alertDetails.Reduce, + } + if len(rule.ResType) == 0 { + if alertDetails.DB == monitor.METRIC_DATABASE_TELE { + rule.ResType = monitor.METRIC_RES_TYPE_HOST + } + } + if alert.Frequency < 60 { + rule.Period = fmt.Sprintf("%ds", alert.Frequency) + } else { + rule.Period = fmt.Sprintf("%dm", alert.Frequency/60) + } + rule.AlertDuration = alert.For / alert.Frequency + if rule.AlertDuration == 0 { + rule.AlertDuration = 1 + } + if silentPeriod > 0 { + rule.SilentPeriod = fmt.Sprintf("%dm", silentPeriod/60) + } + return rule +} + +var fileSize = []string{"bps", "Bps", "byte"} + +func RationalizeValueFromUnit(value float64, unit string, opt string) string { + if utils.IsInStringArray(unit, fileSize) { + if unit == "byte" { + return (FormatFileSize(value, unit, float64(1024))) + } + return FormatFileSize(value, unit, float64(1000)) + } + if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt { + return fmt.Sprintf("%0.4f%s", value*100, unit) + } + return fmt.Sprintf("%0.4f%s", value, unit) +} + +// 单位转换 保留四位小数 +func FormatFileSize(fileSize float64, unit string, unitsize float64) (size string) { + if fileSize < unitsize { + return fmt.Sprintf("%.4f%s", fileSize, unit) + } else if fileSize < (unitsize * unitsize) { + return fmt.Sprintf("%.4fK%s", float64(fileSize)/float64(unitsize), unit) + } else if fileSize < (unitsize * unitsize * unitsize) { + return fmt.Sprintf("%.4fM%s", float64(fileSize)/float64(unitsize*unitsize), unit) + } else if fileSize < (unitsize * unitsize * unitsize * unitsize) { + return fmt.Sprintf("%.4fG%s", float64(fileSize)/float64(unitsize*unitsize*unitsize), unit) + } else if fileSize < (unitsize * unitsize * unitsize * unitsize * unitsize) { + return fmt.Sprintf("%.4fT%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize), unit) + } else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024) + return fmt.Sprintf("%.4fE%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize*unitsize), unit) + } +} + type SCompanyInfo struct { Copyright string `json:"copyright"` Name string `json:"name"` diff --git a/pkg/monitor/models/monitor_resource.go b/pkg/monitor/models/monitor_resource.go index 2d1415aee9..468c8a75cc 100644 --- a/pkg/monitor/models/monitor_resource.go +++ b/pkg/monitor/models/monitor_resource.go @@ -262,12 +262,14 @@ func (man *SMonitorResourceManager) FetchCustomizeColumns( return rows } -func (self *SMonitorResource) AttachAlert(ctx context.Context, userCred mcclient.TokenCredential, alertId string) (*SMonitorResourceAlert, error) { +func (self *SMonitorResource) AttachAlert(ctx context.Context, userCred mcclient.TokenCredential, alertId string, metric string, match monitor.EvalMatch) (*SMonitorResourceAlert, error) { iModel, _ := db.NewModelObject(MonitorResourceAlertManager) input := monitor.MonitorResourceJointCreateInput{ MonitorResourceId: self.ResId, AlertId: alertId, AlertState: monitor.MONITOR_RESOURCE_ALERT_STATUS_ATTACH, + Metric: metric, + Data: match, } data := input.JSON(&input) err := data.Unmarshal(iModel) @@ -379,21 +381,47 @@ func (manager *SMonitorResourceManager) GetPropertyAlert(ctx context.Context, us return result, nil } -func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context, - userCred mcclient.TokenCredential, alertRecord *SAlertRecord) error { +func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJointByRecord(ctx context.Context, userCred mcclient.TokenCredential, record *SAlertRecord) error { + matches, _ := record.GetEvalData() + input := &UpdateMonitorResourceAlertInput{ + AlertId: record.AlertId, + Matches: matches, + ResType: record.ResType, + AlertState: record.State, + SendState: record.SendState, + TriggerTime: record.CreatedAt, + AlertRecordId: record.GetId(), + } + if err := manager.UpdateMonitorResourceAttachJoint(ctx, userCred, input); err != nil { + return errors.Wrap(err, "UpdateMonitorResourceAttachJoint") + } + return nil +} + +type UpdateMonitorResourceAlertInput struct { + AlertId string + Matches []monitor.EvalMatch + ResType string + AlertState string + SendState string + TriggerTime time.Time + AlertRecordId string +} + +func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context, userCred mcclient.TokenCredential, input *UpdateMonitorResourceAlertInput) error { //if !utils.IsInStringArray(alertRecord.ResType, []string{monitor.METRIC_RES_TYPE_HOST, // monitor.METRIC_RES_TYPE_GUEST, monitor.METRIC_RES_TYPE_AGENT}) { // return nil //} - resType := alertRecord.ResType + resType := input.ResType if resType == monitor.METRIC_RES_TYPE_AGENT { resType = monitor.METRIC_RES_TYPE_GUEST } - matches, _ := alertRecord.GetEvalData() + matches := input.Matches errs := make([]error, 0) matchResourceIds := make([]string, 0) - for _, matche := range matches { - resId := matche.Tags[monitor.MEASUREMENT_TAG_ID[alertRecord.ResType]] + for _, match := range matches { + resId := match.Tags[monitor.MEASUREMENT_TAG_ID[input.ResType]] if len(resId) == 0 { continue } @@ -404,24 +432,22 @@ func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx con continue } for _, res := range monitorResources { - err := res.UpdateAttachJoint(alertRecord, matche) + err := res.UpdateAttachJoint(ctx, userCred, input, match) if err != nil { - errs = append(errs, err) + errs = append(errs, errors.Wrap(err, "UpdateAttachJoint")) } } } - resourceAlerts, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{AlertId: alertRecord.AlertId}) + resourceAlerts, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{ + AlertId: input.AlertId, + AlertState: input.AlertState, + }) if err != nil { - return errors.Wrapf(err, "get monitor_resource_joint by alertId:%s err", alertRecord.AlertId) + return errors.Wrapf(err, "get monitor_resource_joint by alertId: %s", input.AlertId) } deleteJointIds := make([]int64, 0) for _, joint := range resourceAlerts { - evalData, err := joint.GetData() - if err != nil { - log.Warningf("get data of monitor_resource_alert %s: %s", jsonutils.Marshal(joint), err) - continue - } - metricName := evalData.Metric + metricName := joint.Metric isMetricFound := false for _, match := range matches { if match.Metric == metricName { @@ -437,51 +463,49 @@ func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx con if len(deleteJointIds) != 0 { err = MonitorResourceAlertManager.DetachJoint(ctx, userCred, monitor.MonitorResourceJointListInput{JointId: deleteJointIds}) if err != nil { - return errors.Wrapf(err, "DetachJoint by alertId:%s err", alertRecord.AlertId) + return errors.Wrapf(err, "DetachJoint by alertId:%s err", input.AlertId) } } return errors.NewAggregate(errs) } -func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match monitor.EvalMatch) error { - joints, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{MonitorResourceId: self. - ResId, AlertId: alertRecord.AlertId}) +func (self *SMonitorResource) UpdateAttachJoint(ctx context.Context, userCred mcclient.TokenCredential, input *UpdateMonitorResourceAlertInput, match monitor.EvalMatch) error { + joints, err := MonitorResourceAlertManager.GetJoinsByListInput( + monitor.MonitorResourceJointListInput{ + MonitorResourceId: self.ResId, + AlertId: input.AlertId, + Metric: match.Metric, + }) if err != nil { - return errors.Wrapf(err, "SMonitorResource:%s UpdateAttachJoint err", self.Name) + return errors.Wrapf(err, "SMonitorResource: %s(%s) get joints by monitorResourceId %q , metric %q and alertId %q", self.Name, self.Id, self.ResId, match.Metric, input.AlertId) } errs := make([]error, 0) updateJoints := make([]SMonitorResourceAlert, 0) for _, joint := range joints { - jointMatch := new(monitor.EvalMatch) - if joint.Data != nil { - if err := joint.Data.Unmarshal(jointMatch); err != nil { - return errors.Wrapf(err, "unmarshal joint %s to monitor.EvalMatch", jsonutils.Marshal(joint)) - } - if jointMatch.Metric == match.Metric { - tmpJoint := joint - updateJoints = append(updateJoints, tmpJoint) - } + if joint.Metric == match.Metric { + tmpJoint := joint + updateJoints = append(updateJoints, tmpJoint) } } // 报警时发现没有进行关联，增加attach if len(updateJoints) == 0 { - newJoint, err := self.AttachAlert(context.Background(), nil, alertRecord.AlertId) + newJoint, err := self.AttachAlert(ctx, userCred, input.AlertId, match.Metric, match) if err != nil { log.Errorf("attach alert error: %s", err) } log.Infof("Attach Alert joint: %#v, match: %s", newJoint, jsonutils.Marshal(match)) - if err := newJoint.UpdateAlertRecordData(alertRecord, &match); err != nil { + if err := newJoint.UpdateAlertRecordData(input, &match); err != nil { errs = append(errs, errors.Wrapf(err, "new joint %s:%s %s:%s UpdateAlertRecordData err", MonitorResourceAlertManager.GetMasterFieldName(), self.ResId, - MonitorResourceAlertManager.GetSlaveFieldName(), alertRecord.AlertId)) + MonitorResourceAlertManager.GetSlaveFieldName(), input.AlertId)) } } else { for _, joint := range updateJoints { - err := joint.UpdateAlertRecordData(alertRecord, &match) + err := joint.UpdateAlertRecordData(input, &match) if err != nil { errs = append(errs, errors.Wrapf(err, "joint %s:%s %s:%s UpdateAlertRecordData err", MonitorResourceAlertManager.GetMasterFieldName(), self.ResId, - MonitorResourceAlertManager.GetSlaveFieldName(), alertRecord.AlertId)) + MonitorResourceAlertManager.GetSlaveFieldName(), input.AlertId)) } } } @@ -489,7 +513,6 @@ func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match errs = append(errs, errors.Wrapf(err, "UpdateAlertState")) } return errors.NewAggregate(errs) - } func (manager *SMonitorResourceManager) GetResourceObj(id string) (bool, jsonutils.JSONObject) { diff --git a/pkg/monitor/models/monitor_resource_alert.go b/pkg/monitor/models/monitor_resource_alert.go index e70e2fb627..567cf8d50b 100644 --- a/pkg/monitor/models/monitor_resource_alert.go +++ b/pkg/monitor/models/monitor_resource_alert.go @@ -58,6 +58,7 @@ type SMonitorResourceAlert struct { AlertId string `width:"36" charset:"ascii" list:"user" create:"required" index:"true"` AlertRecordId string `width:"36" charset:"ascii" list:"user" update:"user"` ResType string `width:"36" charset:"ascii" list:"user" update:"user" json:"res_type"` + Metric string `width:"36" charset:"ascii" list:"user" create:"required" json:"metric"` AlertState string `width:"18" charset:"ascii" default:"init" list:"user" update:"user"` SendState string `width:"18" charset:"ascii" default:"ok" list:"user" update:"user"` TriggerTime time.Time `list:"user" update:"user" json:"trigger_time"` @@ -123,9 +124,15 @@ func (manager *SMonitorResourceAlertManager) GetJoinsByListInput(input monitor.M if len(input.AlertId) != 0 { query.Equals(manager.GetSlaveFieldName(), input.AlertId) } + if len(input.Metric) > 0 { + query.Equals("metric", input.Metric) + } if len(input.JointId) != 0 { query.In("row_id", input.JointId) } + if len(input.AlertState) > 0 { + query = query.Equals("alert_state", input.AlertState) + } err := db.FetchModelObjects(manager, query, &joints) if err != nil { return nil, errors.Wrapf(err, "FetchModelObjects by GetJoinsByMasterId:%s err", input.MonitorResourceId) @@ -133,21 +140,24 @@ func (manager *SMonitorResourceAlertManager) GetJoinsByListInput(input monitor.M return joints, nil } -func (obj *SMonitorResourceAlert) UpdateAlertRecordData(record *SAlertRecord, match *monitor.EvalMatch) error { - sendState := record.SendState +func (obj *SMonitorResourceAlert) UpdateAlertRecordData(input *UpdateMonitorResourceAlertInput, match *monitor.EvalMatch) error { + sendState := input.SendState if _, ok := match.Tags[monitor.ALERT_RESOURCE_RECORD_SHIELD_KEY]; ok { sendState = monitor.SEND_STATE_SHIELD } if _, err := db.Update(obj, func() error { - obj.AlertRecordId = record.GetId() - obj.ResType = record.ResType - obj.AlertState = record.State + if input.AlertRecordId != "" { + obj.AlertRecordId = input.AlertRecordId + } + obj.ResType = input.ResType + obj.AlertState = input.AlertState obj.SendState = sendState - obj.TriggerTime = record.CreatedAt + obj.TriggerTime = input.TriggerTime + obj.Metric = match.Metric obj.Data = jsonutils.Marshal(match) return nil }); err != nil { - return err + return errors.Wrap(err, "db.Update") } return nil } @@ -169,6 +179,9 @@ func (m *SMonitorResourceAlertManager) ListItemFilter(ctx context.Context, q *sq if err != nil { return q, errors.Wrap(err, "SJointResourceBaseManager ListItemFilter err") } + if len(input.AlertState) > 0 { + q = q.Equals("alert_state", input.AlertState) + } if input.Alerting { q = q.Equals("alert_state", monitor.AlertStateAlerting) resQ := MonitorResourceManager.Query("res_id") @@ -207,8 +220,10 @@ func (m *SMonitorResourceAlertManager) ListItemFilter(ctx context.Context, q *sq if len(input.MonitorResourceId) != 0 { q = q.Filter(sqlchemy.Equals(q.Field("monitor_resource_id"), input.MonitorResourceId)) } - q = q.Filter(sqlchemy.In(q.Field(m.GetSlaveFieldName()), alertQuery.SubQuery())) - q = q.Filter(sqlchemy.In(q.Field("alert_record_id"), AlertRecordManager.Query("id").SubQuery())) + if !input.AllState { + q = q.Filter(sqlchemy.In(q.Field(m.GetSlaveFieldName()), alertQuery.SubQuery())) + q = q.Filter(sqlchemy.In(q.Field("alert_record_id"), AlertRecordManager.Query("id").SubQuery())) + } return q, nil } @@ -228,8 +243,7 @@ func (man *SMonitorResourceAlertManager) FetchCustomizeColumns( return rows } -func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJointDetails) monitor. - MonitorResourceJointDetails { +func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJointDetails) monitor.MonitorResourceJointDetails { resources, err := MonitorResourceManager.GetMonitorResources(monitor.MonitorResourceListInput{ResId: []string{obj. MonitorResourceId}}) if err != nil { @@ -249,7 +263,6 @@ func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJ log.Errorf("get alertRecord:%s err:%v", obj.AlertRecordId, err) return detail } - detail.Level = record.Level detail.AlertRule = record.AlertRule detail.SendState = record.SendState detail.State = record.State @@ -260,13 +273,18 @@ func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJ return detail } detail.AlertName = alert.Name + detail.Level = alert.Level + if len(obj.AlertRecordId) == 0 { + silentPeriod, _ := alert.GetSilentPeriod() + rule, _ := alert.GetAlertRules(silentPeriod) + detail.AlertRule = jsonutils.Marshal(rule) + } now := time.Now() shields, err := AlertRecordShieldManager.GetRecordShields(monitor.AlertRecordShieldListInput{ResId: obj.MonitorResourceId, AlertId: obj.AlertId, EndTime: &now}) if err != nil { - log.Errorf("SMonitorResourceAlert get GetRecordShields by resId: %s,alertId: %s, err: %v", - obj.MonitorResourceId, obj.AlertId, err) + log.Errorf("SMonitorResourceAlert get GetRecordShields by resId: %s, alertId: %s, err: %v", obj.MonitorResourceId, obj.AlertId, err) return detail } if len(shields) != 0 {