diff --git a/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT b/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT index df4b0d3d6e..099431554f 100644 --- a/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT +++ b/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT @@ -3,7 +3,7 @@ 触发时间: {{.start_time}} 报警级别: {{.level}} 触发条件: {{.description | unescaped}} -资源数量：{{len .matches}} +报警数量：{{len .matches}} 资源名称：{{.resource_name}} 详情地址: {{.web_url}} \ No newline at end of file diff --git a/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT b/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT index 723fd671e2..26b97e70bc 100644 --- a/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT +++ b/build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT @@ -3,7 +3,7 @@ AlertName: {{.name}} Time: {{.start_time}} Level: {{.level}} TriggerCondition: {{.description | unescaped}} -ResourceCount: {{len .matches}} +AlertCount: {{len .matches}} ResourceName: {{.resource_name}} WebUrl: {{.web_url}} diff --git a/cmd/climc/shell/monitor/commonalert.go b/cmd/climc/shell/monitor/commonalert.go index 2303469772..27ff72cebd 100644 --- a/cmd/climc/shell/monitor/commonalert.go +++ b/cmd/climc/shell/monitor/commonalert.go @@ -15,10 +15,136 @@ package monitor import ( + "yunion.io/x/jsonutils" + + monitorapi "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" options "yunion.io/x/onecloud/pkg/mcclient/options/monitor" ) +type CommonAlertTerm struct { + Database string + Measurement string + Operator string // and / or + Field []string + FieldFunc string + + Reduce string + Comparator string + Threshold float64 + Filters []monitorapi.MetricQueryTag + FieldOpt string + Name string + ConditionType string + From string + Interval string + GroupBy string + Level string +} + +func newCommonAlertQuery(tem *CommonAlertTerm) *monitorapi.CommonAlertQuery { + mq := monitorapi.MetricQuery{ + Alias: "", + Tz: "", + Database: tem.Database, + Measurement: tem.Measurement, + Tags: make([]monitorapi.MetricQueryTag, 0), + GroupBy: make([]monitorapi.MetricQueryPart, 0), + Selects: nil, + Interval: "", + Policy: "", + ResultFormat: "", + } + + for _, field := range tem.Field { + sel := monitorapi.MetricQueryPart{ + Type: "field", + Params: []string{field}, + } + selectPart := []monitorapi.MetricQueryPart{sel} + if len(tem.FieldFunc) != 0 { + selectPart = append(selectPart, monitorapi.MetricQueryPart{ + Type: tem.FieldFunc, + Params: []string{}, + }) + } else { + selectPart = append(selectPart, monitorapi.MetricQueryPart{ + Type: "mean", + Params: []string{}, + }) + } + mq.Selects = append(mq.Selects, selectPart) + } + if len(tem.Filters) != 0 { + for _, filter := range tem.Filters { + mq.Tags = append(mq.Tags, filter) + } + } + + alertQ := new(monitorapi.AlertQuery) + alertQ.Model = mq + alertQ.From = "60m" + + commonAlert := monitorapi.CommonAlertQuery{ + AlertQuery: alertQ, + Reduce: tem.Reduce, + Comparator: tem.Comparator, + Threshold: tem.Threshold, + Operator: tem.Operator, + } + if tem.FieldOpt != "" { + commonAlert.FieldOpt = monitorapi.CommonAlertFieldOpt_Division + } + if len(tem.ConditionType) != 0 { + commonAlert.ConditionType = tem.ConditionType + } + if len(tem.GroupBy) != 0 { + commonAlert.Model.GroupBy = append(commonAlert.Model.GroupBy, monitorapi.MetricQueryPart{ + Type: "field", + Params: []string{tem.GroupBy}, + }) + } + return &commonAlert +} + +var ( + cpuTem = &CommonAlertTerm{ + Operator: "or", + Database: "telegraf", + Measurement: "vm_cpu", + Field: []string{"usage_active"}, + Comparator: ">=", + Reduce: "avg", + Threshold: 50, + Name: "lzx-test.cpu.usage_active", + Filters: []monitorapi.MetricQueryTag{ + { + Key: "id", + Operator: "=", + Value: "a0eee5dd-3cfe-4ab1-8c79-aee1a8cf4dab", + }, + }, + } + memTem = &CommonAlertTerm{ + Operator: "or", + Database: "telegraf", + Measurement: "vm_mem", + Field: []string{"used_percent"}, + Comparator: ">=", + Reduce: "avg", + Threshold: 3, + Name: "lzx-test.mem.avaiable", + Filters: []monitorapi.MetricQueryTag{ + { + Key: "id", + Operator: "=", + Value: "a0eee5dd-3cfe-4ab1-8c79-aee1a8cf4dab", + }, + }, + } +) + func init() { cmd := NewResourceCmd(modules.CommonAlerts) cmd.Create(new(options.CommonAlertCreateOptions)) @@ -28,4 +154,34 @@ func init() { cmd.Perform("disable", &options.CommonAlertShowOptions{}) cmd.BatchDelete(new(options.CommonAlertDeleteOptions)) cmd.Perform("config", &options.CommonAlertUpdateOptions{}) + + type TestOpt struct { + NAME string + RobotIds []string + Users []string + } + R(&TestOpt{}, "monitor-commonalert-create-mul-test", "create test monitor common alert", func(s *mcclient.ClientSession, opt *TestOpt) error { + cpuQ := newCommonAlertQuery(cpuTem) + memQ := newCommonAlertQuery(memTem) + input := monitorapi.CommonAlertCreateInput{ + CommonMetricInputQuery: monitorapi.CommonMetricInputQuery{ + MetricQuery: []*monitorapi.CommonAlertQuery{ + cpuQ, + memQ, + }, + }, + AlertCreateInput: monitorapi.AlertCreateInput{ + Name: opt.NAME, + }, + CommonAlertCreateBaseInput: monitorapi.CommonAlertCreateBaseInput{ + Recipients: opt.Users, + RobotIds: opt.RobotIds, + Channel: []string{"webconsole"}, + AlertType: monitorapi.CommonAlertNomalAlertType, + Scope: "system", + }, + } + _, err := modules.CommonAlerts.Create(s, jsonutils.Marshal(input)) + return err + }) } diff --git a/pkg/apis/monitor/alertrecord.go b/pkg/apis/monitor/alertrecord.go index ba3493e02c..34784350d2 100644 --- a/pkg/apis/monitor/alertrecord.go +++ b/pkg/apis/monitor/alertrecord.go @@ -71,12 +71,12 @@ type AlertRecordCreateInput struct { AlertId string `json:"alert_id"` // 报警级别 - Level string `json:"level"` - State string `json:"state"` - SendState string `json:"send_state"` - ResType string `json:"res_type"` - EvalData []*EvalMatch `json:"eval_data"` - AlertRule AlertRecordRule + Level string `json:"level"` + State string `json:"state"` + SendState string `json:"send_state"` + ResType string `json:"res_type"` + EvalData []*EvalMatch `json:"eval_data"` + AlertRule []*AlertRecordRule `json:"alert_rule"` } type AlertRecordRule struct { diff --git a/pkg/apis/monitor/commalert.go b/pkg/apis/monitor/commalert.go index b7993ce2e0..592927c997 100644 --- a/pkg/apis/monitor/commalert.go +++ b/pkg/apis/monitor/commalert.go @@ -103,6 +103,8 @@ type CommonAlertQuery struct { //field yunsuan FieldOpt string `json:"field_opt"` ConditionType string `json:"condition_type"` + // Operator should be chosen from 'and | or' + Operator string `json:"operator"` } type CommonAlertListInput struct { @@ -159,6 +161,7 @@ type CommonAlertDetails struct { } type CommonAlertMetricDetails struct { + Operator string `json:"operator"` Comparator string `json:"comparator"` Threshold float64 `json:"threshold"` WithinRange []float64 `json:"within_range"` diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index 307ff6cfd6..2937bcc0e1 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -309,6 +309,11 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series moni if len(context.Rule.Message) == 0 { context.Rule.Message = msg } + op := alertDetails.Operator + if op != "" && c.Index > 0 { + msg = fmt.Sprintf("%s %s", strings.ToUpper(op), msg) + } + context.Rule.TriggeredMessages = append(context.Rule.TriggeredMessages, msg) return evalMatch, nil } diff --git a/pkg/monitor/alerting/eval_context.go b/pkg/monitor/alerting/eval_context.go index 5e475b1055..c4b170addb 100644 --- a/pkg/monitor/alerting/eval_context.go +++ b/pkg/monitor/alerting/eval_context.go @@ -199,6 +199,9 @@ func getNewStateInternal(c *EvalContext) monitor.AlertStateType { func (c *EvalContext) GetNotificationTemplateConfig() monitor.NotificationTemplateConfig { desc := c.Rule.Message + if len(c.Rule.TriggeredMessages) > 0 { + desc = strings.Join(c.Rule.TriggeredMessages, " ") + } if c.Error != nil { if desc != "" { desc += "\n" @@ -248,10 +251,9 @@ func (c *EvalContext) GetResourceNameOfMathes(matches []monitor.EvalMatch) strin } for i, match := range matches { if name, ok := match.Tags["name"]; ok { - names.WriteString(name) - names.WriteString(fmt.Sprintf("(%s)", match.ValueStr)) + names.WriteString(fmt.Sprintf("%s.%s(%s)", name, match.Metric, match.ValueStr)) if i < len(matches)-1 { - names.WriteString("、") + names.WriteString(", ") } } } diff --git a/pkg/monitor/alerting/notifier.go b/pkg/monitor/alerting/notifier.go index 5d27f2be15..cb8e454182 100644 --- a/pkg/monitor/alerting/notifier.go +++ b/pkg/monitor/alerting/notifier.go @@ -176,12 +176,12 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, State: string(evalCtx.Rule.State), SendState: monitor.SEND_STATE_OK, EvalData: matches, - AlertRule: newAlertRecordRule(evalCtx), + AlertRule: newAlertRecordRules(evalCtx), } if !shouldNotify { recordCreateInput.SendState = monitor.SEND_STATE_SILENT } - recordCreateInput.ResType = recordCreateInput.AlertRule.ResType + recordCreateInput.ResType = recordCreateInput.AlertRule[0].ResType if len(recordCreateInput.ResType) == 0 { recordCreateInput.ResType = monitor.METRIC_RES_TYPE_HOST } @@ -272,10 +272,19 @@ func InitNotifier(config NotificationConfig) (Notifier, error) { return plug.(Notifier), nil } -func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule { +func newAlertRecordRules(ctx *EvalContext) []*monitor.AlertRecordRule { + rules := make([]*monitor.AlertRecordRule, 0) + for i := range ctx.Rule.RuleDescription { + rule := newAlertRecordRule(ctx, i) + rules = append(rules, rule) + } + return rules +} + +func newAlertRecordRule(evalCtx *EvalContext, idx int) *monitor.AlertRecordRule { alertRule := monitor.AlertRecordRule{} if len(evalCtx.Rule.RuleDescription) != 0 { - alertRule = evalCtx.Rule.RuleDescription[0].AlertRecordRule + alertRule = evalCtx.Rule.RuleDescription[idx].AlertRecordRule } if evalCtx.Rule.Frequency < 60 { alertRule.Period = fmt.Sprintf("%ds", evalCtx.Rule.Frequency) @@ -291,5 +300,5 @@ func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule { if evalCtx.Rule.SilentPeriod != 0 { alertRule.SilentPeriod = fmt.Sprintf("%dm", evalCtx.Rule.SilentPeriod/60) } - return alertRule + return &alertRule } diff --git a/pkg/monitor/alerting/rule.go b/pkg/monitor/alerting/rule.go index 258be0fca7..ffec3bc482 100644 --- a/pkg/monitor/alerting/rule.go +++ b/pkg/monitor/alerting/rule.go @@ -45,11 +45,13 @@ func init() { // Rule is the in-memory version of an alert rule. type Rule struct { - Id string - Frequency int64 - Title string - Name string - Message string + Id string + Frequency int64 + Title string + Name string + Message string + // 使用 TriggeredMessages 存储触发的条件，替代 Message + TriggeredMessages []string LastStateChange time.Time For time.Duration NoDataState monitor.NoDataOption @@ -115,6 +117,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { model.Title = ruleDef.GetTitle() model.Name = ruleDef.Name model.Message = ruleDef.Message + model.TriggeredMessages = make([]string, 0) model.State = monitor.AlertStateType(ruleDef.State) model.LastStateChange = ruleDef.LastStateChange model.For = time.Duration(ruleDef.For) diff --git a/pkg/monitor/models/commonalert.go b/pkg/monitor/models/commonalert.go index 1902cc4720..fd0edb1cb0 100644 --- a/pkg/monitor/models/commonalert.go +++ b/pkg/monitor/models/commonalert.go @@ -269,14 +269,16 @@ func (man *SCommonAlertManager) ValidateCreateData( } data.Name = name - alertCreateInput := man.toAlertCreatInput(data) + alertCreateInput, err := man.toAlertCreatInput(data) + if err != nil { + return data, errors.Wrap(err, "to alert creation input") + } alertCreateInput, err = AlertManager.ValidateCreateData(ctx, userCred, ownerId, query, alertCreateInput) if err != nil { return data, err } data.AlertCreateInput = alertCreateInput return data, nil - } func (man *SCommonAlertManager) genName(ctx context.Context, ownerId mcclient.IIdentityProvider, name string) (string, @@ -841,6 +843,7 @@ func getCommonAlertMetricDetailsFromCondition(cond *monitor.AlertCondition, metricDetails.DB = db metricDetails.Groupby = groupby metricDetails.Filters = cond.Query.Model.Tags + metricDetails.Operator = cond.Operator //fill measurement\field desciption info getMetricDescriptionDetails(metricDetails) @@ -924,7 +927,7 @@ func getQueryEvalType(evalType string) string { return typ } -func (man *SCommonAlertManager) toAlertCreatInput(input monitor.CommonAlertCreateInput) monitor.AlertCreateInput { +func (man *SCommonAlertManager) toAlertCreatInput(input monitor.CommonAlertCreateInput) (monitor.AlertCreateInput, error) { freq, _ := time.ParseDuration(input.Period) ret := new(monitor.AlertCreateInput) ret.Name = input.Name @@ -947,12 +950,18 @@ func (man *SCommonAlertManager) toAlertCreatInput(input monitor.CommonAlertCreat Params: []float64{fieldOperatorThreshold(metricquery.FieldOpt, metricquery.Threshold)}}, Operator: "and", } + if metricquery.Operator != "" { + if !sets.NewString("and", "or").Has(metricquery.Operator) { + return *ret, httperrors.NewInputParameterError("invalid operator %s", metricquery.Operator) + } + condition.Operator = metricquery.Operator + } if metricquery.FieldOpt != "" { condition.Reducer.Operators = []string{metricquery.FieldOpt} } ret.Settings.Conditions = append(ret.Settings.Conditions, condition) } - return *ret + return *ret, nil } func fieldOperatorThreshold(opt string, threshold float64) float64 { @@ -1053,7 +1062,10 @@ func (alert *SCommonAlert) ValidateUpdateData( if err != nil { return data, errors.Wrap(err, "updataInput Unmarshal err") } - alertCreateInput := alert.getUpdateAlertInput(*updataInput) + alertCreateInput, err := alert.getUpdateAlertInput(*updataInput) + if err != nil { + return data, errors.Wrap(err, "getUpdateAlertInput") + } alertCreateInput, err = AlertManager.ValidateCreateData(ctx, userCred, nil, query, alertCreateInput) if err != nil { return data, err @@ -1120,14 +1132,13 @@ func (alert *SCommonAlert) UpdateNotification(ctx context.Context, userCred mccl return err } -func (alert *SCommonAlert) getUpdateAlertInput(updateInput monitor.CommonAlertUpdateInput) monitor.AlertCreateInput { +func (alert *SCommonAlert) getUpdateAlertInput(updateInput monitor.CommonAlertUpdateInput) (monitor.AlertCreateInput, error) { input := monitor.CommonAlertCreateInput{ CommonMetricInputQuery: updateInput.CommonMetricInputQuery, Period: updateInput.Period, } input.AlertDuration = updateInput.AlertDuration - alertCreateInput := CommonAlertManager.toAlertCreatInput(input) - return alertCreateInput + return CommonAlertManager.toAlertCreatInput(input) } func (alert *SCommonAlert) CustomizeDelete(