feat(monitor): support multiple metrics alert (#20156)

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT

@@ -3,7 +3,7 @@
 触发时间: {{.start_time}}
 报警级别: {{.level}}
 触发条件: {{.description | unescaped}}
-资源数量：{{len .matches}}
+报警数量：{{len .matches}}
 资源名称：{{.resource_name}}
 
 详情地址: {{.web_url}}

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT

@@ -3,7 +3,7 @@ AlertName: {{.name}}
 Time: {{.start_time}}
 Level: {{.level}}
 TriggerCondition: {{.description | unescaped}}
-ResourceCount: {{len .matches}}
+AlertCount: {{len .matches}}
 ResourceName: {{.resource_name}}
 
 WebUrl: {{.web_url}}

cmd/climc/shell/monitor/commonalert.go

@@ -15,10 +15,136 @@
 package monitor
 
 import (
+	"yunion.io/x/jsonutils"
+
+	monitorapi "yunion.io/x/onecloud/pkg/apis/monitor"
+	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
 	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
 )
 
+type CommonAlertTerm struct {
+	Database    string
+	Measurement string
+	Operator    string // and / or
+	Field       []string
+	FieldFunc   string
+
+	Reduce        string
+	Comparator    string
+	Threshold     float64
+	Filters       []monitorapi.MetricQueryTag
+	FieldOpt      string
+	Name          string
+	ConditionType string
+	From          string
+	Interval      string
+	GroupBy       string
+	Level         string
+}
+
+func newCommonAlertQuery(tem *CommonAlertTerm) *monitorapi.CommonAlertQuery {
+	mq := monitorapi.MetricQuery{
+		Alias:        "",
+		Tz:           "",
+		Database:     tem.Database,
+		Measurement:  tem.Measurement,
+		Tags:         make([]monitorapi.MetricQueryTag, 0),
+		GroupBy:      make([]monitorapi.MetricQueryPart, 0),
+		Selects:      nil,
+		Interval:     "",
+		Policy:       "",
+		ResultFormat: "",
+	}
+
+	for _, field := range tem.Field {
+		sel := monitorapi.MetricQueryPart{
+			Type:   "field",
+			Params: []string{field},
+		}
+		selectPart := []monitorapi.MetricQueryPart{sel}
+		if len(tem.FieldFunc) != 0 {
+			selectPart = append(selectPart, monitorapi.MetricQueryPart{
+				Type:   tem.FieldFunc,
+				Params: []string{},
+			})
+		} else {
+			selectPart = append(selectPart, monitorapi.MetricQueryPart{
+				Type:   "mean",
+				Params: []string{},
+			})
+		}
+		mq.Selects = append(mq.Selects, selectPart)
+	}
+	if len(tem.Filters) != 0 {
+		for _, filter := range tem.Filters {
+			mq.Tags = append(mq.Tags, filter)
+		}
+	}
+
+	alertQ := new(monitorapi.AlertQuery)
+	alertQ.Model = mq
+	alertQ.From = "60m"
+
+	commonAlert := monitorapi.CommonAlertQuery{
+		AlertQuery: alertQ,
+		Reduce:     tem.Reduce,
+		Comparator: tem.Comparator,
+		Threshold:  tem.Threshold,
+		Operator:   tem.Operator,
+	}
+	if tem.FieldOpt != "" {
+		commonAlert.FieldOpt = monitorapi.CommonAlertFieldOpt_Division
+	}
+	if len(tem.ConditionType) != 0 {
+		commonAlert.ConditionType = tem.ConditionType
+	}
+	if len(tem.GroupBy) != 0 {
+		commonAlert.Model.GroupBy = append(commonAlert.Model.GroupBy, monitorapi.MetricQueryPart{
+			Type:   "field",
+			Params: []string{tem.GroupBy},
+		})
+	}
+	return &commonAlert
+}
+
+var (
+	cpuTem = &CommonAlertTerm{
+		Operator:    "or",
+		Database:    "telegraf",
+		Measurement: "vm_cpu",
+		Field:       []string{"usage_active"},
+		Comparator:  ">=",
+		Reduce:      "avg",
+		Threshold:   50,
+		Name:        "lzx-test.cpu.usage_active",
+		Filters: []monitorapi.MetricQueryTag{
+			{
+				Key:      "id",
+				Operator: "=",
+				Value:    "a0eee5dd-3cfe-4ab1-8c79-aee1a8cf4dab",
+			},
+		},
+	}
+	memTem = &CommonAlertTerm{
+		Operator:    "or",
+		Database:    "telegraf",
+		Measurement: "vm_mem",
+		Field:       []string{"used_percent"},
+		Comparator:  ">=",
+		Reduce:      "avg",
+		Threshold:   3,
+		Name:        "lzx-test.mem.avaiable",
+		Filters: []monitorapi.MetricQueryTag{
+			{
+				Key:      "id",
+				Operator: "=",
+				Value:    "a0eee5dd-3cfe-4ab1-8c79-aee1a8cf4dab",
+			},
+		},
+	}
+)
+
 func init() {
 	cmd := NewResourceCmd(modules.CommonAlerts)
 	cmd.Create(new(options.CommonAlertCreateOptions))
@@ -28,4 +154,34 @@ func init() {
 	cmd.Perform("disable", &options.CommonAlertShowOptions{})
 	cmd.BatchDelete(new(options.CommonAlertDeleteOptions))
 	cmd.Perform("config", &options.CommonAlertUpdateOptions{})
+
+	type TestOpt struct {
+		NAME     string
+		RobotIds []string
+		Users    []string
+	}
+	R(&TestOpt{}, "monitor-commonalert-create-mul-test", "create test monitor common alert", func(s *mcclient.ClientSession, opt *TestOpt) error {
+		cpuQ := newCommonAlertQuery(cpuTem)
+		memQ := newCommonAlertQuery(memTem)
+		input := monitorapi.CommonAlertCreateInput{
+			CommonMetricInputQuery: monitorapi.CommonMetricInputQuery{
+				MetricQuery: []*monitorapi.CommonAlertQuery{
+					cpuQ,
+					memQ,
+				},
+			},
+			AlertCreateInput: monitorapi.AlertCreateInput{
+				Name: opt.NAME,
+			},
+			CommonAlertCreateBaseInput: monitorapi.CommonAlertCreateBaseInput{
+				Recipients: opt.Users,
+				RobotIds:   opt.RobotIds,
+				Channel:    []string{"webconsole"},
+				AlertType:  monitorapi.CommonAlertNomalAlertType,
+				Scope:      "system",
+			},
+		}
+		_, err := modules.CommonAlerts.Create(s, jsonutils.Marshal(input))
+		return err
+	})
 }

pkg/apis/monitor/alertrecord.go

@@ -71,12 +71,12 @@ type AlertRecordCreateInput struct {
 
 	AlertId string `json:"alert_id"`
 	// 报警级别
-	Level     string       `json:"level"`
-	State     string       `json:"state"`
-	SendState string       `json:"send_state"`
-	ResType   string       `json:"res_type"`
-	EvalData  []*EvalMatch `json:"eval_data"`
-	AlertRule AlertRecordRule
+	Level     string             `json:"level"`
+	State     string             `json:"state"`
+	SendState string             `json:"send_state"`
+	ResType   string             `json:"res_type"`
+	EvalData  []*EvalMatch       `json:"eval_data"`
+	AlertRule []*AlertRecordRule `json:"alert_rule"`
 }
 
 type AlertRecordRule struct {

pkg/apis/monitor/commalert.go

@@ -103,6 +103,8 @@ type CommonAlertQuery struct {
 	//field yunsuan
 	FieldOpt      string `json:"field_opt"`
 	ConditionType string `json:"condition_type"`
+	// Operator should be chosen from 'and | or'
+	Operator string `json:"operator"`
 }
 
 type CommonAlertListInput struct {
@@ -159,6 +161,7 @@ type CommonAlertDetails struct {
 }
 
 type CommonAlertMetricDetails struct {
+	Operator      string    `json:"operator"`
 	Comparator    string    `json:"comparator"`
 	Threshold     float64   `json:"threshold"`
 	WithinRange   []float64 `json:"within_range"`

pkg/monitor/alerting/conditions/query.go

@@ -309,6 +309,11 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series moni
 	if len(context.Rule.Message) == 0 {
 		context.Rule.Message = msg
 	}
+	op := alertDetails.Operator
+	if op != "" && c.Index > 0 {
+		msg = fmt.Sprintf("%s %s", strings.ToUpper(op), msg)
+	}
+	context.Rule.TriggeredMessages = append(context.Rule.TriggeredMessages, msg)
 	return evalMatch, nil
 }
 

pkg/monitor/alerting/eval_context.go

@@ -199,6 +199,9 @@ func getNewStateInternal(c *EvalContext) monitor.AlertStateType {
 
 func (c *EvalContext) GetNotificationTemplateConfig() monitor.NotificationTemplateConfig {
 	desc := c.Rule.Message
+	if len(c.Rule.TriggeredMessages) > 0 {
+		desc = strings.Join(c.Rule.TriggeredMessages, " ")
+	}
 	if c.Error != nil {
 		if desc != "" {
 			desc += "\n"
@@ -248,10 +251,9 @@ func (c *EvalContext) GetResourceNameOfMathes(matches []monitor.EvalMatch) strin
 	}
 	for i, match := range matches {
 		if name, ok := match.Tags["name"]; ok {
-			names.WriteString(name)
-			names.WriteString(fmt.Sprintf("(%s)", match.ValueStr))
+			names.WriteString(fmt.Sprintf("%s.%s(%s)", name, match.Metric, match.ValueStr))
 			if i < len(matches)-1 {
-				names.WriteString("、")
+				names.WriteString(", ")
 			}
 		}
 	}

pkg/monitor/alerting/notifier.go

@@ -176,12 +176,12 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext,
 		State:     string(evalCtx.Rule.State),
 		SendState: monitor.SEND_STATE_OK,
 		EvalData:  matches,
-		AlertRule: newAlertRecordRule(evalCtx),
+		AlertRule: newAlertRecordRules(evalCtx),
 	}
 	if !shouldNotify {
 		recordCreateInput.SendState = monitor.SEND_STATE_SILENT
 	}
-	recordCreateInput.ResType = recordCreateInput.AlertRule.ResType
+	recordCreateInput.ResType = recordCreateInput.AlertRule[0].ResType
 	if len(recordCreateInput.ResType) == 0 {
 		recordCreateInput.ResType = monitor.METRIC_RES_TYPE_HOST
 	}
@@ -272,10 +272,19 @@ func InitNotifier(config NotificationConfig) (Notifier, error) {
 	return plug.(Notifier), nil
 }
 
-func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule {
+func newAlertRecordRules(ctx *EvalContext) []*monitor.AlertRecordRule {
+	rules := make([]*monitor.AlertRecordRule, 0)
+	for i := range ctx.Rule.RuleDescription {
+		rule := newAlertRecordRule(ctx, i)
+		rules = append(rules, rule)
+	}
+	return rules
+}
+
+func newAlertRecordRule(evalCtx *EvalContext, idx int) *monitor.AlertRecordRule {
 	alertRule := monitor.AlertRecordRule{}
 	if len(evalCtx.Rule.RuleDescription) != 0 {
-		alertRule = evalCtx.Rule.RuleDescription[0].AlertRecordRule
+		alertRule = evalCtx.Rule.RuleDescription[idx].AlertRecordRule
 	}
 	if evalCtx.Rule.Frequency < 60 {
 		alertRule.Period = fmt.Sprintf("%ds", evalCtx.Rule.Frequency)
@@ -291,5 +300,5 @@ func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule {
 	if evalCtx.Rule.SilentPeriod != 0 {
 		alertRule.SilentPeriod = fmt.Sprintf("%dm", evalCtx.Rule.SilentPeriod/60)
 	}
-	return alertRule
+	return &alertRule
 }

pkg/monitor/alerting/rule.go

@@ -45,11 +45,13 @@ func init() {
 
 // Rule is the in-memory version of an alert rule.
 type Rule struct {
-	Id                  string
-	Frequency           int64
-	Title               string
-	Name                string
-	Message             string
+	Id        string
+	Frequency int64
+	Title     string
+	Name      string
+	Message   string
+	// 使用 TriggeredMessages 存储触发的条件，替代 Message
+	TriggeredMessages   []string
 	LastStateChange     time.Time
 	For                 time.Duration
 	NoDataState         monitor.NoDataOption
@@ -115,6 +117,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 	model.Title = ruleDef.GetTitle()
 	model.Name = ruleDef.Name
 	model.Message = ruleDef.Message
+	model.TriggeredMessages = make([]string, 0)
 	model.State = monitor.AlertStateType(ruleDef.State)
 	model.LastStateChange = ruleDef.LastStateChange
 	model.For = time.Duration(ruleDef.For)

pkg/monitor/models/commonalert.go

@@ -269,14 +269,16 @@ func (man *SCommonAlertManager) ValidateCreateData(
 	}
 	data.Name = name
 
-	alertCreateInput := man.toAlertCreatInput(data)
+	alertCreateInput, err := man.toAlertCreatInput(data)
+	if err != nil {
+		return data, errors.Wrap(err, "to alert creation input")
+	}
 	alertCreateInput, err = AlertManager.ValidateCreateData(ctx, userCred, ownerId, query, alertCreateInput)
 	if err != nil {
 		return data, err
 	}
 	data.AlertCreateInput = alertCreateInput
 	return data, nil
-
 }
 
 func (man *SCommonAlertManager) genName(ctx context.Context, ownerId mcclient.IIdentityProvider, name string) (string,
@@ -841,6 +843,7 @@ func getCommonAlertMetricDetailsFromCondition(cond *monitor.AlertCondition,
 	metricDetails.DB = db
 	metricDetails.Groupby = groupby
 	metricDetails.Filters = cond.Query.Model.Tags
+	metricDetails.Operator = cond.Operator
 
 	//fill measurement\field desciption info
 	getMetricDescriptionDetails(metricDetails)
@@ -924,7 +927,7 @@ func getQueryEvalType(evalType string) string {
 	return typ
 }
 
-func (man *SCommonAlertManager) toAlertCreatInput(input monitor.CommonAlertCreateInput) monitor.AlertCreateInput {
+func (man *SCommonAlertManager) toAlertCreatInput(input monitor.CommonAlertCreateInput) (monitor.AlertCreateInput, error) {
 	freq, _ := time.ParseDuration(input.Period)
 	ret := new(monitor.AlertCreateInput)
 	ret.Name = input.Name
@@ -947,12 +950,18 @@ func (man *SCommonAlertManager) toAlertCreatInput(input monitor.CommonAlertCreat
 				Params: []float64{fieldOperatorThreshold(metricquery.FieldOpt, metricquery.Threshold)}},
 			Operator: "and",
 		}
+		if metricquery.Operator != "" {
+			if !sets.NewString("and", "or").Has(metricquery.Operator) {
+				return *ret, httperrors.NewInputParameterError("invalid operator %s", metricquery.Operator)
+			}
+			condition.Operator = metricquery.Operator
+		}
 		if metricquery.FieldOpt != "" {
 			condition.Reducer.Operators = []string{metricquery.FieldOpt}
 		}
 		ret.Settings.Conditions = append(ret.Settings.Conditions, condition)
 	}
-	return *ret
+	return *ret, nil
 }
 
 func fieldOperatorThreshold(opt string, threshold float64) float64 {
@@ -1053,7 +1062,10 @@ func (alert *SCommonAlert) ValidateUpdateData(
 		if err != nil {
 			return data, errors.Wrap(err, "updataInput Unmarshal err")
 		}
-		alertCreateInput := alert.getUpdateAlertInput(*updataInput)
+		alertCreateInput, err := alert.getUpdateAlertInput(*updataInput)
+		if err != nil {
+			return data, errors.Wrap(err, "getUpdateAlertInput")
+		}
 		alertCreateInput, err = AlertManager.ValidateCreateData(ctx, userCred, nil, query, alertCreateInput)
 		if err != nil {
 			return data, err
@@ -1120,14 +1132,13 @@ func (alert *SCommonAlert) UpdateNotification(ctx context.Context, userCred mccl
 	return err
 }
 
-func (alert *SCommonAlert) getUpdateAlertInput(updateInput monitor.CommonAlertUpdateInput) monitor.AlertCreateInput {
+func (alert *SCommonAlert) getUpdateAlertInput(updateInput monitor.CommonAlertUpdateInput) (monitor.AlertCreateInput, error) {
 	input := monitor.CommonAlertCreateInput{
 		CommonMetricInputQuery: updateInput.CommonMetricInputQuery,
 		Period:                 updateInput.Period,
 	}
 	input.AlertDuration = updateInput.AlertDuration
-	alertCreateInput := CommonAlertManager.toAlertCreatInput(input)
-	return alertCreateInput
+	return CommonAlertManager.toAlertCreatInput(input)
 }
 
 func (alert *SCommonAlert) CustomizeDelete(