fix(monitor): 修复报警恢复时包含其他资源的问题 (#21542)

pkg/apis/monitor/alert.go

@@ -198,6 +198,7 @@ type EvalMatch struct {
 	Tags         map[string]string    `json:"tags"`
 	Unit         string               `json:"unit"`
 	AlertDetails jsonutils.JSONObject `json:"alert_details"`
+	IsRecovery   bool                 `json:"is_recovery"`
 }
 
 type AlertTestRunOutput struct {

pkg/apis/monitor/template.go

@@ -19,10 +19,10 @@ type NotificationTemplateCreateInput struct {
 }
 
 type NotificationTemplateConfig struct {
-	Title        string      `json:"title"`
-	Name         string      `json:"name"`
-	ResourceName string      `json:"resource_name"`
-	Matches      []EvalMatch `json:"matches"`
+	Title        string       `json:"title"`
+	Name         string       `json:"name"`
+	ResourceName string       `json:"resource_name"`
+	Matches      []*EvalMatch `json:"matches"`
 	// PrevAlertState AlertStateType `json:"prev_alert_state"`
 	// State AlertStateType `json:"state"`
 	NoDataFound bool   `json:"no_data"`

pkg/apis/monitor/unifiedmonitor_query.go

@@ -65,6 +65,27 @@ var (
 	}
 )
 
+func GetMeasurementTagIdKeyByResType(resType string) string {
+	return MEASUREMENT_TAG_ID[resType]
+}
+
+func GetMeasurementTagIdKeyByResTypeWithDefault(resType string) string {
+	tagId := GetMeasurementTagIdKeyByResType(resType)
+	if len(tagId) == 0 {
+		tagId = "host_id"
+	}
+	return tagId
+}
+
+func GetMeasurementResourceId(tags map[string]string, resType string) string {
+	return tags[GetMeasurementTagIdKeyByResType(resType)]
+}
+
+func GetResourceIdFromTagWithDefault(tags map[string]string, resType string) string {
+	tagId := GetMeasurementTagIdKeyByResTypeWithDefault(resType)
+	return tags[tagId]
+}
+
 type MetricFunc struct {
 	FieldOptType  []string            `json:"field_opt_type"`
 	FieldOptValue map[string][]string `json:"field_opt_value"`

pkg/monitor/alerting/conditions/helper.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package conditions
+
+import (
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apis/monitor"
+	"yunion.io/x/onecloud/pkg/monitor/models"
+)
+
+func OkEvalMatchSetIsRecovery(
+	alert *models.SCommonAlert,
+	resId string,
+	em *monitor.EvalMatch,
+) error {
+	isAlerting, err := alert.IsResourceMetricAlerting(resId, em.Metric)
+	if err != nil {
+		return errors.Wrap(err, "check if previous alert is alerting")
+	}
+	if isAlerting {
+		em.IsRecovery = true
+	}
+	return nil
+}

pkg/monitor/alerting/conditions/nodataquery.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apis/monitor"
@@ -39,7 +40,7 @@ const (
 )
 
 func init() {
-	alerting.RegisterCondition("nodata_query", func(model *monitor.AlertCondition, index int) (alerting.Condition,
+	alerting.RegisterCondition(monitor.METRIC_QUERY_TYPE_NO_DATA, func(model *monitor.AlertCondition, index int) (alerting.Condition,
 		error) {
 		return newNoDataQueryCondition(model, index)
 	})
@@ -60,12 +61,13 @@ func (c *NoDataQueryCondition) Eval(context *alerting.EvalContext) (*alerting.Co
 	var matches []*monitor.EvalMatch
 	var alertOkmatches []*monitor.EvalMatch
 	normalHostIds := make(map[string]*monitor.EvalMatch, 0)
+	alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error")
+	}
 serLoop:
 	for _, series := range seriesList {
-		tagId := monitor.MEASUREMENT_TAG_ID[context.Rule.RuleDescription[0].ResType]
-		if len(tagId) == 0 {
-			tagId = "host_id"
-		}
+		tagId := monitor.GetMeasurementTagIdKeyByResTypeWithDefault(context.Rule.RuleDescription[0].ResType)
 		for key, val := range series.Tags {
 			if key == tagId {
 				if len(context.Rule.RuleDescription) == 0 {
@@ -79,7 +81,7 @@ serLoop:
 				}
 
 				reducedValue, valStrArr := c.Reducer.Reduce(series)
-				match, err := c.NewEvalMatch(context, *series, nil, reducedValue, valStrArr, false)
+				match, err := c.NewEvalMatch(alert, context, *series, nil, reducedValue, valStrArr, false)
 				if err != nil {
 					return nil, errors.Wrap(err, "NoDataQueryCondition NewEvalMatch error")
 				}
@@ -94,15 +96,18 @@ serLoop:
 	}
 	for _, host := range allResources {
 		id, _ := host.GetString("id")
-		evalMatch, err := c.NewNoDataEvalMatch(context, host)
+		evalMatch, err := c.NewNoDataEvalMatch(context, alert, host)
 		if err != nil {
-			return nil, errors.Wrap(err, "NewNoDataEvalMatch error")
+			return nil, errors.Wrapf(err, "NewNoDataEvalMatch of host: %s", host.String())
 		}
 		if normalMatch, ok := normalHostIds[id]; !ok {
 			c.createEvalMatchTagFromHostJson(context, evalMatch, host)
 			matches = append(matches, evalMatch)
 		} else {
 			c.createEvalMatchTagFromHostJson(context, normalMatch, host)
+			if err := OkEvalMatchSetIsRecovery(alert, id, normalMatch); err != nil {
+				log.Warningf("[NoDataQuery] set eval match %s to recovered: %v", jsonutils.Marshal(normalMatch), err)
+			}
 			alertOkmatches = append(alertOkmatches, normalMatch)
 		}
 	}
@@ -115,7 +120,7 @@ serLoop:
 	}, nil
 }
 
-func (c *NoDataQueryCondition) convertTagsQuery(evalContext *alerting.EvalContext,
+/*func (c *NoDataQueryCondition) convertTagsQuery(evalContext *alerting.EvalContext,
 	query *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	alertDetails, err := c.GetCommonAlertDetails(evalContext)
 	if err != nil {
@@ -136,14 +141,10 @@ func (c *NoDataQueryCondition) convertTagsQuery(evalContext *alerting.EvalContex
 		filterCount++
 	}
 	return query, nil
-}
+}*/
 
-func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext, host jsonutils.JSONObject) (*monitor.EvalMatch, error) {
+func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext, alert *models.SCommonAlert, host jsonutils.JSONObject) (*monitor.EvalMatch, error) {
 	evalMatch := new(monitor.EvalMatch)
-	alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id)
-	if err != nil {
-		return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error")
-	}
 	settings, _ := alert.GetSettings()
 	alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(c.Index, &settings.Conditions[c.Index])
 	evalMatch.Metric = fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field)
@@ -164,8 +165,11 @@ func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext,
 	return evalMatch, nil
 }
 
-func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalContext *alerting.EvalContext, evalMatch *monitor.EvalMatch,
-	host jsonutils.JSONObject) {
+func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(
+	evalContext *alerting.EvalContext,
+	evalMatch *monitor.EvalMatch,
+	host jsonutils.JSONObject,
+) {
 	evalMatch.Tags = make(map[string]string, 0)
 
 	ip, _ := host.GetString(HOST_TAG_IP)
@@ -186,7 +190,7 @@ func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalContext *alert
 	case monitor.METRIC_RES_TYPE_OSS:
 	default:
 		evalMatch.Tags["host"] = name
-		evalMatch.Tags[monitor.MEASUREMENT_TAG_ID[monitor.METRIC_RES_TYPE_HOST]] = id
+		evalMatch.Tags[monitor.GetMeasurementTagIdKeyByResType(monitor.METRIC_RES_TYPE_HOST)] = id
 		evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_RES_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_RES_TYPE
 		evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_HOST_TYPE_HOST
 	}

pkg/monitor/alerting/conditions/query.go

@@ -159,10 +159,11 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio
 	var matches []*monitor.EvalMatch
 	var alertOkmatches []*monitor.EvalMatch
 
-	//allResources, err := c.GetQueryResources()
-	//if err != nil {
-	//	return nil, errors.Wrap(err, "GetQueryResources err")
-	//}
+	alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error")
+	}
+
 	for _, series := range seriesList {
 		if len(c.ResType) != 0 {
 			isLatestOfSerie, resource := c.serieIsLatestResource(nil, series)
@@ -193,17 +194,21 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio
 			meta = &metas[0]
 		}
 		if evalMatch {
-			match, err := c.NewEvalMatch(context, *series, meta, reducedValue, valStrArr, evalMatch)
+			match, err := c.NewEvalMatch(alert, context, *series, meta, reducedValue, valStrArr, evalMatch)
 			if err != nil {
 				return nil, errors.Wrap(err, "NewEvalMatch error")
 			}
 			matches = append(matches, match)
 		}
 		if reducedValue != nil && !evalMatch {
-			match, err := c.NewEvalMatch(context, *series, meta, reducedValue, valStrArr, evalMatch)
+			match, err := c.NewEvalMatch(alert, context, *series, meta, reducedValue, valStrArr, evalMatch)
 			if err != nil {
 				return nil, errors.Wrap(err, "NewEvalMatch error")
 			}
+			resId := monitor.GetResourceIdFromTagWithDefault(match.Tags, c.ResType)
+			if err := OkEvalMatchSetIsRecovery(alert, resId, match); err != nil {
+				log.Warningf("[Query] set eval match %s to recovered: %v", jsonutils.Marshal(match), err)
+			}
 			alertOkmatches = append(alertOkmatches, match)
 		}
 	}
@@ -241,12 +246,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio
 }
 
 func (c *QueryCondition) serieIsLatestResource(resources map[string]jsonutils.JSONObject, series *monitor.TimeSeries) (bool, jsonutils.JSONObject) {
-	tagId := monitor.MEASUREMENT_TAG_ID[c.ResType]
-	if len(tagId) == 0 {
-		tagId = "host_id"
-	}
-	resId := series.Tags[tagId]
-	log.Debugf("serieIsLatestResource use tagId %q, found resId %q", tagId, resId)
+	resId := monitor.GetResourceIdFromTagWithDefault(series.Tags, c.ResType)
 	if len(resources) != 0 {
 		resource, ok := resources[resId]
 		if !ok {
@@ -279,9 +279,14 @@ func (c *QueryCondition) FillSerieByResourceField(resource jsonutils.JSONObject,
 	}
 }
 
-func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series monitor.TimeSeries, meta *monitor.QueryResultMeta, value *float64, valStrArr []string, isMatch bool) (*monitor.EvalMatch, error) {
+func (c *QueryCondition) NewEvalMatch(
+	alert *models.SCommonAlert,
+	context *alerting.EvalContext,
+	series monitor.TimeSeries,
+	meta *monitor.QueryResultMeta,
+	value *float64, valStrArr []string, isMatch bool) (*monitor.EvalMatch, error) {
 	evalMatch := new(monitor.EvalMatch)
-	alertDetails, err := c.GetCommonAlertDetails(context)
+	alertDetails, err := c.GetCommonAlertDetails(alert)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error")
 	}
@@ -353,11 +358,7 @@ func (m *meterFetchImp) FetchCustomizeEvalMatch(context *alerting.EvalContext, e
 	return nil
 }
 
-func (c *QueryCondition) GetCommonAlertDetails(context *alerting.EvalContext) (*monitor.CommonAlertMetricDetails, error) {
-	alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id)
-	if err != nil {
-		return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error")
-	}
+func (c *QueryCondition) GetCommonAlertDetails(alert *models.SCommonAlert) (*monitor.CommonAlertMetricDetails, error) {
 	settings, _ := alert.GetSettings()
 	alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(c.Index, &settings.Conditions[c.Index])
 	return alertDetails, nil
@@ -530,21 +531,20 @@ func (c *QueryCondition) checkGroupByField() {
 	}
 	for i, group := range c.Query.Model.GroupBy {
 		if group.Params[0] == "*" {
-			c.Query.Model.GroupBy[i].Params = []string{monitor.MEASUREMENT_TAG_ID[metricMeasurement.ResType]}
+			c.Query.Model.GroupBy[i].Params = []string{monitor.GetMeasurementTagIdKeyByResType(metricMeasurement.ResType)}
 		}
 	}
 }
 
 func (c *QueryCondition) setResType() {
+	metricMeasurement, _ := models.MetricMeasurementManager.GetCache().Get(c.Query.Model.Measurement)
+	if metricMeasurement != nil {
+		c.ResType = metricMeasurement.ResType
+	}
 	var resType = monitor.METRIC_RES_TYPE_HOST
 	if len(c.Query.Model.GroupBy) == 0 {
 		return
 	}
-	metricMeasurement, _ := models.MetricMeasurementManager.GetCache().Get(c.Query.Model.Measurement)
-	if metricMeasurement != nil {
-		resType = metricMeasurement.ResType
-		c.ResType = resType
-	}
 	// NOTE: shouldn't set ResType when tenant_id and domain_id within GroupBy
 	/* if len(resType) != 0 && c.Query.Model.GroupBy[0].Params[0] != monitor.
 		MEASUREMENT_TAG_ID[resType] {
@@ -576,7 +576,7 @@ func (c *QueryCondition) GetQueryResources(s *mcclient.ClientSession, scope stri
 
 func (c *QueryCondition) getOnecloudResources(s *mcclient.ClientSession, scope string, showDetails bool) ([]jsonutils.JSONObject, error) {
 	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewStringArray([]string{"running", "ready"}), "status")
+	queryStatus := []string{"running", "ready"}
 	// query.Add(jsonutils.NewString("true"), "admin")
 	//if len(c.Query.Model.Tags) != 0 {
 	//	query, err = c.convertTagsQuery(evalContext, query)
@@ -592,6 +592,7 @@ func (c *QueryCondition) getOnecloudResources(s *mcclient.ClientSession, scope s
 	switch c.ResType {
 	case monitor.METRIC_RES_TYPE_HOST:
 		query.Set("host_type", jsonutils.NewString(hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR))
+		queryStatus = append(queryStatus, "unknown")
 		query.Set("enabled", jsonutils.NewInt(1))
 		manager = &mc_mds.Hosts
 	case monitor.METRIC_RES_TYPE_GUEST:
@@ -622,6 +623,7 @@ func (c *QueryCondition) getOnecloudResources(s *mcclient.ClientSession, scope s
 		manager = &mc_mds.Hosts
 	}
 
+	query.Add(jsonutils.NewStringArray(queryStatus), "status")
 	allResources, err = ListAllResources(s, manager, query, scope, showDetails)
 	if err != nil {
 		return nil, errors.Wrapf(err, "ListAllResources for %s with query %s, scope: %s, showDetails: %v", manager.GetKeyword(), query, scope, showDetails)

pkg/monitor/alerting/eval_context.go

@@ -54,12 +54,13 @@ type EvalContext struct {
 // NewEvalContext is the EvalContext constructor.
 func NewEvalContext(alertCtx context.Context, userCred mcclient.TokenCredential, rule *Rule) *EvalContext {
 	return &EvalContext{
-		Ctx:            alertCtx,
-		UserCred:       userCred,
-		StartTime:      time.Now(),
-		Rule:           rule,
-		EvalMatches:    make([]*monitor.EvalMatch, 0),
-		PrevAlertState: rule.State,
+		Ctx:                alertCtx,
+		UserCred:           userCred,
+		StartTime:          time.Now(),
+		Rule:               rule,
+		EvalMatches:        make([]*monitor.EvalMatch, 0),
+		AlertOkEvalMatches: make([]*monitor.EvalMatch, 0),
+		PrevAlertState:     rule.State,
 	}
 }
 
@@ -192,7 +193,7 @@ func getNewStateInternal(c *EvalContext) monitor.AlertStateType {
 	return monitor.AlertStateOK
 }
 
-func (c *EvalContext) GetNotificationTemplateConfig() monitor.NotificationTemplateConfig {
+func (c *EvalContext) GetNotificationTemplateConfig(matches []*monitor.EvalMatch) monitor.NotificationTemplateConfig {
 	desc := c.Rule.Message
 	if len(c.Rule.TriggeredMessages) > 0 {
 		desc = strings.Join(c.Rule.TriggeredMessages, " ")
@@ -207,14 +208,15 @@ func (c *EvalContext) GetNotificationTemplateConfig() monitor.NotificationTempla
 	return monitor.NotificationTemplateConfig{
 		Title:        c.GetNotificationTitle(),
 		Name:         c.Rule.Name,
-		ResourceName: c.GetResourceNameOfMathes(nil),
-		Matches:      c.GetEvalMatches(),
-		StartTime:    c.StartTime.In(tz).Format("2006-01-02 15:04:05"),
-		EndTime:      c.EndTime.In(tz).Format("2006-01-02 15:04:05"),
-		Description:  desc,
-		Level:        c.Rule.Level,
-		NoDataFound:  c.NoDataFound,
-		WebUrl:       c.GetCallbackURLPrefix(),
+		ResourceName: c.GetResourceNameOfMatches(matches),
+		Matches:      matches,
+		//Matches:      c.GetEvalMatches(),
+		StartTime:   c.StartTime.In(tz).Format("2006-01-02 15:04:05"),
+		EndTime:     c.EndTime.In(tz).Format("2006-01-02 15:04:05"),
+		Description: desc,
+		Level:       c.Rule.Level,
+		NoDataFound: c.NoDataFound,
+		WebUrl:      c.GetCallbackURLPrefix(),
 	}
 }
 
@@ -239,11 +241,8 @@ func (c *EvalContext) GetEvalMatches() []monitor.EvalMatch {
 	return ret
 }
 
-func (c *EvalContext) GetResourceNameOfMathes(matches []monitor.EvalMatch) string {
+func (c *EvalContext) GetResourceNameOfMatches(matches []*monitor.EvalMatch) string {
 	names := strings.Builder{}
-	if matches == nil {
-		matches = c.GetEvalMatches()
-	}
 	for i, match := range matches {
 		if name, ok := match.Tags["name"]; ok {
 			names.WriteString(fmt.Sprintf("%s.%s(%s)", name, match.Metric, match.ValueStr))
@@ -254,3 +253,18 @@ func (c *EvalContext) GetResourceNameOfMathes(matches []monitor.EvalMatch) strin
 	}
 	return names.String()
 }
+
+func (c *EvalContext) GetRecoveredMatches() []*monitor.EvalMatch {
+	ret := make([]*monitor.EvalMatch, 0)
+	for i := range c.AlertOkEvalMatches {
+		m := c.AlertOkEvalMatches[i]
+		if m.IsRecovery {
+			ret = append(ret, m)
+		}
+	}
+	return ret
+}
+
+func (c *EvalContext) HasRecoveredMatches() bool {
+	return len(c.GetRecoveredMatches()) != 0
+}

pkg/monitor/alerting/notifier.go

@@ -224,7 +224,7 @@ func (n *notificationService) dealNeedShieldEvalMatches(evalCtx *EvalContext, ma
 	}
 filterMatch:
 	for i := range match {
-		input.ResId = match[i].Tags[monitor.MEASUREMENT_TAG_ID[input.ResType]]
+		input.ResId = monitor.GetMeasurementResourceId(match[i].Tags, input.ResType)
 		alertRecordShields, err := models.AlertRecordShieldManager.GetRecordShields(input)
 		if err != nil {
 			log.Errorf("GetRecordShields byAlertId:%s,err:%v", input.AlertId, err)

pkg/monitor/alerting/notifiers/base.go

@@ -56,6 +56,10 @@ func (n *NotifierBase) ShouldNotify(_ context.Context, evalCtx *alerting.EvalCon
 	prevState := evalCtx.PrevAlertState
 	newState := evalCtx.Rule.State
 
+	if evalCtx.HasRecoveredMatches() {
+		return true
+	}
+
 	// Do not notify if alert state is no_data
 	if newState == monitor.AlertStateNoData {
 		return false

pkg/monitor/alerting/notifiers/dingding.go

@@ -86,11 +86,26 @@ func newDingdingNotifier(config alerting.NotificationConfig) (alerting.Notifier,
 	}, nil
 }
 
-func (dd *DingDingNotifier) Notify(ctx *alerting.EvalContext, _ jsonutils.JSONObject) error {
+func (dd *DingDingNotifier) Notify(evalCtx *alerting.EvalContext, d jsonutils.JSONObject) error {
 	log.Infof("Sending alert notification to dingding")
+	errs := []error{}
+	if len(evalCtx.EvalMatches) > 0 {
+		if err := dd.notify(evalCtx, evalCtx.EvalMatches, false, d); err != nil {
+			errs = append(errs, errors.Wrap(err, "notify alerting matches"))
+		}
+	}
+	if evalCtx.HasRecoveredMatches() {
+		if err := dd.notify(evalCtx, evalCtx.GetRecoveredMatches(), true, d); err != nil {
+			errs = append(errs, errors.Wrap(err, "notify recovered matches"))
+		}
+	}
+	return errors.NewAggregate(errs)
+}
+
+func (dd *DingDingNotifier) notify(ctx *alerting.EvalContext, matches []*monitor.EvalMatch, isRecoverd bool, _ jsonutils.JSONObject) error {
 	// msgUrl, err := ctx.GetRuleURL()
 
-	body, err := dd.genBody(ctx)
+	body, err := dd.genBody(ctx, matches, isRecoverd)
 	if err != nil {
 		return err
 	}
@@ -101,7 +116,7 @@ func (dd *DingDingNotifier) Notify(ctx *alerting.EvalContext, _ jsonutils.JSONOb
 	return SendWebRequestSync(ctx.Ctx, input)
 }
 
-func (dd *DingDingNotifier) genBody(ctx *alerting.EvalContext) ([]byte, error) {
+func (dd *DingDingNotifier) genBody(ctx *alerting.EvalContext, matches []*monitor.EvalMatch, isRecoverd bool) ([]byte, error) {
 	q := url.Values{
 		"pc_slide": {"false"},
 		// "url": {messageURL},
@@ -113,7 +128,7 @@ func (dd *DingDingNotifier) genBody(ctx *alerting.EvalContext) ([]byte, error) {
 
 	log.Infof("messageUrl: " + messageURL)
 
-	config := GetNotifyTemplateConfig(ctx)
+	config := GetNotifyTemplateConfig(ctx, isRecoverd, matches)
 	contentConfig := templates.NewTemplateConfig(config)
 	content, err := contentConfig.GenerateMarkdown()
 	if err != nil {

pkg/monitor/alerting/notifiers/feishu.go

@@ -156,15 +156,15 @@ func (fs *FeishuNotifier) getMetricsMod(config monitor.NotificationTemplateConfi
 	inElems := make([]*feishu.CardElement, 0)
 	for idx, m := range config.Matches {
 		hrE := feishu.NewCardElementHR()
-		mE := fs.getMetricElem(idx+1, m)
-		mTE := fs.getMetricTagElem(m)
+		mE := fs.getMetricElem(idx+1, *m)
+		mTE := fs.getMetricTagElem(*m)
 		inElems = append(inElems, hrE, mE, mTE)
 	}
 	return inElems
 }
 
 func (fs *FeishuNotifier) genCard(ctx *alerting.EvalContext, chatId string) (*feishu.MsgReq, error) {
-	config := GetNotifyTemplateConfig(ctx)
+	config := GetNotifyTemplateConfig(ctx, false, ctx.EvalMatches)
 	commonElem := fs.getCommonInfoMod(config)
 
 	msElems := fs.getMetricsMod(config)
@@ -193,7 +193,7 @@ func (fs *FeishuNotifier) genCard(ctx *alerting.EvalContext, chatId string) (*fe
 }
 
 func (fs *FeishuNotifier) genMsg(ctx *alerting.EvalContext, chatId string) (*feishu.MsgReq, error) {
-	config := GetNotifyTemplateConfig(ctx)
+	config := GetNotifyTemplateConfig(ctx, false, ctx.EvalMatches)
 	// 富文本: https://open.feishu.cn/document/ukTMukTMukTM/uMDMxEjLzATMx4yMwETM
 	return &feishu.MsgReq{
 		ChatId:  chatId,

pkg/monitor/alerting/notifiers/onecloud.go

@@ -115,63 +115,60 @@ func getAdminSession() *mcclient.ClientSession {
 	return auth.GetAdminSession(context.Background(), options.Options.Region)
 }
 
-func GetNotifyTemplateConfig(ctx *alerting.EvalContext) monitor.NotificationTemplateConfig {
-	priority := notify.NotifyPriorityNormal
-	level := "普通"
-	switch ctx.Rule.Level {
-	case "", "normal":
-		priority = notify.NotifyPriorityNormal
-	case "important":
-		priority = notify.NotifyPriorityImportant
-		level = "重要"
-	case "fatal", "critical":
-		priority = notify.NotifyPriorityCritical
-		level = "致命"
-	}
-	topic := fmt.Sprintf("[%s]", level)
-
-	isRecovery := false
-	if ctx.Rule.State == monitor.AlertStateOK {
-		isRecovery = true
-		topic = fmt.Sprintf("%s %s 告警已恢复", topic, ctx.GetRuleTitle())
-	} else if ctx.NoDataFound {
-		topic = fmt.Sprintf("%s %s 暂无数据", topic, ctx.GetRuleTitle())
-	} else {
-		topic = fmt.Sprintf("%s %s 发生告警", topic, ctx.GetRuleTitle())
-	}
-	config := ctx.GetNotificationTemplateConfig()
-	config.Title = topic
-	config.Level = level
-	config.Priority = string(priority)
-	config.IsRecovery = isRecovery
-	return config
+func GetNotifyTemplateConfig(ctx *alerting.EvalContext, isRecoverd bool, matches []*monitor.EvalMatch) monitor.NotificationTemplateConfig {
+	return getNotifyTemplateConfigOfLang(ctx, matches, isRecoverd, language.Chinese)
 }
 
-func GetNotifyTemplateConfigOfEN(ctx *alerting.EvalContext) monitor.NotificationTemplateConfig {
+func getNotifyTemplateConfigOfLang(ctx *alerting.EvalContext,
+	matches []*monitor.EvalMatch, isRecovered bool, lang language.Tag) monitor.NotificationTemplateConfig {
 	priority := notify.NotifyPriorityNormal
-	level := "Normal"
+	levelNormal := "普通"
+	levelImportant := "重要"
+	levelCritial := "致命"
+	msgRecovered := "告警已恢复"
+	msgNoData := "暂无数据"
+	msgAlerting := "发生告警"
+
+	transMap := map[string]string{
+		levelNormal:    "Normal",
+		levelImportant: "Important",
+		levelCritial:   "Critical",
+		msgRecovered:   "Alarm recovered",
+		msgNoData:      "No data available",
+		msgAlerting:    "Alerting",
+	}
+	trans := func(input string) string {
+		if lang == language.English {
+			return transMap[input]
+		}
+		return input
+	}
+
+	level := levelNormal
 	switch ctx.Rule.Level {
 	case "", "normal":
 		priority = notify.NotifyPriorityNormal
 	case "important":
 		priority = notify.NotifyPriorityImportant
-		level = "Important"
+		level = levelNormal
 	case "fatal", "critical":
 		priority = notify.NotifyPriorityCritical
-		level = "Critical"
+		level = levelNormal
 	}
-	topic := fmt.Sprintf("[%s]", level)
+	topic := fmt.Sprintf("[%s]", trans(level))
 
 	isRecovery := false
-	if ctx.Rule.State == monitor.AlertStateOK {
+	var hintMsg string
+	if ctx.Rule.State == monitor.AlertStateOK || isRecovered {
 		isRecovery = true
-		topic = fmt.Sprintf("%s %s Alarm recovered", topic, ctx.GetRuleTitle())
+		hintMsg = msgRecovered
 	} else if ctx.NoDataFound {
-		topic = fmt.Sprintf("%s %s No data available", topic, ctx.GetRuleTitle())
+		hintMsg = msgNoData
 	} else {
-		topic = fmt.Sprintf("%s %s Alarm", topic, ctx.GetRuleTitle())
+		hintMsg = msgAlerting
 	}
-	config := ctx.GetNotificationTemplateConfig()
+	topic = fmt.Sprintf("%s %s %s", topic, ctx.GetRuleTitle(), trans(hintMsg))
+	config := ctx.GetNotificationTemplateConfig(matches)
 	config.Title = topic
 	config.Level = level
 	config.Priority = string(priority)
@@ -249,14 +246,28 @@ func getLangBystr(str string) language.Tag {
 }
 
 func (oc *OneCloudNotifier) notifyByContextLang(ctx context.Context, evalCtx *alerting.EvalContext, uids []string) error {
-	var config monitor.NotificationTemplateConfig
-	lang := appctx.Lang(ctx)
-	switch lang {
-	case language.English:
-		config = GetNotifyTemplateConfigOfEN(evalCtx)
-	default:
-		config = GetNotifyTemplateConfig(evalCtx)
+	errs := []error{}
+	if len(evalCtx.EvalMatches) > 0 {
+		if err := oc.notifyMatchesByContextLang(ctx, evalCtx, evalCtx.EvalMatches, uids, false); err != nil {
+			errs = append(errs, errors.Wrapf(err, "notify alerting matches"))
+		}
 	}
+	if evalCtx.HasRecoveredMatches() {
+		if err := oc.notifyMatchesByContextLang(ctx, evalCtx, evalCtx.GetRecoveredMatches(), uids, true); err != nil {
+			errs = append(errs, errors.Wrapf(err, "notify recovered matches"))
+		}
+	}
+	return errors.NewAggregate(errs)
+}
+
+func (oc *OneCloudNotifier) notifyMatchesByContextLang(
+	ctx context.Context,
+	evalCtx *alerting.EvalContext,
+	matches []*monitor.EvalMatch,
+	uids []string,
+	isRecoverd bool) error {
+	lang := appctx.Lang(ctx)
+	config := getNotifyTemplateConfigOfLang(evalCtx, matches, isRecoverd, lang)
 	oc.filterMatchTagsForConfig(&config, ctx)
 
 	contentConfig := oc.buildContent(config)
@@ -516,7 +527,7 @@ func SendNotifyInfo(base *sendnotifyBase, imp Isendnotify) error {
 			split = len(tmpMatches)
 		}
 		base.config.Matches = tmpMatches[i:split]
-		base.config.ResourceName = base.evalCtx.GetResourceNameOfMathes(base.config.Matches)
+		base.config.ResourceName = base.evalCtx.GetResourceNameOfMatches(base.config.Matches)
 		err := imp.execNotifyFunc()
 		if err != nil {
 			return err

pkg/monitor/models/commonalert.go

@@ -1533,6 +1533,21 @@ func (alert *SCommonAlert) GetAlertRule(settings *monitor.AlertSetting, index in
 	return rule
 }
 
+func (alert *SCommonAlert) GetResourceAlert(resourceId string, metric string) (*SMonitorResourceAlert, error) {
+	return MonitorResourceAlertManager.GetResourceAlert(alert.GetId(), resourceId, metric)
+}
+
+func (alert *SCommonAlert) IsResourceMetricAlerting(resourceId string, metric string) (bool, error) {
+	ra, err := alert.GetResourceAlert(resourceId, metric)
+	if err != nil {
+		return false, errors.Wrapf(err, "GetResourceAlert")
+	}
+	if ra.AlertState == string(monitor.AlertStateAlerting) {
+		return true, nil
+	}
+	return false, nil
+}
+
 var fileSize = []string{"bps", "Bps", "byte"}
 
 func RationalizeValueFromUnit(value float64, unit string, opt string) string {

pkg/monitor/models/monitor_resource.go

@@ -409,10 +409,6 @@ type UpdateMonitorResourceAlertInput struct {
 }
 
 func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context, userCred mcclient.TokenCredential, input *UpdateMonitorResourceAlertInput) error {
-	//if !utils.IsInStringArray(alertRecord.ResType, []string{monitor.METRIC_RES_TYPE_HOST,
-	//	monitor.METRIC_RES_TYPE_GUEST, monitor.METRIC_RES_TYPE_AGENT}) {
-	//	return nil
-	//}
 	resType := input.ResType
 	if resType == monitor.METRIC_RES_TYPE_AGENT {
 		resType = monitor.METRIC_RES_TYPE_GUEST
@@ -421,7 +417,7 @@ func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx con
 	errs := make([]error, 0)
 	matchResourceIds := make([]string, 0)
 	for _, match := range matches {
-		resId := match.Tags[monitor.MEASUREMENT_TAG_ID[input.ResType]]
+		resId := monitor.GetMeasurementResourceId(match.Tags, input.ResType)
 		if len(resId) == 0 {
 			continue
 		}

pkg/monitor/models/monitor_resource_alert.go

@@ -339,3 +339,12 @@ func (manager *SMonitorResourceAlertManager) ListItemExportKeys(ctx context.Cont
 func (m *SMonitorResourceAlertManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man db.FilterByOwnerProvider, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery {
 	return q
 }
+
+func (m *SMonitorResourceAlertManager) GetResourceAlert(alertId string, resourceId string, metric string) (*SMonitorResourceAlert, error) {
+	q := m.Query().Equals("alert_id", alertId).Equals("monitor_resource_id", resourceId).Equals("metric", metric)
+	obj := new(SMonitorResourceAlert)
+	if err := q.First(obj); err != nil {
+		return nil, errors.Wrapf(err, "query resource alert by alert_id: %q, resource_id: %q, metric: %q", alertId, resourceId, metric)
+	}
+	return obj, nil
+}

pkg/monitor/models/unifiedmonitor.go

@@ -493,7 +493,7 @@ func checkQueryGroupBy(query *monitor.AlertQuery, inputQuery *monitor.MetricQuer
 	}
 	tagId := ""
 	if metricMeasurement != nil {
-		tagId = monitor.MEASUREMENT_TAG_ID[metricMeasurement.ResType]
+		tagId = monitor.GetMeasurementTagIdKeyByResType(metricMeasurement.ResType)
 	}
 	if len(tagId) == 0 || (len(inputQuery.Slimit) != 0 && len(inputQuery.Soffset) != 0) {
 		tagId = "*"

pkg/monitor/subscriptionmodel/subcription.go

@@ -419,7 +419,7 @@ func (n *SSubscriptionManager) doNotify(nIds []string, evalCtx *alerting.EvalCon
 }
 
 func (self *SSubscriptionManager) notifyBySysConfig(evalContext alerting.EvalContext) error {
-	config := notifiers.GetNotifyTemplateConfig(&evalContext)
+	config := notifiers.GetNotifyTemplateConfig(&evalContext, false, evalContext.EvalMatches)
 	contentConfig := templates.NewTemplateConfig(config)
 	content, err := contentConfig.GenerateMarkdown()
 	if err != nil {

pkg/monitor/validators/validators.go

@@ -47,7 +47,7 @@ var (
 	CommonAlertReducerFieldOpts = []string{"/"}
 	CommonAlertNotifyTypes      = []string{"email", "mobile", "dingtalk", "webconsole", "feishu"}
 
-	ConditionTypes = []string{"query", "nodata_query"}
+	ConditionTypes = []string{"query", monitor.METRIC_QUERY_TYPE_NO_DATA}
 )
 
 func ValidateAlertCreateInput(input monitor.AlertCreateInput) error {