feat(monitor): multiple alerts of resource (#20924)

pkg/apis/monitor/monitor_resource_alert.go

@@ -28,18 +28,23 @@ type MonitorResourceJointListInput struct {
 	MonitorResourceId string  `json:"monitor_resource_id"`
 	AlertId           string  `json:"alert_id"`
 	JointId           []int64 `json:"joint_id"`
-	Alerting          bool    `json:"alertinng"`
+	Alerting          bool    `json:"alerting"`
+	AlertState        string  `json:"alert_state"`
 	SendState         string  `json:"send_state"`
 	ResType           string  `json:"res_type"`
+	Metric            string  `json:"metric"`
 	ResName           string  `json:"res_name"`
 	AlertName         string  `json:"alert_name"`
 	Level             string  `json:"level"`
+	// 查询所有状态
+	AllState bool `json:"all_state"`
 }
 
 type MonitorResourceJointCreateInput struct {
 	apis.Meta
 	MonitorResourceId string `json:"monitor_resource_id"`
 	AlertId           string `json:"alert_id"`
+	Metric            string `json:"metric"`
 
 	AlertRecordId string    `width:"36" charset:"ascii" list:"user"  update:"user"`
 	AlertState    string    `width:"18" charset:"ascii" list:"user"  update:"user"`

pkg/mcclient/modules/monitor/mod_monitor_resource.go

@@ -51,7 +51,7 @@ func NewMonitorResourceManager() *SMonitorResourceManager {
 
 func newAlertResourceAlertManager() *SMonitorResourceAlertManager {
 	man := modules.NewJointMonitorV2Manager("monitorresourcealert", "monitorresourcealerts",
-		[]string{"monitor_resource_id", "alert_id", "res_name", "res_type", "alert_name", "alert_state", "send_state", "level",
+		[]string{"monitor_resource_id", "alert_id", "res_name", "res_type", "metric", "alert_name", "alert_state", "send_state", "level",
 			"trigger_time", "data"},
 		[]string{},
 		MonitorResourceManager, CommonAlerts)

pkg/mcclient/options/monitor/monitor_resource_alert.go

@@ -26,6 +26,7 @@ type MonitorResourceAlertListOptions struct {
 	AlertId           string `help:"ID  of alert" json:"alert_id"`
 	Alerting          bool   `help:"search alerting resource" json:"alerting"`
 	SendState         string `json:"send_state"`
+	AllState          bool   `help:"Show all state" json:"all_state"`
 }
 
 func (o *MonitorResourceAlertListOptions) GetMasterOpt() string {

pkg/monitor/alerting/conditions/query.go

@@ -296,7 +296,7 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series moni
 	evalMatch.Unit = alertDetails.FieldDescription.Unit
 	evalMatch.Tags = c.filterTags(series.Tags, *alertDetails)
 	evalMatch.Value = value
-	evalMatch.ValueStr = alerting.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit,
+	evalMatch.ValueStr = models.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit,
 		alertDetails.FieldOpt)
 	if alertDetails.GetPointStr {
 		evalMatch.ValueStr = c.jointPointStr(series, evalMatch.ValueStr, valStrArr)
@@ -305,7 +305,7 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series moni
 	//c.newRuleDescription(context, alertDetails)
 	//evalMatch.Condition = c.GenerateFormatCond(meta, queryKeyInfo).String()
 	msg := fmt.Sprintf("%s.%s %s %s", alertDetails.Measurement, alertDetails.Field,
-		alertDetails.Comparator, alerting.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, ""))
+		alertDetails.Comparator, models.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, ""))
 	if len(context.Rule.Message) == 0 {
 		context.Rule.Message = msg
 	}

pkg/monitor/alerting/eval_context.go

@@ -43,7 +43,6 @@ type EvalContext struct {
 	StartTime          time.Time
 	EndTime            time.Time
 	Rule               *Rule
-	//RuleDescription    *RuleDescription
 
 	NoDataFound    bool
 	PrevAlertState monitor.AlertStateType
@@ -52,10 +51,6 @@ type EvalContext struct {
 	UserCred mcclient.TokenCredential
 }
 
-type RuleDescription struct {
-	monitor.AlertRecordRule
-}
-
 // NewEvalContext is the EvalContext constructor.
 func NewEvalContext(alertCtx context.Context, userCred mcclient.TokenCredential, rule *Rule) *EvalContext {
 	return &EvalContext{

pkg/monitor/alerting/notifier.go

@@ -16,7 +16,6 @@ package alerting
 
 import (
 	"database/sql"
-	"fmt"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -39,11 +38,13 @@ func newNotificationService() *notificationService {
 }
 
 func (n *notificationService) SendIfNeeded(evalCtx *EvalContext) error {
-	notifierStates, err := n.getNeededNotifiers(evalCtx.Rule.Notifications, evalCtx)
+	notifierStates, shouldNotify, err := n.getNeededNotifiers(evalCtx.Rule.Notifications, evalCtx)
 	if err != nil {
 		return errors.Wrap(err, "failed to get alert notifiers")
 	}
 
+	n.syncResources(evalCtx, shouldNotify)
+
 	if len(notifierStates) == 0 {
 		return nil
 	}
@@ -98,10 +99,10 @@ func (n *notificationService) sendNotifications(evalCtx *EvalContext, states not
 	return nil
 }
 
-func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalContext) (notifierStateSlice, error) {
+func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalContext) (notifierStateSlice, bool, error) {
 	notis, err := models.NotificationManager.GetNotificationsWithDefault(nIds)
 	if err != nil {
-		return nil, errors.Wrapf(err, "GetNotificationsWithDefault with %v", nIds)
+		return nil, false, errors.Wrapf(err, "GetNotificationsWithDefault with %v", nIds)
 	}
 
 	var result notifierStateSlice
@@ -143,6 +144,11 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon
 			})
 		}
 	}
+
+	return result, shouldNotify, nil
+}
+
+func (n *notificationService) syncResources(evalCtx *EvalContext, shouldNotify bool) {
 	if shouldNotify || evalCtx.Rule.State == monitor.AlertStateAlerting {
 		go func() {
 			if err := n.createAlertRecordWhenNotify(evalCtx, shouldNotify); err != nil {
@@ -156,7 +162,13 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon
 		}()
 	}
 
-	return result, nil
+	if len(evalCtx.AlertOkEvalMatches) > 0 {
+		go func() {
+			if err := n.syncMonitorResourceAlerts(evalCtx); err != nil {
+				log.Errorf("syncMonitorResourceAlerts error: %v", err)
+			}
+		}()
+	}
 }
 
 func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, shouldNotify bool) error {
@@ -166,7 +178,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext,
 	} else {
 		matches = evalCtx.AlertOkEvalMatches
 	}
-	n.dealNeedShieldEvalMatchs(evalCtx, matches)
+	n.dealNeedShieldEvalMatches(evalCtx, matches)
 	recordCreateInput := monitor.AlertRecordCreateInput{
 		StandaloneResourceCreateInput: apis.StandaloneResourceCreateInput{
 			GenerateName: evalCtx.Rule.Name,
@@ -176,7 +188,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext,
 		State:     string(evalCtx.Rule.State),
 		SendState: monitor.SEND_STATE_OK,
 		EvalData:  matches,
-		AlertRule: newAlertRecordRules(evalCtx),
+		AlertRule: evalCtx.Rule.RuleDescription,
 	}
 	if !shouldNotify {
 		recordCreateInput.SendState = monitor.SEND_STATE_SILENT
@@ -187,8 +199,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext,
 	}
 	createData := recordCreateInput.JSON(recordCreateInput)
 	alert, _ := models.CommonAlertManager.GetAlert(evalCtx.Rule.Id)
-	record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(),
-		createData, evalCtx.UserCred)
+	record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), createData, evalCtx.UserCred)
 	if err != nil {
 		return errors.Wrapf(err, "db.DoCreate")
 	}
@@ -206,7 +217,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext,
 	return nil
 }
 
-func (n *notificationService) dealNeedShieldEvalMatchs(evalCtx *EvalContext, match []*monitor.EvalMatch) {
+func (n *notificationService) dealNeedShieldEvalMatches(evalCtx *EvalContext, match []*monitor.EvalMatch) {
 	input := monitor.AlertRecordShieldListInput{
 		ResType: evalCtx.Rule.RuleDescription[0].ResType,
 		AlertId: evalCtx.Rule.Id,
@@ -238,6 +249,32 @@ func (n *notificationService) detachAlertResourceWhenNodata(evalCtx *EvalContext
 	}
 }
 
+func (n *notificationService) syncMonitorResourceAlerts(evalCtx *EvalContext) error {
+	if len(evalCtx.AlertOkEvalMatches) == 0 {
+		log.Infof("alert_ok_eval_matches is empty, skip syncMonitorResourceAlerts")
+		return nil
+	}
+	// only sync resource not need notify
+	matches := make([]monitor.EvalMatch, len(evalCtx.AlertOkEvalMatches))
+	for i := range evalCtx.AlertOkEvalMatches {
+		matches[i] = *evalCtx.AlertOkEvalMatches[i]
+	}
+	alertRule := evalCtx.Rule.RuleDescription
+	input := &models.UpdateMonitorResourceAlertInput{
+		AlertId:       evalCtx.Rule.Id,
+		Matches:       matches,
+		ResType:       alertRule[0].ResType,
+		AlertState:    string(monitor.AlertStateOK),
+		SendState:     monitor.SEND_STATE_SILENT,
+		TriggerTime:   time.Now(),
+		AlertRecordId: "",
+	}
+	if err := models.MonitorResourceManager.UpdateMonitorResourceAttachJoint(evalCtx.Ctx, evalCtx.UserCred, input); err != nil {
+		return errors.Wrap(err, "UpdateMonitorResourceAttachJoint")
+	}
+	return nil
+}
+
 type NotifierPlugin struct {
 	Type               string
 	Factory            NotifierFactory
@@ -271,34 +308,3 @@ func InitNotifier(config NotificationConfig) (Notifier, error) {
 	}
 	return plug.(Notifier), nil
 }
-
-func newAlertRecordRules(ctx *EvalContext) []*monitor.AlertRecordRule {
-	rules := make([]*monitor.AlertRecordRule, 0)
-	for i := range ctx.Rule.RuleDescription {
-		rule := newAlertRecordRule(ctx, i)
-		rules = append(rules, rule)
-	}
-	return rules
-}
-
-func newAlertRecordRule(evalCtx *EvalContext, idx int) *monitor.AlertRecordRule {
-	alertRule := monitor.AlertRecordRule{}
-	if len(evalCtx.Rule.RuleDescription) != 0 {
-		alertRule = evalCtx.Rule.RuleDescription[idx].AlertRecordRule
-	}
-	if evalCtx.Rule.Frequency < 60 {
-		alertRule.Period = fmt.Sprintf("%ds", evalCtx.Rule.Frequency)
-	} else {
-		alertRule.Period = fmt.Sprintf("%dm", evalCtx.Rule.Frequency/60)
-	}
-
-	alertRule.AlertDuration = int64(evalCtx.Rule.For) / evalCtx.Rule.Frequency
-	if alertRule.AlertDuration == 0 {
-		alertRule.AlertDuration = 1
-	}
-
-	if evalCtx.Rule.SilentPeriod != 0 {
-		alertRule.SilentPeriod = fmt.Sprintf("%dm", evalCtx.Rule.SilentPeriod/60)
-	}
-	return &alertRule
-}

pkg/monitor/alerting/rule.go

@@ -16,14 +16,12 @@ package alerting
 
 import (
 	"context"
-	"fmt"
 	"regexp"
 	"strconv"
 	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -61,7 +59,7 @@ type Rule struct {
 	Notifications       []string
 	// AlertRuleTags       []*models.AlertRuleTag
 	Level           string
-	RuleDescription []*RuleDescription
+	RuleDescription []*monitor.AlertRecordRule
 
 	StateChanges int
 
@@ -124,7 +122,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 	model.NoDataState = monitor.NoDataOption(ruleDef.NoDataState)
 	model.ExecutionErrorState = monitor.ExecutionErrorOption(ruleDef.ExecutionErrorState)
 	model.StateChanges = ruleDef.StateChanges
-	model.RuleDescription = make([]*RuleDescription, 0)
+	model.RuleDescription = make([]*monitor.AlertRecordRule, 0)
 
 	model.Frequency = ruleDef.Frequency
 	// frequency cannot be zero since that would not execute the alert rule.
@@ -135,7 +133,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 	model.CustomizeConfig = ruleDef.CustomizeConfig
 	settings, err := ruleDef.GetSettings()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "get settings")
 	}
 
 	model.Level = ruleDef.Level
@@ -158,7 +156,6 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 		return nil, errors.Wrap(err, "GetCommonAlert error")
 	}
 	for index, condition := range settings.Conditions {
-		alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(index, &settings.Conditions[index])
 		condType := condition.Type
 		factory, exist := conditionFactories[condType]
 		if !exist {
@@ -168,7 +165,8 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "construct query condition %s", jsonutils.Marshal(condition))
 		}
-		newRuleDescription(model, alertDetails)
+		ruleDesc := alert.GetAlertRule(settings, index, model.SilentPeriod)
+		model.RuleDescription = append(model.RuleDescription, ruleDesc)
 		model.Conditions = append(model.Conditions, queryCond)
 	}
 
@@ -178,62 +176,6 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
 	return model, nil
 }
 
-func newRuleDescription(rule *Rule, alertDetails *monitor.CommonAlertMetricDetails) {
-	ruleDes := RuleDescription{
-		AlertRecordRule: monitor.AlertRecordRule{
-			ResType:         alertDetails.ResType,
-			Metric:          fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field),
-			Measurement:     alertDetails.Measurement,
-			Database:        alertDetails.DB,
-			MeasurementDesc: alertDetails.MeasurementDisplayName,
-			Field:           alertDetails.Field,
-			FieldDesc:       alertDetails.FieldDescription.DisplayName,
-			Comparator:      alertDetails.Comparator,
-			Threshold:       RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""),
-			ConditionType:   alertDetails.ConditionType,
-			Reducer:         alertDetails.Reduce,
-		},
-	}
-	if len(ruleDes.ResType) == 0 {
-		if alertDetails.DB == monitor.METRIC_DATABASE_TELE {
-			ruleDes.ResType = monitor.METRIC_RES_TYPE_HOST
-		}
-	}
-	rule.RuleDescription = append(rule.RuleDescription, &ruleDes)
-}
-
-var fileSize = []string{"bps", "Bps", "byte"}
-
-func RationalizeValueFromUnit(value float64, unit string, opt string) string {
-	if utils.IsInStringArray(unit, fileSize) {
-		if unit == "byte" {
-			return (FormatFileSize(value, unit, float64(1024)))
-		}
-		return FormatFileSize(value, unit, float64(1000))
-	}
-	if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt {
-		return fmt.Sprintf("%0.4f%s", value*100, unit)
-	}
-	return fmt.Sprintf("%0.4f%s", value, unit)
-}
-
-// 单位转换 保留四位小数
-func FormatFileSize(fileSize float64, unit string, unitsize float64) (size string) {
-	if fileSize < unitsize {
-		return fmt.Sprintf("%.4f%s", fileSize, unit)
-	} else if fileSize < (unitsize * unitsize) {
-		return fmt.Sprintf("%.4fK%s", float64(fileSize)/float64(unitsize), unit)
-	} else if fileSize < (unitsize * unitsize * unitsize) {
-		return fmt.Sprintf("%.4fM%s", float64(fileSize)/float64(unitsize*unitsize), unit)
-	} else if fileSize < (unitsize * unitsize * unitsize * unitsize) {
-		return fmt.Sprintf("%.4fG%s", float64(fileSize)/float64(unitsize*unitsize*unitsize), unit)
-	} else if fileSize < (unitsize * unitsize * unitsize * unitsize * unitsize) {
-		return fmt.Sprintf("%.4fT%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize), unit)
-	} else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024)
-		return fmt.Sprintf("%.4fE%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize*unitsize), unit)
-	}
-}
-
 type AlertRuleTester struct{}
 
 func NewAlertRuleTester() models.AlertTestRunner {

pkg/monitor/models/alertrecord.go

@@ -332,9 +332,9 @@ getNewMatchTag:
 
 func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	record.SStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
-	err := MonitorResourceManager.UpdateMonitorResourceAttachJoint(ctx, userCred, record)
+	err := MonitorResourceManager.UpdateMonitorResourceAttachJointByRecord(ctx, userCred, record)
 	if err != nil {
-		log.Errorf("UpdateMonitorResourceAttachJoint error: %v", err)
+		log.Errorf("UpdateMonitorResourceAttachJointByRecord error: %v", err)
 	}
 	if err := GetAlertResourceManager().ReconcileFromRecord(ctx, userCred, ownerId, record); err != nil {
 		log.Errorf("Reconcile from alert record error: %v", err)
@@ -357,7 +357,6 @@ func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Con
 	records := make([]SAlertRecord, 0)
 	query := manager.Query()
 	query = query.LE("created_at", timeutils.MysqlTime(time.Now().Add(-time.Hour*24*30)))
-	log.Errorf("query:%s", query.String())
 	err := db.FetchModelObjects(manager, query, &records)
 	if err != nil {
 		log.Errorf("fetch records ofthirty days ago err:%v", err)
@@ -366,7 +365,7 @@ func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Con
 	for i, _ := range records {
 		err := db.DeleteModel(ctx, userCred, &records[i])
 		if err != nil {
-			log.Errorf("delete expire record:%s err:%v", records[i].GetId(), err)
+			log.Errorf("delete expire record: %s err: %v", records[i].GetId(), err)
 		}
 	}
 }

pkg/monitor/models/commonalert.go

@@ -473,7 +473,7 @@ func (alert *SCommonAlert) PostCreate(ctx context.Context,
 		log.Errorln(errors.Wrap(err, "Alert PerformSetScope"))
 	}
 	CommonAlertManager.SetSubscriptionAlert(alert)
-	alert.StartUpdateMonitorAlertJointTask(ctx, userCred)
+	//alert.StartUpdateMonitorAlertJointTask(ctx, userCred)
 }
 
 func (man *SCommonAlertManager) ListItemFilter(
@@ -1112,7 +1112,7 @@ func (alert *SCommonAlert) PostUpdate(
 		alert.setMetaName(ctx, userCred, updateInput.MetaName)
 	}
 	CommonAlertManager.SetSubscriptionAlert(alert)
-	alert.StartUpdateMonitorAlertJointTask(ctx, userCred)
+	//alert.StartUpdateMonitorAlertJointTask(ctx, userCred)
 }
 
 func (alert *SCommonAlert) UpdateNotification(ctx context.Context, userCred mcclient.TokenCredential,
@@ -1322,7 +1322,7 @@ func (alert *SCommonAlert) PerformEnable(ctx context.Context, userCred mcclient.
 	if err != nil {
 		return nil, errors.Wrap(err, "EnabledPerformEnable")
 	}
-	alert.StartUpdateMonitorAlertJointTask(ctx, userCred)
+	//alert.StartUpdateMonitorAlertJointTask(ctx, userCred)
 	return nil, nil
 }
 
@@ -1406,61 +1406,25 @@ func (alert *SCommonAlert) UpdateMonitorResourceJoint(ctx context.Context, userC
 	if err != nil {
 		return errors.Wrapf(err, "TestRunAlert %s", alert.GetName())
 	}
-	resourceIds := make([]string, 0)
-	for _, em := range ret.EvalMatches {
-		resourceKeyId := monitor.MEASUREMENT_TAG_ID[resType]
-		resourceId := em.Tags[resourceKeyId]
-		if len(resourceId) == 0 {
-			continue
+	if len(ret.AlertOKEvalMatches) > 0 {
+		matches := make([]monitor.EvalMatch, len(ret.AlertOKEvalMatches))
+		for i := range ret.AlertOKEvalMatches {
+			matches[i] = *ret.AlertOKEvalMatches[i]
 		}
-		resourceIds = append(resourceIds, resourceId)
-	}
-	deleteJointIds := make([]int64, 0)
-	joints, _ := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{AlertId: alert.GetId()})
-jointLoop:
-	for _, joint := range joints {
-		for i, resId := range resourceIds {
-			if resId == joint.MonitorResourceId {
-				resourceIds = append(resourceIds[0:i], resourceIds[i+1:]...)
-				continue jointLoop
-			}
+		input := &UpdateMonitorResourceAlertInput{
+			AlertId:       alert.GetId(),
+			Matches:       matches,
+			ResType:       resType,
+			AlertState:    string(monitor.AlertStateOK),
+			SendState:     monitor.SEND_STATE_SILENT,
+			TriggerTime:   time.Now(),
+			AlertRecordId: "",
 		}
-		// 排除近期有报警状态的情况：system.uptime
-		if joint.AlertState == monitor.MONITOR_RESOURCE_ALERT_STATUS_ALERTING && time.Now().Sub(joint.TriggerTime).
-			Minutes() < 30 {
-			continue
+		if err := MonitorResourceManager.UpdateMonitorResourceAttachJoint(ctx, userCred, input); err != nil {
+			return errors.Wrap(err, "UpdateMonitorResourceAttachJoint")
 		}
-		deleteJointIds = append(deleteJointIds, joint.RowId)
-	}
-
-	if len(resourceIds) == 0 && len(deleteJointIds) == 0 {
 		return nil
 	}
-	//  sync joints should be deleted
-	if len(deleteJointIds) > 0 {
-		err := MonitorResourceAlertManager.DetachJoint(ctx, userCred, monitor.MonitorResourceJointListInput{JointId: deleteJointIds})
-		if err != nil {
-			return errors.Wrapf(err, "DetachJoint by alert %s(%s)", alert.GetName(), alert.GetId())
-		}
-	}
-
-	if len(resourceIds) > 0 {
-		monitorResources, _ := MonitorResourceManager.GetMonitorResources(monitor.MonitorResourceListInput{ResId: resourceIds})
-		errs := make([]error, 0)
-		for _, monRes := range monitorResources {
-			resDesc := fmt.Sprintf("%s/%s/%s", monRes.ResType, monRes.GetName(), monRes.ResId)
-			if _, err := monRes.AttachAlert(ctx, userCred, alert.GetId()); err != nil {
-				errs = append(errs, errors.Wrapf(err, "AttachAlert %s to %s", alert.GetName(), resDesc))
-			}
-			if err := monRes.UpdateAlertState(); err != nil {
-				errs = append(errs, errors.Wrapf(err, "UpdateAlertState for monitor resource %s", resDesc))
-			}
-		}
-
-		if len(errs) != 0 {
-			return errors.NewAggregate(errs)
-		}
-	}
 	return nil
 }
 
@@ -1505,6 +1469,100 @@ func (alert *SCommonAlert) UpdateResType() error {
 	return nil
 }
 
+func (alert *SCommonAlert) GetSilentPeriod() (int64, error) {
+	notis, err := alert.GetNotifications()
+	if err != nil {
+		return 0, errors.Wrap(err, "GetNotifications")
+	}
+	for _, n := range notis {
+		noti, _ := n.GetNotification()
+		if noti.Frequency != 0 {
+			return noti.Frequency, nil
+		}
+	}
+	return 0, nil
+}
+
+func (alert *SCommonAlert) GetAlertRules(silentPeriod int64) ([]*monitor.AlertRecordRule, error) {
+	rules := make([]*monitor.AlertRecordRule, 0)
+	settings, err := alert.GetSettings()
+	if err != nil {
+		return nil, errors.Wrapf(err, "get alert %s settings", alert.GetId())
+	}
+	for index := range settings.Conditions {
+		rule := alert.GetAlertRule(settings, index, silentPeriod)
+		rules = append(rules, rule)
+	}
+	return rules, nil
+}
+
+func (alert *SCommonAlert) GetAlertRule(settings *monitor.AlertSetting, index int, silentPeriod int64) *monitor.AlertRecordRule {
+	alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(index, &settings.Conditions[index])
+	rule := &monitor.AlertRecordRule{
+		ResType:         alertDetails.ResType,
+		Metric:          fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field),
+		Measurement:     alertDetails.Measurement,
+		Database:        alertDetails.DB,
+		MeasurementDesc: alertDetails.MeasurementDisplayName,
+		Field:           alertDetails.Field,
+		FieldDesc:       alertDetails.FieldDescription.DisplayName,
+		Comparator:      alertDetails.Comparator,
+		Threshold:       RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""),
+		ConditionType:   alertDetails.ConditionType,
+		Reducer:         alertDetails.Reduce,
+	}
+	if len(rule.ResType) == 0 {
+		if alertDetails.DB == monitor.METRIC_DATABASE_TELE {
+			rule.ResType = monitor.METRIC_RES_TYPE_HOST
+		}
+	}
+	if alert.Frequency < 60 {
+		rule.Period = fmt.Sprintf("%ds", alert.Frequency)
+	} else {
+		rule.Period = fmt.Sprintf("%dm", alert.Frequency/60)
+	}
+	rule.AlertDuration = alert.For / alert.Frequency
+	if rule.AlertDuration == 0 {
+		rule.AlertDuration = 1
+	}
+	if silentPeriod > 0 {
+		rule.SilentPeriod = fmt.Sprintf("%dm", silentPeriod/60)
+	}
+	return rule
+}
+
+var fileSize = []string{"bps", "Bps", "byte"}
+
+func RationalizeValueFromUnit(value float64, unit string, opt string) string {
+	if utils.IsInStringArray(unit, fileSize) {
+		if unit == "byte" {
+			return (FormatFileSize(value, unit, float64(1024)))
+		}
+		return FormatFileSize(value, unit, float64(1000))
+	}
+	if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt {
+		return fmt.Sprintf("%0.4f%s", value*100, unit)
+	}
+	return fmt.Sprintf("%0.4f%s", value, unit)
+}
+
+// 单位转换 保留四位小数
+func FormatFileSize(fileSize float64, unit string, unitsize float64) (size string) {
+	if fileSize < unitsize {
+		return fmt.Sprintf("%.4f%s", fileSize, unit)
+	} else if fileSize < (unitsize * unitsize) {
+		return fmt.Sprintf("%.4fK%s", float64(fileSize)/float64(unitsize), unit)
+	} else if fileSize < (unitsize * unitsize * unitsize) {
+		return fmt.Sprintf("%.4fM%s", float64(fileSize)/float64(unitsize*unitsize), unit)
+	} else if fileSize < (unitsize * unitsize * unitsize * unitsize) {
+		return fmt.Sprintf("%.4fG%s", float64(fileSize)/float64(unitsize*unitsize*unitsize), unit)
+	} else if fileSize < (unitsize * unitsize * unitsize * unitsize * unitsize) {
+		return fmt.Sprintf("%.4fT%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize), unit)
+	} else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024)
+		return fmt.Sprintf("%.4fE%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize*unitsize), unit)
+	}
+}
+
 type SCompanyInfo struct {
 	Copyright string `json:"copyright"`
 	Name      string `json:"name"`

pkg/monitor/models/monitor_resource.go

@@ -262,12 +262,14 @@ func (man *SMonitorResourceManager) FetchCustomizeColumns(
 	return rows
 }
 
-func (self *SMonitorResource) AttachAlert(ctx context.Context, userCred mcclient.TokenCredential, alertId string) (*SMonitorResourceAlert, error) {
+func (self *SMonitorResource) AttachAlert(ctx context.Context, userCred mcclient.TokenCredential, alertId string, metric string, match monitor.EvalMatch) (*SMonitorResourceAlert, error) {
 	iModel, _ := db.NewModelObject(MonitorResourceAlertManager)
 	input := monitor.MonitorResourceJointCreateInput{
 		MonitorResourceId: self.ResId,
 		AlertId:           alertId,
 		AlertState:        monitor.MONITOR_RESOURCE_ALERT_STATUS_ATTACH,
+		Metric:            metric,
+		Data:              match,
 	}
 	data := input.JSON(&input)
 	err := data.Unmarshal(iModel)
@@ -379,21 +381,47 @@ func (manager *SMonitorResourceManager) GetPropertyAlert(ctx context.Context, us
 	return result, nil
 }
 
-func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context,
-	userCred mcclient.TokenCredential, alertRecord *SAlertRecord) error {
+func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJointByRecord(ctx context.Context, userCred mcclient.TokenCredential, record *SAlertRecord) error {
+	matches, _ := record.GetEvalData()
+	input := &UpdateMonitorResourceAlertInput{
+		AlertId:       record.AlertId,
+		Matches:       matches,
+		ResType:       record.ResType,
+		AlertState:    record.State,
+		SendState:     record.SendState,
+		TriggerTime:   record.CreatedAt,
+		AlertRecordId: record.GetId(),
+	}
+	if err := manager.UpdateMonitorResourceAttachJoint(ctx, userCred, input); err != nil {
+		return errors.Wrap(err, "UpdateMonitorResourceAttachJoint")
+	}
+	return nil
+}
+
+type UpdateMonitorResourceAlertInput struct {
+	AlertId       string
+	Matches       []monitor.EvalMatch
+	ResType       string
+	AlertState    string
+	SendState     string
+	TriggerTime   time.Time
+	AlertRecordId string
+}
+
+func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context, userCred mcclient.TokenCredential, input *UpdateMonitorResourceAlertInput) error {
 	//if !utils.IsInStringArray(alertRecord.ResType, []string{monitor.METRIC_RES_TYPE_HOST,
 	//	monitor.METRIC_RES_TYPE_GUEST, monitor.METRIC_RES_TYPE_AGENT}) {
 	//	return nil
 	//}
-	resType := alertRecord.ResType
+	resType := input.ResType
 	if resType == monitor.METRIC_RES_TYPE_AGENT {
 		resType = monitor.METRIC_RES_TYPE_GUEST
 	}
-	matches, _ := alertRecord.GetEvalData()
+	matches := input.Matches
 	errs := make([]error, 0)
 	matchResourceIds := make([]string, 0)
-	for _, matche := range matches {
-		resId := matche.Tags[monitor.MEASUREMENT_TAG_ID[alertRecord.ResType]]
+	for _, match := range matches {
+		resId := match.Tags[monitor.MEASUREMENT_TAG_ID[input.ResType]]
 		if len(resId) == 0 {
 			continue
 		}
@@ -404,24 +432,22 @@ func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx con
 			continue
 		}
 		for _, res := range monitorResources {
-			err := res.UpdateAttachJoint(alertRecord, matche)
+			err := res.UpdateAttachJoint(ctx, userCred, input, match)
 			if err != nil {
-				errs = append(errs, err)
+				errs = append(errs, errors.Wrap(err, "UpdateAttachJoint"))
 			}
 		}
 	}
-	resourceAlerts, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{AlertId: alertRecord.AlertId})
+	resourceAlerts, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{
+		AlertId:    input.AlertId,
+		AlertState: input.AlertState,
+	})
 	if err != nil {
-		return errors.Wrapf(err, "get monitor_resource_joint by alertId:%s err", alertRecord.AlertId)
+		return errors.Wrapf(err, "get monitor_resource_joint by alertId: %s", input.AlertId)
 	}
 	deleteJointIds := make([]int64, 0)
 	for _, joint := range resourceAlerts {
-		evalData, err := joint.GetData()
-		if err != nil {
-			log.Warningf("get data of monitor_resource_alert %s: %s", jsonutils.Marshal(joint), err)
-			continue
-		}
-		metricName := evalData.Metric
+		metricName := joint.Metric
 		isMetricFound := false
 		for _, match := range matches {
 			if match.Metric == metricName {
@@ -437,51 +463,49 @@ func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx con
 	if len(deleteJointIds) != 0 {
 		err = MonitorResourceAlertManager.DetachJoint(ctx, userCred, monitor.MonitorResourceJointListInput{JointId: deleteJointIds})
 		if err != nil {
-			return errors.Wrapf(err, "DetachJoint by alertId:%s err", alertRecord.AlertId)
+			return errors.Wrapf(err, "DetachJoint by alertId:%s err", input.AlertId)
 		}
 	}
 	return errors.NewAggregate(errs)
 }
 
-func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match monitor.EvalMatch) error {
-	joints, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{MonitorResourceId: self.
-		ResId, AlertId: alertRecord.AlertId})
+func (self *SMonitorResource) UpdateAttachJoint(ctx context.Context, userCred mcclient.TokenCredential, input *UpdateMonitorResourceAlertInput, match monitor.EvalMatch) error {
+	joints, err := MonitorResourceAlertManager.GetJoinsByListInput(
+		monitor.MonitorResourceJointListInput{
+			MonitorResourceId: self.ResId,
+			AlertId:           input.AlertId,
+			Metric:            match.Metric,
+		})
 	if err != nil {
-		return errors.Wrapf(err, "SMonitorResource:%s UpdateAttachJoint err", self.Name)
+		return errors.Wrapf(err, "SMonitorResource: %s(%s) get joints by monitorResourceId %q , metric %q and alertId %q", self.Name, self.Id, self.ResId, match.Metric, input.AlertId)
 	}
 	errs := make([]error, 0)
 	updateJoints := make([]SMonitorResourceAlert, 0)
 	for _, joint := range joints {
-		jointMatch := new(monitor.EvalMatch)
-		if joint.Data != nil {
-			if err := joint.Data.Unmarshal(jointMatch); err != nil {
-				return errors.Wrapf(err, "unmarshal joint %s to monitor.EvalMatch", jsonutils.Marshal(joint))
-			}
-			if jointMatch.Metric == match.Metric {
-				tmpJoint := joint
-				updateJoints = append(updateJoints, tmpJoint)
-			}
+		if joint.Metric == match.Metric {
+			tmpJoint := joint
+			updateJoints = append(updateJoints, tmpJoint)
 		}
 	}
 	// 报警时发现没有进行关联，增加attach
 	if len(updateJoints) == 0 {
-		newJoint, err := self.AttachAlert(context.Background(), nil, alertRecord.AlertId)
+		newJoint, err := self.AttachAlert(ctx, userCred, input.AlertId, match.Metric, match)
 		if err != nil {
 			log.Errorf("attach alert error: %s", err)
 		}
 		log.Infof("Attach Alert joint: %#v, match: %s", newJoint, jsonutils.Marshal(match))
-		if err := newJoint.UpdateAlertRecordData(alertRecord, &match); err != nil {
+		if err := newJoint.UpdateAlertRecordData(input, &match); err != nil {
 			errs = append(errs, errors.Wrapf(err, "new joint %s:%s %s:%s UpdateAlertRecordData err",
 				MonitorResourceAlertManager.GetMasterFieldName(), self.ResId,
-				MonitorResourceAlertManager.GetSlaveFieldName(), alertRecord.AlertId))
+				MonitorResourceAlertManager.GetSlaveFieldName(), input.AlertId))
 		}
 	} else {
 		for _, joint := range updateJoints {
-			err := joint.UpdateAlertRecordData(alertRecord, &match)
+			err := joint.UpdateAlertRecordData(input, &match)
 			if err != nil {
 				errs = append(errs, errors.Wrapf(err, "joint %s:%s %s:%s UpdateAlertRecordData err",
 					MonitorResourceAlertManager.GetMasterFieldName(), self.ResId,
-					MonitorResourceAlertManager.GetSlaveFieldName(), alertRecord.AlertId))
+					MonitorResourceAlertManager.GetSlaveFieldName(), input.AlertId))
 			}
 		}
 	}
@@ -489,7 +513,6 @@ func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match
 		errs = append(errs, errors.Wrapf(err, "UpdateAlertState"))
 	}
 	return errors.NewAggregate(errs)
-
 }
 
 func (manager *SMonitorResourceManager) GetResourceObj(id string) (bool, jsonutils.JSONObject) {

pkg/monitor/models/monitor_resource_alert.go

@@ -58,6 +58,7 @@ type SMonitorResourceAlert struct {
 	AlertId           string               `width:"36" charset:"ascii" list:"user" create:"required" index:"true"`
 	AlertRecordId     string               `width:"36" charset:"ascii" list:"user" update:"user"`
 	ResType           string               `width:"36" charset:"ascii" list:"user" update:"user" json:"res_type"`
+	Metric            string               `width:"36" charset:"ascii" list:"user" create:"required" json:"metric"`
 	AlertState        string               `width:"18" charset:"ascii" default:"init" list:"user" update:"user"`
 	SendState         string               `width:"18" charset:"ascii" default:"ok" list:"user" update:"user"`
 	TriggerTime       time.Time            `list:"user"  update:"user" json:"trigger_time"`
@@ -123,9 +124,15 @@ func (manager *SMonitorResourceAlertManager) GetJoinsByListInput(input monitor.M
 	if len(input.AlertId) != 0 {
 		query.Equals(manager.GetSlaveFieldName(), input.AlertId)
 	}
+	if len(input.Metric) > 0 {
+		query.Equals("metric", input.Metric)
+	}
 	if len(input.JointId) != 0 {
 		query.In("row_id", input.JointId)
 	}
+	if len(input.AlertState) > 0 {
+		query = query.Equals("alert_state", input.AlertState)
+	}
 	err := db.FetchModelObjects(manager, query, &joints)
 	if err != nil {
 		return nil, errors.Wrapf(err, "FetchModelObjects by GetJoinsByMasterId:%s err", input.MonitorResourceId)
@@ -133,21 +140,24 @@ func (manager *SMonitorResourceAlertManager) GetJoinsByListInput(input monitor.M
 	return joints, nil
 }
 
-func (obj *SMonitorResourceAlert) UpdateAlertRecordData(record *SAlertRecord, match *monitor.EvalMatch) error {
-	sendState := record.SendState
+func (obj *SMonitorResourceAlert) UpdateAlertRecordData(input *UpdateMonitorResourceAlertInput, match *monitor.EvalMatch) error {
+	sendState := input.SendState
 	if _, ok := match.Tags[monitor.ALERT_RESOURCE_RECORD_SHIELD_KEY]; ok {
 		sendState = monitor.SEND_STATE_SHIELD
 	}
 	if _, err := db.Update(obj, func() error {
-		obj.AlertRecordId = record.GetId()
-		obj.ResType = record.ResType
-		obj.AlertState = record.State
+		if input.AlertRecordId != "" {
+			obj.AlertRecordId = input.AlertRecordId
+		}
+		obj.ResType = input.ResType
+		obj.AlertState = input.AlertState
 		obj.SendState = sendState
-		obj.TriggerTime = record.CreatedAt
+		obj.TriggerTime = input.TriggerTime
+		obj.Metric = match.Metric
 		obj.Data = jsonutils.Marshal(match)
 		return nil
 	}); err != nil {
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -169,6 +179,9 @@ func (m *SMonitorResourceAlertManager) ListItemFilter(ctx context.Context, q *sq
 	if err != nil {
 		return q, errors.Wrap(err, "SJointResourceBaseManager ListItemFilter err")
 	}
+	if len(input.AlertState) > 0 {
+		q = q.Equals("alert_state", input.AlertState)
+	}
 	if input.Alerting {
 		q = q.Equals("alert_state", monitor.AlertStateAlerting)
 		resQ := MonitorResourceManager.Query("res_id")
@@ -207,8 +220,10 @@ func (m *SMonitorResourceAlertManager) ListItemFilter(ctx context.Context, q *sq
 	if len(input.MonitorResourceId) != 0 {
 		q = q.Filter(sqlchemy.Equals(q.Field("monitor_resource_id"), input.MonitorResourceId))
 	}
-	q = q.Filter(sqlchemy.In(q.Field(m.GetSlaveFieldName()), alertQuery.SubQuery()))
-	q = q.Filter(sqlchemy.In(q.Field("alert_record_id"), AlertRecordManager.Query("id").SubQuery()))
+	if !input.AllState {
+		q = q.Filter(sqlchemy.In(q.Field(m.GetSlaveFieldName()), alertQuery.SubQuery()))
+		q = q.Filter(sqlchemy.In(q.Field("alert_record_id"), AlertRecordManager.Query("id").SubQuery()))
+	}
 	return q, nil
 }
 
@@ -228,8 +243,7 @@ func (man *SMonitorResourceAlertManager) FetchCustomizeColumns(
 	return rows
 }
 
-func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJointDetails) monitor.
-	MonitorResourceJointDetails {
+func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJointDetails) monitor.MonitorResourceJointDetails {
 	resources, err := MonitorResourceManager.GetMonitorResources(monitor.MonitorResourceListInput{ResId: []string{obj.
 		MonitorResourceId}})
 	if err != nil {
@@ -249,7 +263,6 @@ func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJ
 			log.Errorf("get alertRecord:%s err:%v", obj.AlertRecordId, err)
 			return detail
 		}
-		detail.Level = record.Level
 		detail.AlertRule = record.AlertRule
 		detail.SendState = record.SendState
 		detail.State = record.State
@@ -260,13 +273,18 @@ func (obj *SMonitorResourceAlert) getMoreDetails(detail monitor.MonitorResourceJ
 		return detail
 	}
 	detail.AlertName = alert.Name
+	detail.Level = alert.Level
+	if len(obj.AlertRecordId) == 0 {
+		silentPeriod, _ := alert.GetSilentPeriod()
+		rule, _ := alert.GetAlertRules(silentPeriod)
+		detail.AlertRule = jsonutils.Marshal(rule)
+	}
 
 	now := time.Now()
 	shields, err := AlertRecordShieldManager.GetRecordShields(monitor.AlertRecordShieldListInput{ResId: obj.MonitorResourceId,
 		AlertId: obj.AlertId, EndTime: &now})
 	if err != nil {
-		log.Errorf("SMonitorResourceAlert get GetRecordShields by resId: %s,alertId: %s, err: %v",
-			obj.MonitorResourceId, obj.AlertId, err)
+		log.Errorf("SMonitorResourceAlert get GetRecordShields by resId: %s, alertId: %s, err: %v", obj.MonitorResourceId, obj.AlertId, err)
 		return detail
 	}
 	if len(shields) != 0 {