diff --git a/pkg/apis/monitor/alert.go b/pkg/apis/monitor/alert.go index 7122fa2e1e..655122848b 100644 --- a/pkg/apis/monitor/alert.go +++ b/pkg/apis/monitor/alert.go @@ -165,14 +165,12 @@ type ResultLogEntry struct { // EvalMatch represents the series violating the threshold. type EvalMatch struct { - Condition string `json:"condition"` - Value *float64 `json:"value"` - ValueStr string `json:"value_str"` - Metric string `json:"metric"` - MeasurementDesc string `json:"measurement_desc"` - FieldDesc string `json:"field_desc"` - Tags map[string]string `json:"tags"` - Unit string `json:"unit"` + Condition string `json:"condition"` + Value *float64 `json:"value"` + ValueStr string `json:"value_str"` + Metric string `json:"metric"` + Tags map[string]string `json:"tags"` + Unit string `json:"unit"` } type AlertTestRunOutput struct { diff --git a/pkg/apis/monitor/alertrecord.go b/pkg/apis/monitor/alertrecord.go index 1d5b0f5991..66b5e266bb 100644 --- a/pkg/apis/monitor/alertrecord.go +++ b/pkg/apis/monitor/alertrecord.go @@ -36,6 +36,7 @@ type AlertRecordRule struct { Metric string `json:"metric"` Measurement string `json:"measurement"` MeasurementDesc string `json:"measurement_desc"` + ResType string `json:"res_type"` Field string `json:"field"` FieldDesc string `json:"field_desc"` // 比较运算符, 比如: >, <, >=, <= diff --git a/pkg/apis/monitor/unifiedmonitor_const.go b/pkg/apis/monitor/unifiedmonitor_const.go index 01570039bd..8615ffb271 100644 --- a/pkg/apis/monitor/unifiedmonitor_const.go +++ b/pkg/apis/monitor/unifiedmonitor_const.go @@ -20,6 +20,14 @@ var ( "oss": "oss_name", "cloudaccount": "cloudaccount_name", } + MEASUREMENT_TAG_ID = map[string]string{ + "host": "host_id", + "guest": "vm_id", + "redis": "redis_id", + "rds": "rds_id", + "oss": "oss_id", + "cloudaccount": "cloudaccount_id", + } AlertReduceFunc = map[string]string{ "avg": "average value", "sum": "Summation", diff --git a/pkg/monitor/alerting/conditions/nodataquery.go b/pkg/monitor/alerting/conditions/nodataquery.go index 2c318efbfd..d6df040a41 100644 --- a/pkg/monitor/alerting/conditions/nodataquery.go +++ b/pkg/monitor/alerting/conditions/nodataquery.go @@ -3,7 +3,6 @@ package conditions import ( "context" "fmt" - "strings" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" @@ -50,8 +49,22 @@ func (c *NoDataQueryCondition) Eval(context *alerting.EvalContext) (*alerting.Co normalHostIds := make(map[string]*monitor.EvalMatch, 0) serLoop: for _, series := range seriesList { + tagId := monitor.MEASUREMENT_TAG_ID[context.Rule.RuleDescription[0].ResType] + if len(tagId) == 0 { + tagId = "host_id" + } for key, val := range series.Tags { - if strings.Contains(key, "host_id") { + if key == tagId { + if len(context.Rule.RuleDescription) == 0 { + return &alerting.ConditionResult{ + Firing: false, + NoDataFound: true, + Operator: c.Operator, + EvalMatches: matches, + AlertOkEvalMatches: alertOkmatches, + }, nil + } + reducedValue, valStrArr := c.Reducer.Reduce(series) match, err := c.NewEvalMatch(context, *series, nil, reducedValue, valStrArr) if err != nil { @@ -62,7 +75,7 @@ serLoop: } } } - allHosts, err := c.getOnecloudHosts() + allHosts, err := c.getOnecloudResources(context) if err != nil { return nil, errors.Wrap(err, "NoDataQueryCondition getOnecloudHosts error") } @@ -73,10 +86,10 @@ serLoop: return nil, errors.Wrap(err, "NewNoDataEvalMatch error") } if normalMatch, ok := normalHostIds[id]; !ok { - c.createEvalMatchTagFromHostJson(evalMatch, host) + c.createEvalMatchTagFromHostJson(context, evalMatch, host) matches = append(matches, evalMatch) } else { - c.createEvalMatchTagFromHostJson(normalMatch, host) + c.createEvalMatchTagFromHostJson(context, normalMatch, host) alertOkmatches = append(alertOkmatches, normalMatch) } } @@ -89,15 +102,37 @@ serLoop: }, nil } -func (c *NoDataQueryCondition) getOnecloudHosts() ([]jsonutils.JSONObject, error) { +func (c *NoDataQueryCondition) getOnecloudResources(evalContext *alerting.EvalContext) ([]jsonutils.JSONObject, error) { + var err error + allResources := make([]jsonutils.JSONObject, 0) + if len(evalContext.Rule.RuleDescription) == 0 { + return []jsonutils.JSONObject{}, nil + } query := jsonutils.NewDict() - query.Set("brand", jsonutils.NewString(hostconsts.TELEGRAF_TAG_ONECLOUD_BRAND)) - query.Set("host-type", jsonutils.NewString(hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR)) - allHosts, err := ListAllResources(&mc_mds.Hosts, query) + query.Add(jsonutils.NewStringArray([]string{"running", "ready"}), "status") + query.Add(jsonutils.NewString("true"), "admin") + switch evalContext.Rule.RuleDescription[0].ResType { + case monitor.METRIC_RES_TYPE_HOST: + query.Set("host-type", jsonutils.NewString(hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR)) + allResources, err = ListAllResources(&mc_mds.Hosts, query) + case monitor.METRIC_RES_TYPE_GUEST: + case monitor.METRIC_RES_TYPE_RDS: + allResources, err = ListAllResources(&mc_mds.DBInstance, query) + case monitor.METRIC_RES_TYPE_REDIS: + allResources, err = ListAllResources(&mc_mds.ElasticCache, query) + case monitor.METRIC_RES_TYPE_OSS: + allResources, err = ListAllResources(&mc_mds.Buckets, query) + default: + query := jsonutils.NewDict() + query.Set("brand", jsonutils.NewString(hostconsts.TELEGRAF_TAG_ONECLOUD_BRAND)) + query.Set("host-type", jsonutils.NewString(hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR)) + allResources, err = ListAllResources(&mc_mds.Hosts, query) + } + if err != nil { return nil, errors.Wrap(err, "NoDataQueryCondition Host list error") } - return allHosts, nil + return allResources, nil } func ListAllResources(manager modulebase.Manager, params *jsonutils.JSONDict) ([]jsonutils.JSONObject, error) { @@ -145,18 +180,17 @@ func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext, } evalMatch.Unit = alertDetails.FieldDescription.Unit msg := fmt.Sprintf("%s.%s %s %s", alertDetails.Measurement, alertDetails.Field, - alertDetails.Comparator, c.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, "")) + alertDetails.Comparator, alerting.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, "")) if len(context.Rule.Message) == 0 { context.Rule.Message = msg } //evalMatch.Condition = c.GenerateFormatCond(meta, queryKeyInfo).String() evalMatch.ValueStr = NO_DATA - evalMatch.MeasurementDesc = alertDetails.MeasurementDisplayName - evalMatch.FieldDesc = alertDetails.FieldDescription.DisplayName return evalMatch, nil } -func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalMatch *monitor.EvalMatch, host jsonutils.JSONObject) { +func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalContext *alerting.EvalContext, evalMatch *monitor.EvalMatch, + host jsonutils.JSONObject) { evalMatch.Tags = make(map[string]string, 0) ip, _ := host.GetString(HOST_TAG_IP) @@ -164,11 +198,20 @@ func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalMatch *monitor brand, _ := host.GetString(HOST_TAG_BRAND) evalMatch.Tags["ip"] = ip evalMatch.Tags[HOST_TAG_NAME] = name - evalMatch.Tags["host"] = name evalMatch.Tags[HOST_TAG_BRAND] = brand - evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_RES_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_RES_TYPE - evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_HOST_TYPE_HOST + switch evalContext.Rule.RuleDescription[0].ResType { + case monitor.METRIC_RES_TYPE_HOST: + evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_RES_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_RES_TYPE + evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_HOST_TYPE_HOST + case monitor.METRIC_RES_TYPE_GUEST: + case monitor.METRIC_RES_TYPE_RDS: + case monitor.METRIC_RES_TYPE_REDIS: + case monitor.METRIC_RES_TYPE_OSS: + default: + evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_RES_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_RES_TYPE + evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_HOST_TYPE_HOST + } } func newNoDataQueryCondition(model *monitor.AlertCondition, index int) (*NoDataQueryCondition, error) { diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index 967679ab15..e43f0268d0 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -21,7 +21,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis/monitor" "yunion.io/x/onecloud/pkg/monitor/alerting" @@ -211,37 +210,22 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series tsdb } evalMatch.Unit = alertDetails.FieldDescription.Unit msg := fmt.Sprintf("%s.%s %s %s", alertDetails.Measurement, alertDetails.Field, - alertDetails.Comparator, c.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, "")) + alertDetails.Comparator, alerting.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, "")) if len(context.Rule.Message) == 0 { context.Rule.Message = msg } //evalMatch.Condition = c.GenerateFormatCond(meta, queryKeyInfo).String() evalMatch.Tags = c.filterTags(series.Tags, *alertDetails) evalMatch.Value = value - evalMatch.ValueStr = c.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit, + evalMatch.ValueStr = alerting.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit, alertDetails.FieldOpt) if alertDetails.GetPointStr { evalMatch.ValueStr = c.jointPointStr(series, evalMatch.ValueStr, valStrArr) } - c.newRuleDescription(context, alertDetails) + //c.newRuleDescription(context, alertDetails) return evalMatch, nil } -func (c *QueryCondition) newRuleDescription(context *alerting.EvalContext, alertDetails *monitor.CommonAlertMetricDetails) { - ruleDes := alerting.RuleDescription{ - AlertRecordRule: monitor.AlertRecordRule{ - Metric: fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field), - Measurement: alertDetails.Measurement, - MeasurementDesc: alertDetails.MeasurementDisplayName, - Field: alertDetails.Field, - FieldDesc: alertDetails.FieldDescription.DisplayName, - Comparator: alertDetails.Comparator, - Threshold: c.RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""), - }, - } - context.RuleDescription = &ruleDes -} - func (c *QueryCondition) jointPointStr(series tsdb.TimeSeries, value string, valStrArr []string) string { str := "" for i := 0; i < len(valStrArr); i++ { @@ -254,38 +238,6 @@ func (c *QueryCondition) jointPointStr(series tsdb.TimeSeries, value string, val return str } -var fileSize = []string{"bps", "Bps", "byte"} - -func (c *QueryCondition) RationalizeValueFromUnit(value float64, unit string, opt string) string { - if utils.IsInStringArray(unit, fileSize) { - if unit == "byte" { - return (formatFileSize(value, unit, float64(1024))) - } - return formatFileSize(value, unit, float64(1000)) - } - if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt { - return fmt.Sprintf("%0.4f%s", value*100, unit) - } - return fmt.Sprintf("%0.4f%s", value, unit) -} - -// 单位转换 保留四位小数 -func formatFileSize(fileSize float64, unit string, unitsize float64) (size string) { - if fileSize < unitsize { - return fmt.Sprintf("%.4f%s", fileSize, unit) - } else if fileSize < (unitsize * unitsize) { - return fmt.Sprintf("%.4fK%s", float64(fileSize)/float64(unitsize), unit) - } else if fileSize < (unitsize * unitsize * unitsize) { - return fmt.Sprintf("%.4fM%s", float64(fileSize)/float64(unitsize*unitsize), unit) - } else if fileSize < (unitsize * unitsize * unitsize * unitsize) { - return fmt.Sprintf("%.4fG%s", float64(fileSize)/float64(unitsize*unitsize*unitsize), unit) - } else if fileSize < (unitsize * unitsize * unitsize * unitsize * unitsize) { - return fmt.Sprintf("%.4fT%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize), unit) - } else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024) - return fmt.Sprintf("%.4fE%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize*unitsize), unit) - } -} - type queryResult struct { series tsdb.TimeSeriesSlice metas []tsdb.QueryResultMeta diff --git a/pkg/monitor/alerting/eval_context.go b/pkg/monitor/alerting/eval_context.go index 9845e8f823..5086f7fa67 100644 --- a/pkg/monitor/alerting/eval_context.go +++ b/pkg/monitor/alerting/eval_context.go @@ -42,7 +42,7 @@ type EvalContext struct { StartTime time.Time EndTime time.Time Rule *Rule - RuleDescription *RuleDescription + //RuleDescription *RuleDescription NoDataFound bool PrevAlertState monitor.AlertStateType diff --git a/pkg/monitor/alerting/notifier.go b/pkg/monitor/alerting/notifier.go index f5cd2eff41..8133247212 100644 --- a/pkg/monitor/alerting/notifier.go +++ b/pkg/monitor/alerting/notifier.go @@ -219,8 +219,8 @@ func InitNotifier(config NotificationConfig) (Notifier, error) { func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule { alertRule := monitor.AlertRecordRule{} - if evalCtx.RuleDescription != nil { - alertRule = evalCtx.RuleDescription.AlertRecordRule + if len(evalCtx.Rule.RuleDescription) != 0 { + alertRule = evalCtx.Rule.RuleDescription[0].AlertRecordRule } if evalCtx.Rule.Frequency < 60 { alertRule.Period = fmt.Sprintf("%ds", evalCtx.Rule.Frequency) diff --git a/pkg/monitor/alerting/rule.go b/pkg/monitor/alerting/rule.go index ff8eb67216..d22d17edd1 100644 --- a/pkg/monitor/alerting/rule.go +++ b/pkg/monitor/alerting/rule.go @@ -16,12 +16,14 @@ package alerting import ( "context" + "fmt" "regexp" "strconv" "time" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis/monitor" "yunion.io/x/onecloud/pkg/mcclient" @@ -56,7 +58,8 @@ type Rule struct { Conditions []Condition Notifications []string // AlertRuleTags []*models.AlertRuleTag - Level string + Level string + RuleDescription []*RuleDescription StateChanges int } @@ -114,6 +117,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { model.NoDataState = monitor.NoDataOption(ruleDef.NoDataState) model.ExecutionErrorState = monitor.ExecutionErrorOption(ruleDef.ExecutionErrorState) model.StateChanges = ruleDef.StateChanges + model.RuleDescription = make([]*RuleDescription, 0) model.Frequency = ruleDef.Frequency // frequency cannot be zero since that would not execute the alert rule. @@ -138,8 +142,12 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { } model.Notifications = nIds // model.AlertRuleTags = ruleDef.GetTagsFromSettings() - + alert, err := models.CommonAlertManager.GetAlert(ruleDef.Id) + if err != nil { + return nil, errors.Wrap(err, "GetCommonAlert error") + } for index, condition := range settings.Conditions { + alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(index, &settings.Conditions[index]) condType := condition.Type factory, exist := conditionFactories[condType] if !exist { @@ -149,6 +157,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { if err != nil { return nil, errors.Wrapf(err, "construct query condition %s", jsonutils.Marshal(condition)) } + newRuleDescription(model, alertDetails) model.Conditions = append(model.Conditions, queryCond) } @@ -158,6 +167,53 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { return model, nil } +func newRuleDescription(rule *Rule, alertDetails *monitor.CommonAlertMetricDetails) { + ruleDes := RuleDescription{ + AlertRecordRule: monitor.AlertRecordRule{ + Metric: fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field), + Measurement: alertDetails.Measurement, + MeasurementDesc: alertDetails.MeasurementDisplayName, + Field: alertDetails.Field, + FieldDesc: alertDetails.FieldDescription.DisplayName, + Comparator: alertDetails.Comparator, + Threshold: RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""), + }, + } + rule.RuleDescription = append(rule.RuleDescription, &ruleDes) +} + +var fileSize = []string{"bps", "Bps", "byte"} + +func RationalizeValueFromUnit(value float64, unit string, opt string) string { + if utils.IsInStringArray(unit, fileSize) { + if unit == "byte" { + return (FormatFileSize(value, unit, float64(1024))) + } + return FormatFileSize(value, unit, float64(1000)) + } + if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt { + return fmt.Sprintf("%0.4f%s", value*100, unit) + } + return fmt.Sprintf("%0.4f%s", value, unit) +} + +// 单位转换 保留四位小数 +func FormatFileSize(fileSize float64, unit string, unitsize float64) (size string) { + if fileSize < unitsize { + return fmt.Sprintf("%.4f%s", fileSize, unit) + } else if fileSize < (unitsize * unitsize) { + return fmt.Sprintf("%.4fK%s", float64(fileSize)/float64(unitsize), unit) + } else if fileSize < (unitsize * unitsize * unitsize) { + return fmt.Sprintf("%.4fM%s", float64(fileSize)/float64(unitsize*unitsize), unit) + } else if fileSize < (unitsize * unitsize * unitsize * unitsize) { + return fmt.Sprintf("%.4fG%s", float64(fileSize)/float64(unitsize*unitsize*unitsize), unit) + } else if fileSize < (unitsize * unitsize * unitsize * unitsize * unitsize) { + return fmt.Sprintf("%.4fT%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize), unit) + } else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024) + return fmt.Sprintf("%.4fE%s", float64(fileSize)/float64(unitsize*unitsize*unitsize*unitsize*unitsize), unit) + } +} + type AlertRuleTester struct{} func NewAlertRuleTester() models.AlertTestRunner {