From 259ddccb047ce6818c35282f65ae9a59011ec00f Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 6 Nov 2024 18:52:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(monitor):=20=E4=BF=AE=E5=A4=8D=E6=8A=A5?= =?UTF-8?q?=E8=AD=A6=E6=81=A2=E5=A4=8D=E6=97=B6=E5=8C=85=E5=90=AB=E5=85=B6?= =?UTF-8?q?=E4=BB=96=E8=B5=84=E6=BA=90=E7=9A=84=E9=97=AE=E9=A2=98=20(#2154?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/apis/monitor/alert.go | 1 + pkg/apis/monitor/template.go | 8 +- pkg/apis/monitor/unifiedmonitor_query.go | 21 ++++ pkg/monitor/alerting/conditions/helper.go | 37 ++++++ .../alerting/conditions/nodataquery.go | 40 ++++--- pkg/monitor/alerting/conditions/query.go | 54 ++++----- pkg/monitor/alerting/eval_context.go | 52 +++++---- pkg/monitor/alerting/notifier.go | 2 +- pkg/monitor/alerting/notifiers/base.go | 4 + pkg/monitor/alerting/notifiers/dingding.go | 23 +++- pkg/monitor/alerting/notifiers/feishu.go | 8 +- pkg/monitor/alerting/notifiers/onecloud.go | 107 ++++++++++-------- pkg/monitor/models/commonalert.go | 15 +++ pkg/monitor/models/monitor_resource.go | 6 +- pkg/monitor/models/monitor_resource_alert.go | 9 ++ pkg/monitor/models/unifiedmonitor.go | 2 +- pkg/monitor/subscriptionmodel/subcription.go | 2 +- pkg/monitor/validators/validators.go | 2 +- 18 files changed, 261 insertions(+), 132 deletions(-) create mode 100644 pkg/monitor/alerting/conditions/helper.go diff --git a/pkg/apis/monitor/alert.go b/pkg/apis/monitor/alert.go index 6ab9a04cb9..16dc5b4071 100644 --- a/pkg/apis/monitor/alert.go +++ b/pkg/apis/monitor/alert.go @@ -198,6 +198,7 @@ type EvalMatch struct { Tags map[string]string `json:"tags"` Unit string `json:"unit"` AlertDetails jsonutils.JSONObject `json:"alert_details"` + IsRecovery bool `json:"is_recovery"` } type AlertTestRunOutput struct { diff --git a/pkg/apis/monitor/template.go b/pkg/apis/monitor/template.go index 5900f33af2..8f35f4cff1 100644 --- a/pkg/apis/monitor/template.go +++ b/pkg/apis/monitor/template.go @@ -19,10 +19,10 @@ type NotificationTemplateCreateInput struct { } type NotificationTemplateConfig struct { - Title string `json:"title"` - Name string `json:"name"` - ResourceName string `json:"resource_name"` - Matches []EvalMatch `json:"matches"` + Title string `json:"title"` + Name string `json:"name"` + ResourceName string `json:"resource_name"` + Matches []*EvalMatch `json:"matches"` // PrevAlertState AlertStateType `json:"prev_alert_state"` // State AlertStateType `json:"state"` NoDataFound bool `json:"no_data"` diff --git a/pkg/apis/monitor/unifiedmonitor_query.go b/pkg/apis/monitor/unifiedmonitor_query.go index 64fd91d992..4586bd1bd8 100644 --- a/pkg/apis/monitor/unifiedmonitor_query.go +++ b/pkg/apis/monitor/unifiedmonitor_query.go @@ -65,6 +65,27 @@ var ( } ) +func GetMeasurementTagIdKeyByResType(resType string) string { + return MEASUREMENT_TAG_ID[resType] +} + +func GetMeasurementTagIdKeyByResTypeWithDefault(resType string) string { + tagId := GetMeasurementTagIdKeyByResType(resType) + if len(tagId) == 0 { + tagId = "host_id" + } + return tagId +} + +func GetMeasurementResourceId(tags map[string]string, resType string) string { + return tags[GetMeasurementTagIdKeyByResType(resType)] +} + +func GetResourceIdFromTagWithDefault(tags map[string]string, resType string) string { + tagId := GetMeasurementTagIdKeyByResTypeWithDefault(resType) + return tags[tagId] +} + type MetricFunc struct { FieldOptType []string `json:"field_opt_type"` FieldOptValue map[string][]string `json:"field_opt_value"` diff --git a/pkg/monitor/alerting/conditions/helper.go b/pkg/monitor/alerting/conditions/helper.go new file mode 100644 index 0000000000..995c29044b --- /dev/null +++ b/pkg/monitor/alerting/conditions/helper.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package conditions + +import ( + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/monitor/models" +) + +func OkEvalMatchSetIsRecovery( + alert *models.SCommonAlert, + resId string, + em *monitor.EvalMatch, +) error { + isAlerting, err := alert.IsResourceMetricAlerting(resId, em.Metric) + if err != nil { + return errors.Wrap(err, "check if previous alert is alerting") + } + if isAlerting { + em.IsRecovery = true + } + return nil +} diff --git a/pkg/monitor/alerting/conditions/nodataquery.go b/pkg/monitor/alerting/conditions/nodataquery.go index 994f6027c1..8e7dacaefa 100644 --- a/pkg/monitor/alerting/conditions/nodataquery.go +++ b/pkg/monitor/alerting/conditions/nodataquery.go @@ -18,6 +18,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/apis/monitor" @@ -39,7 +40,7 @@ const ( ) func init() { - alerting.RegisterCondition("nodata_query", func(model *monitor.AlertCondition, index int) (alerting.Condition, + alerting.RegisterCondition(monitor.METRIC_QUERY_TYPE_NO_DATA, func(model *monitor.AlertCondition, index int) (alerting.Condition, error) { return newNoDataQueryCondition(model, index) }) @@ -60,12 +61,13 @@ func (c *NoDataQueryCondition) Eval(context *alerting.EvalContext) (*alerting.Co var matches []*monitor.EvalMatch var alertOkmatches []*monitor.EvalMatch normalHostIds := make(map[string]*monitor.EvalMatch, 0) + alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id) + if err != nil { + return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error") + } serLoop: for _, series := range seriesList { - tagId := monitor.MEASUREMENT_TAG_ID[context.Rule.RuleDescription[0].ResType] - if len(tagId) == 0 { - tagId = "host_id" - } + tagId := monitor.GetMeasurementTagIdKeyByResTypeWithDefault(context.Rule.RuleDescription[0].ResType) for key, val := range series.Tags { if key == tagId { if len(context.Rule.RuleDescription) == 0 { @@ -79,7 +81,7 @@ serLoop: } reducedValue, valStrArr := c.Reducer.Reduce(series) - match, err := c.NewEvalMatch(context, *series, nil, reducedValue, valStrArr, false) + match, err := c.NewEvalMatch(alert, context, *series, nil, reducedValue, valStrArr, false) if err != nil { return nil, errors.Wrap(err, "NoDataQueryCondition NewEvalMatch error") } @@ -94,15 +96,18 @@ serLoop: } for _, host := range allResources { id, _ := host.GetString("id") - evalMatch, err := c.NewNoDataEvalMatch(context, host) + evalMatch, err := c.NewNoDataEvalMatch(context, alert, host) if err != nil { - return nil, errors.Wrap(err, "NewNoDataEvalMatch error") + return nil, errors.Wrapf(err, "NewNoDataEvalMatch of host: %s", host.String()) } if normalMatch, ok := normalHostIds[id]; !ok { c.createEvalMatchTagFromHostJson(context, evalMatch, host) matches = append(matches, evalMatch) } else { c.createEvalMatchTagFromHostJson(context, normalMatch, host) + if err := OkEvalMatchSetIsRecovery(alert, id, normalMatch); err != nil { + log.Warningf("[NoDataQuery] set eval match %s to recovered: %v", jsonutils.Marshal(normalMatch), err) + } alertOkmatches = append(alertOkmatches, normalMatch) } } @@ -115,7 +120,7 @@ serLoop: }, nil } -func (c *NoDataQueryCondition) convertTagsQuery(evalContext *alerting.EvalContext, +/*func (c *NoDataQueryCondition) convertTagsQuery(evalContext *alerting.EvalContext, query *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { alertDetails, err := c.GetCommonAlertDetails(evalContext) if err != nil { @@ -136,14 +141,10 @@ func (c *NoDataQueryCondition) convertTagsQuery(evalContext *alerting.EvalContex filterCount++ } return query, nil -} +}*/ -func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext, host jsonutils.JSONObject) (*monitor.EvalMatch, error) { +func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext, alert *models.SCommonAlert, host jsonutils.JSONObject) (*monitor.EvalMatch, error) { evalMatch := new(monitor.EvalMatch) - alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id) - if err != nil { - return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error") - } settings, _ := alert.GetSettings() alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(c.Index, &settings.Conditions[c.Index]) evalMatch.Metric = fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field) @@ -164,8 +165,11 @@ func (c *NoDataQueryCondition) NewNoDataEvalMatch(context *alerting.EvalContext, return evalMatch, nil } -func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalContext *alerting.EvalContext, evalMatch *monitor.EvalMatch, - host jsonutils.JSONObject) { +func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson( + evalContext *alerting.EvalContext, + evalMatch *monitor.EvalMatch, + host jsonutils.JSONObject, +) { evalMatch.Tags = make(map[string]string, 0) ip, _ := host.GetString(HOST_TAG_IP) @@ -186,7 +190,7 @@ func (c *NoDataQueryCondition) createEvalMatchTagFromHostJson(evalContext *alert case monitor.METRIC_RES_TYPE_OSS: default: evalMatch.Tags["host"] = name - evalMatch.Tags[monitor.MEASUREMENT_TAG_ID[monitor.METRIC_RES_TYPE_HOST]] = id + evalMatch.Tags[monitor.GetMeasurementTagIdKeyByResType(monitor.METRIC_RES_TYPE_HOST)] = id evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_RES_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_RES_TYPE evalMatch.Tags[hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE] = hostconsts.TELEGRAF_TAG_ONECLOUD_HOST_TYPE_HOST } diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index ddae55920d..fad4056186 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -159,10 +159,11 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio var matches []*monitor.EvalMatch var alertOkmatches []*monitor.EvalMatch - //allResources, err := c.GetQueryResources() - //if err != nil { - // return nil, errors.Wrap(err, "GetQueryResources err") - //} + alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id) + if err != nil { + return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error") + } + for _, series := range seriesList { if len(c.ResType) != 0 { isLatestOfSerie, resource := c.serieIsLatestResource(nil, series) @@ -193,17 +194,21 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio meta = &metas[0] } if evalMatch { - match, err := c.NewEvalMatch(context, *series, meta, reducedValue, valStrArr, evalMatch) + match, err := c.NewEvalMatch(alert, context, *series, meta, reducedValue, valStrArr, evalMatch) if err != nil { return nil, errors.Wrap(err, "NewEvalMatch error") } matches = append(matches, match) } if reducedValue != nil && !evalMatch { - match, err := c.NewEvalMatch(context, *series, meta, reducedValue, valStrArr, evalMatch) + match, err := c.NewEvalMatch(alert, context, *series, meta, reducedValue, valStrArr, evalMatch) if err != nil { return nil, errors.Wrap(err, "NewEvalMatch error") } + resId := monitor.GetResourceIdFromTagWithDefault(match.Tags, c.ResType) + if err := OkEvalMatchSetIsRecovery(alert, resId, match); err != nil { + log.Warningf("[Query] set eval match %s to recovered: %v", jsonutils.Marshal(match), err) + } alertOkmatches = append(alertOkmatches, match) } } @@ -241,12 +246,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio } func (c *QueryCondition) serieIsLatestResource(resources map[string]jsonutils.JSONObject, series *monitor.TimeSeries) (bool, jsonutils.JSONObject) { - tagId := monitor.MEASUREMENT_TAG_ID[c.ResType] - if len(tagId) == 0 { - tagId = "host_id" - } - resId := series.Tags[tagId] - log.Debugf("serieIsLatestResource use tagId %q, found resId %q", tagId, resId) + resId := monitor.GetResourceIdFromTagWithDefault(series.Tags, c.ResType) if len(resources) != 0 { resource, ok := resources[resId] if !ok { @@ -279,9 +279,14 @@ func (c *QueryCondition) FillSerieByResourceField(resource jsonutils.JSONObject, } } -func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series monitor.TimeSeries, meta *monitor.QueryResultMeta, value *float64, valStrArr []string, isMatch bool) (*monitor.EvalMatch, error) { +func (c *QueryCondition) NewEvalMatch( + alert *models.SCommonAlert, + context *alerting.EvalContext, + series monitor.TimeSeries, + meta *monitor.QueryResultMeta, + value *float64, valStrArr []string, isMatch bool) (*monitor.EvalMatch, error) { evalMatch := new(monitor.EvalMatch) - alertDetails, err := c.GetCommonAlertDetails(context) + alertDetails, err := c.GetCommonAlertDetails(alert) if err != nil { return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error") } @@ -353,11 +358,7 @@ func (m *meterFetchImp) FetchCustomizeEvalMatch(context *alerting.EvalContext, e return nil } -func (c *QueryCondition) GetCommonAlertDetails(context *alerting.EvalContext) (*monitor.CommonAlertMetricDetails, error) { - alert, err := models.CommonAlertManager.GetAlert(context.Rule.Id) - if err != nil { - return nil, errors.Wrap(err, "GetAlert to NewEvalMatch error") - } +func (c *QueryCondition) GetCommonAlertDetails(alert *models.SCommonAlert) (*monitor.CommonAlertMetricDetails, error) { settings, _ := alert.GetSettings() alertDetails := alert.GetCommonAlertMetricDetailsFromAlertCondition(c.Index, &settings.Conditions[c.Index]) return alertDetails, nil @@ -530,21 +531,20 @@ func (c *QueryCondition) checkGroupByField() { } for i, group := range c.Query.Model.GroupBy { if group.Params[0] == "*" { - c.Query.Model.GroupBy[i].Params = []string{monitor.MEASUREMENT_TAG_ID[metricMeasurement.ResType]} + c.Query.Model.GroupBy[i].Params = []string{monitor.GetMeasurementTagIdKeyByResType(metricMeasurement.ResType)} } } } func (c *QueryCondition) setResType() { + metricMeasurement, _ := models.MetricMeasurementManager.GetCache().Get(c.Query.Model.Measurement) + if metricMeasurement != nil { + c.ResType = metricMeasurement.ResType + } var resType = monitor.METRIC_RES_TYPE_HOST if len(c.Query.Model.GroupBy) == 0 { return } - metricMeasurement, _ := models.MetricMeasurementManager.GetCache().Get(c.Query.Model.Measurement) - if metricMeasurement != nil { - resType = metricMeasurement.ResType - c.ResType = resType - } // NOTE: shouldn't set ResType when tenant_id and domain_id within GroupBy /* if len(resType) != 0 && c.Query.Model.GroupBy[0].Params[0] != monitor. MEASUREMENT_TAG_ID[resType] { @@ -576,7 +576,7 @@ func (c *QueryCondition) GetQueryResources(s *mcclient.ClientSession, scope stri func (c *QueryCondition) getOnecloudResources(s *mcclient.ClientSession, scope string, showDetails bool) ([]jsonutils.JSONObject, error) { query := jsonutils.NewDict() - query.Add(jsonutils.NewStringArray([]string{"running", "ready"}), "status") + queryStatus := []string{"running", "ready"} // query.Add(jsonutils.NewString("true"), "admin") //if len(c.Query.Model.Tags) != 0 { // query, err = c.convertTagsQuery(evalContext, query) @@ -592,6 +592,7 @@ func (c *QueryCondition) getOnecloudResources(s *mcclient.ClientSession, scope s switch c.ResType { case monitor.METRIC_RES_TYPE_HOST: query.Set("host_type", jsonutils.NewString(hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR)) + queryStatus = append(queryStatus, "unknown") query.Set("enabled", jsonutils.NewInt(1)) manager = &mc_mds.Hosts case monitor.METRIC_RES_TYPE_GUEST: @@ -622,6 +623,7 @@ func (c *QueryCondition) getOnecloudResources(s *mcclient.ClientSession, scope s manager = &mc_mds.Hosts } + query.Add(jsonutils.NewStringArray(queryStatus), "status") allResources, err = ListAllResources(s, manager, query, scope, showDetails) if err != nil { return nil, errors.Wrapf(err, "ListAllResources for %s with query %s, scope: %s, showDetails: %v", manager.GetKeyword(), query, scope, showDetails) diff --git a/pkg/monitor/alerting/eval_context.go b/pkg/monitor/alerting/eval_context.go index 0580c385a8..f499e1902c 100644 --- a/pkg/monitor/alerting/eval_context.go +++ b/pkg/monitor/alerting/eval_context.go @@ -54,12 +54,13 @@ type EvalContext struct { // NewEvalContext is the EvalContext constructor. func NewEvalContext(alertCtx context.Context, userCred mcclient.TokenCredential, rule *Rule) *EvalContext { return &EvalContext{ - Ctx: alertCtx, - UserCred: userCred, - StartTime: time.Now(), - Rule: rule, - EvalMatches: make([]*monitor.EvalMatch, 0), - PrevAlertState: rule.State, + Ctx: alertCtx, + UserCred: userCred, + StartTime: time.Now(), + Rule: rule, + EvalMatches: make([]*monitor.EvalMatch, 0), + AlertOkEvalMatches: make([]*monitor.EvalMatch, 0), + PrevAlertState: rule.State, } } @@ -192,7 +193,7 @@ func getNewStateInternal(c *EvalContext) monitor.AlertStateType { return monitor.AlertStateOK } -func (c *EvalContext) GetNotificationTemplateConfig() monitor.NotificationTemplateConfig { +func (c *EvalContext) GetNotificationTemplateConfig(matches []*monitor.EvalMatch) monitor.NotificationTemplateConfig { desc := c.Rule.Message if len(c.Rule.TriggeredMessages) > 0 { desc = strings.Join(c.Rule.TriggeredMessages, " ") @@ -207,14 +208,15 @@ func (c *EvalContext) GetNotificationTemplateConfig() monitor.NotificationTempla return monitor.NotificationTemplateConfig{ Title: c.GetNotificationTitle(), Name: c.Rule.Name, - ResourceName: c.GetResourceNameOfMathes(nil), - Matches: c.GetEvalMatches(), - StartTime: c.StartTime.In(tz).Format("2006-01-02 15:04:05"), - EndTime: c.EndTime.In(tz).Format("2006-01-02 15:04:05"), - Description: desc, - Level: c.Rule.Level, - NoDataFound: c.NoDataFound, - WebUrl: c.GetCallbackURLPrefix(), + ResourceName: c.GetResourceNameOfMatches(matches), + Matches: matches, + //Matches: c.GetEvalMatches(), + StartTime: c.StartTime.In(tz).Format("2006-01-02 15:04:05"), + EndTime: c.EndTime.In(tz).Format("2006-01-02 15:04:05"), + Description: desc, + Level: c.Rule.Level, + NoDataFound: c.NoDataFound, + WebUrl: c.GetCallbackURLPrefix(), } } @@ -239,11 +241,8 @@ func (c *EvalContext) GetEvalMatches() []monitor.EvalMatch { return ret } -func (c *EvalContext) GetResourceNameOfMathes(matches []monitor.EvalMatch) string { +func (c *EvalContext) GetResourceNameOfMatches(matches []*monitor.EvalMatch) string { names := strings.Builder{} - if matches == nil { - matches = c.GetEvalMatches() - } for i, match := range matches { if name, ok := match.Tags["name"]; ok { names.WriteString(fmt.Sprintf("%s.%s(%s)", name, match.Metric, match.ValueStr)) @@ -254,3 +253,18 @@ func (c *EvalContext) GetResourceNameOfMathes(matches []monitor.EvalMatch) strin } return names.String() } + +func (c *EvalContext) GetRecoveredMatches() []*monitor.EvalMatch { + ret := make([]*monitor.EvalMatch, 0) + for i := range c.AlertOkEvalMatches { + m := c.AlertOkEvalMatches[i] + if m.IsRecovery { + ret = append(ret, m) + } + } + return ret +} + +func (c *EvalContext) HasRecoveredMatches() bool { + return len(c.GetRecoveredMatches()) != 0 +} diff --git a/pkg/monitor/alerting/notifier.go b/pkg/monitor/alerting/notifier.go index db97702f5b..d821138146 100644 --- a/pkg/monitor/alerting/notifier.go +++ b/pkg/monitor/alerting/notifier.go @@ -224,7 +224,7 @@ func (n *notificationService) dealNeedShieldEvalMatches(evalCtx *EvalContext, ma } filterMatch: for i := range match { - input.ResId = match[i].Tags[monitor.MEASUREMENT_TAG_ID[input.ResType]] + input.ResId = monitor.GetMeasurementResourceId(match[i].Tags, input.ResType) alertRecordShields, err := models.AlertRecordShieldManager.GetRecordShields(input) if err != nil { log.Errorf("GetRecordShields byAlertId:%s,err:%v", input.AlertId, err) diff --git a/pkg/monitor/alerting/notifiers/base.go b/pkg/monitor/alerting/notifiers/base.go index 00c3b43e18..2f8b3caf8a 100644 --- a/pkg/monitor/alerting/notifiers/base.go +++ b/pkg/monitor/alerting/notifiers/base.go @@ -56,6 +56,10 @@ func (n *NotifierBase) ShouldNotify(_ context.Context, evalCtx *alerting.EvalCon prevState := evalCtx.PrevAlertState newState := evalCtx.Rule.State + if evalCtx.HasRecoveredMatches() { + return true + } + // Do not notify if alert state is no_data if newState == monitor.AlertStateNoData { return false diff --git a/pkg/monitor/alerting/notifiers/dingding.go b/pkg/monitor/alerting/notifiers/dingding.go index 507506aced..a4d176267d 100644 --- a/pkg/monitor/alerting/notifiers/dingding.go +++ b/pkg/monitor/alerting/notifiers/dingding.go @@ -86,11 +86,26 @@ func newDingdingNotifier(config alerting.NotificationConfig) (alerting.Notifier, }, nil } -func (dd *DingDingNotifier) Notify(ctx *alerting.EvalContext, _ jsonutils.JSONObject) error { +func (dd *DingDingNotifier) Notify(evalCtx *alerting.EvalContext, d jsonutils.JSONObject) error { log.Infof("Sending alert notification to dingding") + errs := []error{} + if len(evalCtx.EvalMatches) > 0 { + if err := dd.notify(evalCtx, evalCtx.EvalMatches, false, d); err != nil { + errs = append(errs, errors.Wrap(err, "notify alerting matches")) + } + } + if evalCtx.HasRecoveredMatches() { + if err := dd.notify(evalCtx, evalCtx.GetRecoveredMatches(), true, d); err != nil { + errs = append(errs, errors.Wrap(err, "notify recovered matches")) + } + } + return errors.NewAggregate(errs) +} + +func (dd *DingDingNotifier) notify(ctx *alerting.EvalContext, matches []*monitor.EvalMatch, isRecoverd bool, _ jsonutils.JSONObject) error { // msgUrl, err := ctx.GetRuleURL() - body, err := dd.genBody(ctx) + body, err := dd.genBody(ctx, matches, isRecoverd) if err != nil { return err } @@ -101,7 +116,7 @@ func (dd *DingDingNotifier) Notify(ctx *alerting.EvalContext, _ jsonutils.JSONOb return SendWebRequestSync(ctx.Ctx, input) } -func (dd *DingDingNotifier) genBody(ctx *alerting.EvalContext) ([]byte, error) { +func (dd *DingDingNotifier) genBody(ctx *alerting.EvalContext, matches []*monitor.EvalMatch, isRecoverd bool) ([]byte, error) { q := url.Values{ "pc_slide": {"false"}, // "url": {messageURL}, @@ -113,7 +128,7 @@ func (dd *DingDingNotifier) genBody(ctx *alerting.EvalContext) ([]byte, error) { log.Infof("messageUrl: " + messageURL) - config := GetNotifyTemplateConfig(ctx) + config := GetNotifyTemplateConfig(ctx, isRecoverd, matches) contentConfig := templates.NewTemplateConfig(config) content, err := contentConfig.GenerateMarkdown() if err != nil { diff --git a/pkg/monitor/alerting/notifiers/feishu.go b/pkg/monitor/alerting/notifiers/feishu.go index 32ec071366..dead630e66 100644 --- a/pkg/monitor/alerting/notifiers/feishu.go +++ b/pkg/monitor/alerting/notifiers/feishu.go @@ -156,15 +156,15 @@ func (fs *FeishuNotifier) getMetricsMod(config monitor.NotificationTemplateConfi inElems := make([]*feishu.CardElement, 0) for idx, m := range config.Matches { hrE := feishu.NewCardElementHR() - mE := fs.getMetricElem(idx+1, m) - mTE := fs.getMetricTagElem(m) + mE := fs.getMetricElem(idx+1, *m) + mTE := fs.getMetricTagElem(*m) inElems = append(inElems, hrE, mE, mTE) } return inElems } func (fs *FeishuNotifier) genCard(ctx *alerting.EvalContext, chatId string) (*feishu.MsgReq, error) { - config := GetNotifyTemplateConfig(ctx) + config := GetNotifyTemplateConfig(ctx, false, ctx.EvalMatches) commonElem := fs.getCommonInfoMod(config) msElems := fs.getMetricsMod(config) @@ -193,7 +193,7 @@ func (fs *FeishuNotifier) genCard(ctx *alerting.EvalContext, chatId string) (*fe } func (fs *FeishuNotifier) genMsg(ctx *alerting.EvalContext, chatId string) (*feishu.MsgReq, error) { - config := GetNotifyTemplateConfig(ctx) + config := GetNotifyTemplateConfig(ctx, false, ctx.EvalMatches) // 富文本: https://open.feishu.cn/document/ukTMukTMukTM/uMDMxEjLzATMx4yMwETM return &feishu.MsgReq{ ChatId: chatId, diff --git a/pkg/monitor/alerting/notifiers/onecloud.go b/pkg/monitor/alerting/notifiers/onecloud.go index baaee4490e..0386434dd0 100644 --- a/pkg/monitor/alerting/notifiers/onecloud.go +++ b/pkg/monitor/alerting/notifiers/onecloud.go @@ -115,63 +115,60 @@ func getAdminSession() *mcclient.ClientSession { return auth.GetAdminSession(context.Background(), options.Options.Region) } -func GetNotifyTemplateConfig(ctx *alerting.EvalContext) monitor.NotificationTemplateConfig { - priority := notify.NotifyPriorityNormal - level := "普通" - switch ctx.Rule.Level { - case "", "normal": - priority = notify.NotifyPriorityNormal - case "important": - priority = notify.NotifyPriorityImportant - level = "重要" - case "fatal", "critical": - priority = notify.NotifyPriorityCritical - level = "致命" - } - topic := fmt.Sprintf("[%s]", level) - - isRecovery := false - if ctx.Rule.State == monitor.AlertStateOK { - isRecovery = true - topic = fmt.Sprintf("%s %s 告警已恢复", topic, ctx.GetRuleTitle()) - } else if ctx.NoDataFound { - topic = fmt.Sprintf("%s %s 暂无数据", topic, ctx.GetRuleTitle()) - } else { - topic = fmt.Sprintf("%s %s 发生告警", topic, ctx.GetRuleTitle()) - } - config := ctx.GetNotificationTemplateConfig() - config.Title = topic - config.Level = level - config.Priority = string(priority) - config.IsRecovery = isRecovery - return config +func GetNotifyTemplateConfig(ctx *alerting.EvalContext, isRecoverd bool, matches []*monitor.EvalMatch) monitor.NotificationTemplateConfig { + return getNotifyTemplateConfigOfLang(ctx, matches, isRecoverd, language.Chinese) } -func GetNotifyTemplateConfigOfEN(ctx *alerting.EvalContext) monitor.NotificationTemplateConfig { +func getNotifyTemplateConfigOfLang(ctx *alerting.EvalContext, + matches []*monitor.EvalMatch, isRecovered bool, lang language.Tag) monitor.NotificationTemplateConfig { priority := notify.NotifyPriorityNormal - level := "Normal" + levelNormal := "普通" + levelImportant := "重要" + levelCritial := "致命" + msgRecovered := "告警已恢复" + msgNoData := "暂无数据" + msgAlerting := "发生告警" + + transMap := map[string]string{ + levelNormal: "Normal", + levelImportant: "Important", + levelCritial: "Critical", + msgRecovered: "Alarm recovered", + msgNoData: "No data available", + msgAlerting: "Alerting", + } + trans := func(input string) string { + if lang == language.English { + return transMap[input] + } + return input + } + + level := levelNormal switch ctx.Rule.Level { case "", "normal": priority = notify.NotifyPriorityNormal case "important": priority = notify.NotifyPriorityImportant - level = "Important" + level = levelNormal case "fatal", "critical": priority = notify.NotifyPriorityCritical - level = "Critical" + level = levelNormal } - topic := fmt.Sprintf("[%s]", level) + topic := fmt.Sprintf("[%s]", trans(level)) isRecovery := false - if ctx.Rule.State == monitor.AlertStateOK { + var hintMsg string + if ctx.Rule.State == monitor.AlertStateOK || isRecovered { isRecovery = true - topic = fmt.Sprintf("%s %s Alarm recovered", topic, ctx.GetRuleTitle()) + hintMsg = msgRecovered } else if ctx.NoDataFound { - topic = fmt.Sprintf("%s %s No data available", topic, ctx.GetRuleTitle()) + hintMsg = msgNoData } else { - topic = fmt.Sprintf("%s %s Alarm", topic, ctx.GetRuleTitle()) + hintMsg = msgAlerting } - config := ctx.GetNotificationTemplateConfig() + topic = fmt.Sprintf("%s %s %s", topic, ctx.GetRuleTitle(), trans(hintMsg)) + config := ctx.GetNotificationTemplateConfig(matches) config.Title = topic config.Level = level config.Priority = string(priority) @@ -249,14 +246,28 @@ func getLangBystr(str string) language.Tag { } func (oc *OneCloudNotifier) notifyByContextLang(ctx context.Context, evalCtx *alerting.EvalContext, uids []string) error { - var config monitor.NotificationTemplateConfig - lang := appctx.Lang(ctx) - switch lang { - case language.English: - config = GetNotifyTemplateConfigOfEN(evalCtx) - default: - config = GetNotifyTemplateConfig(evalCtx) + errs := []error{} + if len(evalCtx.EvalMatches) > 0 { + if err := oc.notifyMatchesByContextLang(ctx, evalCtx, evalCtx.EvalMatches, uids, false); err != nil { + errs = append(errs, errors.Wrapf(err, "notify alerting matches")) + } } + if evalCtx.HasRecoveredMatches() { + if err := oc.notifyMatchesByContextLang(ctx, evalCtx, evalCtx.GetRecoveredMatches(), uids, true); err != nil { + errs = append(errs, errors.Wrapf(err, "notify recovered matches")) + } + } + return errors.NewAggregate(errs) +} + +func (oc *OneCloudNotifier) notifyMatchesByContextLang( + ctx context.Context, + evalCtx *alerting.EvalContext, + matches []*monitor.EvalMatch, + uids []string, + isRecoverd bool) error { + lang := appctx.Lang(ctx) + config := getNotifyTemplateConfigOfLang(evalCtx, matches, isRecoverd, lang) oc.filterMatchTagsForConfig(&config, ctx) contentConfig := oc.buildContent(config) @@ -516,7 +527,7 @@ func SendNotifyInfo(base *sendnotifyBase, imp Isendnotify) error { split = len(tmpMatches) } base.config.Matches = tmpMatches[i:split] - base.config.ResourceName = base.evalCtx.GetResourceNameOfMathes(base.config.Matches) + base.config.ResourceName = base.evalCtx.GetResourceNameOfMatches(base.config.Matches) err := imp.execNotifyFunc() if err != nil { return err diff --git a/pkg/monitor/models/commonalert.go b/pkg/monitor/models/commonalert.go index 2db599b5c8..64a165453e 100644 --- a/pkg/monitor/models/commonalert.go +++ b/pkg/monitor/models/commonalert.go @@ -1533,6 +1533,21 @@ func (alert *SCommonAlert) GetAlertRule(settings *monitor.AlertSetting, index in return rule } +func (alert *SCommonAlert) GetResourceAlert(resourceId string, metric string) (*SMonitorResourceAlert, error) { + return MonitorResourceAlertManager.GetResourceAlert(alert.GetId(), resourceId, metric) +} + +func (alert *SCommonAlert) IsResourceMetricAlerting(resourceId string, metric string) (bool, error) { + ra, err := alert.GetResourceAlert(resourceId, metric) + if err != nil { + return false, errors.Wrapf(err, "GetResourceAlert") + } + if ra.AlertState == string(monitor.AlertStateAlerting) { + return true, nil + } + return false, nil +} + var fileSize = []string{"bps", "Bps", "byte"} func RationalizeValueFromUnit(value float64, unit string, opt string) string { diff --git a/pkg/monitor/models/monitor_resource.go b/pkg/monitor/models/monitor_resource.go index 468c8a75cc..b73e8cf34a 100644 --- a/pkg/monitor/models/monitor_resource.go +++ b/pkg/monitor/models/monitor_resource.go @@ -409,10 +409,6 @@ type UpdateMonitorResourceAlertInput struct { } func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context, userCred mcclient.TokenCredential, input *UpdateMonitorResourceAlertInput) error { - //if !utils.IsInStringArray(alertRecord.ResType, []string{monitor.METRIC_RES_TYPE_HOST, - // monitor.METRIC_RES_TYPE_GUEST, monitor.METRIC_RES_TYPE_AGENT}) { - // return nil - //} resType := input.ResType if resType == monitor.METRIC_RES_TYPE_AGENT { resType = monitor.METRIC_RES_TYPE_GUEST @@ -421,7 +417,7 @@ func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx con errs := make([]error, 0) matchResourceIds := make([]string, 0) for _, match := range matches { - resId := match.Tags[monitor.MEASUREMENT_TAG_ID[input.ResType]] + resId := monitor.GetMeasurementResourceId(match.Tags, input.ResType) if len(resId) == 0 { continue } diff --git a/pkg/monitor/models/monitor_resource_alert.go b/pkg/monitor/models/monitor_resource_alert.go index 25f269f739..bea3444f5e 100644 --- a/pkg/monitor/models/monitor_resource_alert.go +++ b/pkg/monitor/models/monitor_resource_alert.go @@ -339,3 +339,12 @@ func (manager *SMonitorResourceAlertManager) ListItemExportKeys(ctx context.Cont func (m *SMonitorResourceAlertManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man db.FilterByOwnerProvider, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { return q } + +func (m *SMonitorResourceAlertManager) GetResourceAlert(alertId string, resourceId string, metric string) (*SMonitorResourceAlert, error) { + q := m.Query().Equals("alert_id", alertId).Equals("monitor_resource_id", resourceId).Equals("metric", metric) + obj := new(SMonitorResourceAlert) + if err := q.First(obj); err != nil { + return nil, errors.Wrapf(err, "query resource alert by alert_id: %q, resource_id: %q, metric: %q", alertId, resourceId, metric) + } + return obj, nil +} diff --git a/pkg/monitor/models/unifiedmonitor.go b/pkg/monitor/models/unifiedmonitor.go index db71a0a561..bb1231cde6 100644 --- a/pkg/monitor/models/unifiedmonitor.go +++ b/pkg/monitor/models/unifiedmonitor.go @@ -493,7 +493,7 @@ func checkQueryGroupBy(query *monitor.AlertQuery, inputQuery *monitor.MetricQuer } tagId := "" if metricMeasurement != nil { - tagId = monitor.MEASUREMENT_TAG_ID[metricMeasurement.ResType] + tagId = monitor.GetMeasurementTagIdKeyByResType(metricMeasurement.ResType) } if len(tagId) == 0 || (len(inputQuery.Slimit) != 0 && len(inputQuery.Soffset) != 0) { tagId = "*" diff --git a/pkg/monitor/subscriptionmodel/subcription.go b/pkg/monitor/subscriptionmodel/subcription.go index f1cc922794..90ad1c2a15 100644 --- a/pkg/monitor/subscriptionmodel/subcription.go +++ b/pkg/monitor/subscriptionmodel/subcription.go @@ -419,7 +419,7 @@ func (n *SSubscriptionManager) doNotify(nIds []string, evalCtx *alerting.EvalCon } func (self *SSubscriptionManager) notifyBySysConfig(evalContext alerting.EvalContext) error { - config := notifiers.GetNotifyTemplateConfig(&evalContext) + config := notifiers.GetNotifyTemplateConfig(&evalContext, false, evalContext.EvalMatches) contentConfig := templates.NewTemplateConfig(config) content, err := contentConfig.GenerateMarkdown() if err != nil { diff --git a/pkg/monitor/validators/validators.go b/pkg/monitor/validators/validators.go index 4c943b1edb..92e45afa90 100644 --- a/pkg/monitor/validators/validators.go +++ b/pkg/monitor/validators/validators.go @@ -47,7 +47,7 @@ var ( CommonAlertReducerFieldOpts = []string{"/"} CommonAlertNotifyTypes = []string{"email", "mobile", "dingtalk", "webconsole", "feishu"} - ConditionTypes = []string{"query", "nodata_query"} + ConditionTypes = []string{"query", monitor.METRIC_QUERY_TYPE_NO_DATA} ) func ValidateAlertCreateInput(input monitor.AlertCreateInput) error {