diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index 89f802931c..89e91163ef 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -268,6 +268,9 @@ type meterFetchImp struct { func (m *meterFetchImp) FetchCustomizeEvalMatch(context *alerting.EvalContext, evalMatch *monitor.EvalMatch, alertDetails *monitor.CommonAlertMetricDetails) error { meterCustomizeConfig := new(monitor.MeterCustomizeConfig) + if context.Rule.CustomizeConfig == nil { + return nil + } err := context.Rule.CustomizeConfig.Unmarshal(meterCustomizeConfig) if err != nil { return err @@ -348,7 +351,7 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange * }) } - resp, err := c.HandleRequest(context.Ctx, ds.ToTSDBDataSource(""), req) + resp, err := c.HandleRequest(context.Ctx, ds.ToTSDBDataSource(c.Query.Model.Database), req) if err != nil { if err == gocontext.DeadlineExceeded { return nil, errors.Error("Alert execution exceeded the timeout") diff --git a/pkg/monitor/alerting/engine.go b/pkg/monitor/alerting/engine.go index 224a147ec7..622a629486 100644 --- a/pkg/monitor/alerting/engine.go +++ b/pkg/monitor/alerting/engine.go @@ -174,7 +174,6 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan evalContext := NewEvalContext(alertCtx, auth.AdminCredential(), job.Rule) evalContext.Ctx = alertCtx - go func() { defer func() { if err := recover(); err != nil { @@ -207,7 +206,8 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan */ if attemptID < options.Options.AlertingMaxAttempts { // span.Finish( - log.Debugf("Job Execution attempt triggered retry, timeMs: %v, alertId: %d", evalContext.GetDurationMs(), attemptID) + log.Warningf("Job Execution attempt triggered retry, timeMs: %v, alertId: %d", + evalContext.GetDurationMs(), attemptID) attemptChan <- (attemptID + 1) return } @@ -223,11 +223,14 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan // don't reuse the evalContext and get its own context. evalContext.Ctx = resultHandleCtx evalContext.Rule.State = evalContext.GetNewState() + if evalContext.Rule.Name == "cloudaccount_balance.balance" { + log.Errorf("cloudaccount_balance.balance newState:%s", string(evalContext.Rule.State)) + } if err := e.resultHandler.handle(evalContext); err != nil { if xerrors.Is(err, context.Canceled) { - log.Debugf("Result handler returned context.Canceled") + log.Warningf("Result handler returned context.Canceled") } else if xerrors.Is(err, context.DeadlineExceeded) { - log.Debugf("Result handler returned context.DeadlineExceeded") + log.Warningf("Result handler returned context.DeadlineExceeded") } else { log.Errorf("Failed to handle result: %v", err) } diff --git a/pkg/monitor/models/alertrecord.go b/pkg/monitor/models/alertrecord.go index 8f7da97ad2..8c173952cd 100644 --- a/pkg/monitor/models/alertrecord.go +++ b/pkg/monitor/models/alertrecord.go @@ -118,7 +118,11 @@ func (manager *SAlertRecordManager) ListItemFilter( } func (man *SAlertRecordManager) getAlertingRecordQuery() *sqlchemy.SQuery { - alertsQuery := CommonAlertManager.Query("id").Equals("state", monitor.AlertStateAlerting).IsTrue("enabled").IsNull("used_by").SubQuery() + q := CommonAlertManager.Query("id").IsTrue("enabled").IsNull("used_by") + q = q.Filter(sqlchemy.OR(sqlchemy.Equals(q.Field("state"), monitor.AlertStateAlerting), + sqlchemy.Equals(q.Field("state"), monitor.AlertStatePending))) + + alertsQuery := q.SubQuery() recordSub := man.Query().SubQuery() recordQuery := recordSub.Query(recordSub.Field("alert_id"), sqlchemy.MAX("max_created_at", recordSub.Field("created_at"))) @@ -222,6 +226,9 @@ func (man *SAlertRecordManager) ValidateCreateData(ctx context.Context, userCred func (record *SAlertRecord) GetEvalData() ([]monitor.EvalMatch, error) { ret := make([]monitor.EvalMatch, 0) + if record.EvalData == nil { + return ret, nil + } if err := record.EvalData.Unmarshal(&ret); err != nil { return nil, errors.Wrap(err, "unmarshal evalMatchs error") }