fix(monitor): fix monitor slow query

1. 使用apihelper 增量查询监控数据
2. 通过apihelper中set进行对应的资源过滤

cmd/climc/shell/monitor/alertrecordshield.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/monitor_resourece_alert.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/apihelper/apihelper.go

@@ -101,7 +101,7 @@ func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) {
 
 	s := h.adminClientSession(ctx)
 	mss := h.modelSets.Copy()
-	r, err := SyncModelSets(mss, s, h.opts.ListBatchSize)
+	r, err := SyncModelSets(mss, s, h.opts)
 	if err != nil {
 		return false, err
 	}

pkg/apihelper/interface.go

@@ -44,7 +44,7 @@ type IModelSetEmulatedIncluder interface {
 	IncludeEmulated() bool
 }
 
-func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int) (r ModelSetsUpdateResult, err error) {
+func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) {
 	mss := mssOld.ModelSetList()
 	mssNews := mssOld.NewEmpty()
 	for i, msNew := range mssNews.ModelSetList() {
@@ -60,10 +60,11 @@ func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int)
 			ModelManager:  msNew.ModelManager(),
 			MinUpdatedAt:  minUpdatedAt,
 			ModelSet:      msNew,
-			BatchListSize: batchSize,
+			BatchListSize: opt.ListBatchSize,
 
-			IncludeDetails:  false,
-			IncludeEmulated: includeEmulated,
+			IncludeDetails:       opt.IncludeDetails,
+			IncludeEmulated:      includeEmulated,
+			InCludeOtherCloudEnv: opt.InCludeOtherCloudEnv,
 		})
 		if err != nil {
 			return

pkg/apihelper/options.go

@@ -21,6 +21,8 @@ import (
 type Options struct {
 	common_options.CommonOptions
 
-	SyncInterval  int
-	ListBatchSize int
+	SyncInterval         int
+	ListBatchSize        int
+	IncludeDetails       bool
+	InCludeOtherCloudEnv bool
 }

pkg/apihelper/reflect.go

@@ -38,10 +38,11 @@ type GetModelsOptions struct {
 	ModelManager  mcclient_modulebase.IBaseManager
 	ModelSet      IModelSet
 
-	BatchListSize   int
-	MinUpdatedAt    time.Time
-	IncludeDetails  bool
-	IncludeEmulated bool
+	BatchListSize        int
+	MinUpdatedAt         time.Time
+	IncludeDetails       bool
+	IncludeEmulated      bool
+	InCludeOtherCloudEnv bool
 }
 
 func GetModels(opts *GetModelsOptions) error {
@@ -106,15 +107,19 @@ func GetModels(opts *GetModelsOptions) error {
 		ShowEmulated: options.Bool(opts.IncludeEmulated),
 		Filter: []string{
 			minUpdatedAtFilter(minUpdatedAt), // order matters, filter.0
-			"manager_id.isnullorempty()",     // len(manager_id) > 0 is for pubcloud objects
-			"external_id.isnullorempty()",    // len(external_id) > 0 is for pubcloud objects
-			"cloud_env=onpremise",
 		},
 		OrderBy: []string{"updated_at"},
 		Order:   "asc",
 		Limit:   options.Int(opts.BatchListSize),
 		Offset:  options.Int(0),
 	}
+	if !opts.InCludeOtherCloudEnv {
+		listOptions.Filter = append(listOptions.Filter,
+			"manager_id.isnullorempty()",  // len(manager_id) > 0 is for pubcloud objects
+			"external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects
+			"cloud_env=onpremise",
+		)
+	}
 	if !minUpdatedAt.Equal(PseudoZeroTime) {
 		// Only fetching pending deletes when we are doing incremental fetch
 		listOptions.PendingDeleteAll = options.Bool(true)

pkg/apis/monitor/alert_record_shield.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/apis/monitor/alert_resource_record.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/apis/monitor/alertquery_const.go

@@ -15,6 +15,28 @@
 package monitor
 
 var (
+	UnionTags = []string{
+		"host",
+		"host_id",
+		"id",
+		"ips",
+		"name",
+		"zone",
+		"zone_id",
+		"zone_ext_id",
+		"os_type",
+		"status",
+		"cloudregion",
+		"cloudregion_id",
+		"region_ext_id",
+		"tenant",
+		"tenant_id",
+		"brand",
+		"vm_scaling_group_id",
+		"domain_id",
+		"project_domain",
+	}
+
 	ServerTags = map[string]string{
 		"host":             "host",
 		"host_id":          "host_id",

pkg/apis/monitor/unifiedmonitor_const.go

@@ -69,9 +69,11 @@ type MetricFunc struct {
 }
 
 type MetricInputQuery struct {
-	From  string `json:"from"`
-	To    string `json:"to"`
-	Scope string `json:"scope"`
+	From    string `json:"from"`
+	To      string `json:"to"`
+	Scope   string `json:"scope"`
+	Slimit  string `json:"slimit"`
+	Soffset string `json:"soffset"`
 	//default group by
 	Unit        bool          `json:"unit"`
 	Interval    string        `json:"interval"`

pkg/cloudmon/collectors/alertrecordhistorymon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package alertrecordhistorymon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon"

pkg/cloudmon/collectors/alimon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package alimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon"

pkg/cloudmon/collectors/apsaramon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package apsaramon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"

pkg/cloudmon/collectors/awsmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package awsmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon"

pkg/cloudmon/collectors/azuremon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package azuremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon"

pkg/cloudmon/collectors/cloudaccountmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cloudaccountmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon"

pkg/cloudmon/collectors/common/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package common // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common"

pkg/cloudmon/collectors/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package collectors // import "yunion.io/x/onecloud/pkg/cloudmon/collectors"

pkg/cloudmon/collectors/ecloudmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package ecloudmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/ecloudmon"

pkg/cloudmon/collectors/gcpmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package gcpmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon"

pkg/cloudmon/collectors/huaweimon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package huaweimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon"

pkg/cloudmon/collectors/jdmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package jdmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/jdmon"

pkg/cloudmon/collectors/qcmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package qcmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon"

pkg/cloudmon/collectors/storagemon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package storagemon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon"

pkg/cloudmon/collectors/vmwaremon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package vmwaremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon"

pkg/cloudmon/collectors/zstackmon/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package zstackmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon"

pkg/cloudmon/options/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package options // import "yunion.io/x/onecloud/pkg/cloudmon/options"

pkg/compute/regiondrivers/huaweistack.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package regiondrivers
 
 import (

pkg/mcclient/options/identity/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package identity // import "yunion.io/x/onecloud/pkg/mcclient/options/identity"

pkg/mcclient/options/meter/cost_conversions.go

@@ -0,0 +1,72 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package meter
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type CostConversionListOptions struct {
+	options.BaseListOptions
+}
+
+func (opt *CostConversionListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(opt)
+}
+
+type CostConversionCreateOptions struct {
+	Name string
+
+	IsPublicCloud   string `help:"public cloud filter of cost conversion" json:"is_public_cloud"`
+	Brand           string `help:"brand filter of cost conversion" json:"brand"`
+	CloudaccountId  string `help:"cloudaccount filter of cost conversion" json:"cloudaccount_id"`
+	CloudproviderId string `help:"cloudprovider filter of cost conversion" json:"cloudprovider_id"`
+	DomainIdFilter  string `help:"domain filter of cost conversion" json:"domain_id_filter"`
+
+	EnableDate string  `help:"enable date of conversion ratio, example:202107" json:"enable_date"`
+	Ratio      float64 `help:"cost conversion ratio" json:"ratio"`
+}
+
+func (opt *CostConversionCreateOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(opt)
+}
+
+type CostConversionUpdateOptions struct {
+	ID string `help:"ID of cost conversion" json:"-"`
+
+	Ratio float64 `help:"cost conversion ratio" json:"ratio"`
+}
+
+func (opt *CostConversionUpdateOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(opt)
+}
+
+func (opt *CostConversionUpdateOptions) GetId() string {
+	return opt.ID
+}
+
+type CostConversionDeleteOptions struct {
+	ID string `help:"ID of cost conversion" json:"-"`
+}
+
+func (opt *CostConversionDeleteOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(opt)
+}
+
+func (opt *CostConversionDeleteOptions) GetId() string {
+	return opt.ID
+}

pkg/mcclient/options/meter/doc.go

@@ -0,0 +1 @@
+package meter // import "yunion.io/x/onecloud/pkg/mcclient/options/meter"

pkg/mcclient/options/monitor/monitor_resource_alert.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/monitor/alerting/conditions/metricquery.go

@@ -88,12 +88,12 @@ func (query *MetricQueryCondition) ExecuteQuery() (*mq.Metrics, error) {
 		metrics.Series = queryResult.series
 		return &metrics, nil
 	}
-	allResources, err := query.QueryCons[0].GetQueryResources()
-	if err != nil {
-		return nil, errors.Wrap(err, "MetricQueryCondition GetQueryResources err")
-	}
+	//allResources, err := query.QueryCons[0].GetQueryResources()
+	//if err != nil {
+	//	return nil, errors.Wrap(err, "MetricQueryCondition GetQueryResources err")
+	//}
 	for _, serie := range queryResult.series {
-		isLatestOfSerie, resource := query.QueryCons[0].serieIsLatestResource(allResources, serie)
+		isLatestOfSerie, resource := query.QueryCons[0].serieIsLatestResource(nil, serie)
 		if !isLatestOfSerie {
 			continue
 		}

pkg/monitor/alerting/conditions/query.go

@@ -156,13 +156,13 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio
 	var matches []*monitor.EvalMatch
 	var alertOkmatches []*monitor.EvalMatch
 
-	allResources, err := c.GetQueryResources()
-	if err != nil {
-		return nil, errors.Wrap(err, "GetQueryResources err")
-	}
+	//allResources, err := c.GetQueryResources()
+	//if err != nil {
+	//	return nil, errors.Wrap(err, "GetQueryResources err")
+	//}
 	for _, series := range seriesList {
 		if len(c.ResType) != 0 {
-			isLatestOfSerie, resource := c.serieIsLatestResource(allResources, series)
+			isLatestOfSerie, resource := c.serieIsLatestResource(nil, series)
 			if !isLatestOfSerie {
 				continue
 			}
@@ -224,6 +224,9 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio
 			})
 		}
 	}
+	if evalMatchCount == 0 && len(seriesList) != 0 {
+		log.Errorf("sql-meta:%s", metas[0].RawQuery)
+	}
 
 	return &alerting.ConditionResult{
 		Firing:             evalMatchCount > 0,
@@ -241,13 +244,13 @@ func (c *QueryCondition) serieIsLatestResource(resources []jsonutils.JSONObject,
 		tagId = "host_id"
 	}
 	seriId := series.Tags[tagId]
-	for _, resource := range resources {
-		id, _ := resource.GetString("id")
-		if seriId == id {
-			return true, resource
-		}
-	}
-	return false, nil
+	//for _, resource := range resources {
+	//	id, _ := resource.GetString("id")
+	//	if seriId == id {
+	//		return true, resource
+	//	}
+	//}
+	return models.MonitorResourceManager.GetResourceObj(seriId)
 }
 
 func (c *QueryCondition) FillSerieByResourceField(resource jsonutils.JSONObject,
@@ -694,7 +697,8 @@ func (c *QueryCondition) getFilterResources(start int, end int,
 		filterObj := make([]jsonutils.JSONObject, 0)
 		for _, res := range tmp {
 			val, _ := res.GetString(relationKey)
-			if c.Query.Model.Tags[i].Operator == "=" {
+			op := c.Query.Model.Tags[i].Operator
+			if op == "=" || len(op) == 0 {
 				if val == c.Query.Model.Tags[i].Value {
 					filterObj = append(filterObj, res)
 				}

pkg/monitor/metricquery/interfaces.go

@@ -20,8 +20,9 @@ import (
 )
 
 type Metrics struct {
-	Series tsdb.TimeSeriesSlice
-	Metas  []tsdb.QueryResultMeta
+	SeriesTotal int64
+	Series      tsdb.TimeSeriesSlice
+	Metas       []tsdb.QueryResultMeta
 }
 
 type MetricQuery interface {

pkg/monitor/models/modelset.go

@@ -0,0 +1,239 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"yunion.io/x/onecloud/pkg/apihelper"
+	"yunion.io/x/onecloud/pkg/apis/monitor"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	compute_models "yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	mcclient_modules "yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type IMonitorResModelSet interface {
+	apihelper.IModelSet
+	GetResType() string
+}
+
+type (
+	Servers  map[string]*Guest
+	Hosts    map[string]*Host
+	Rds      map[string]*SRds
+	Redis    map[string]*SRedis
+	Oss      map[string]*SOss
+	Accounts map[string]*SAccount
+	Storages map[string]*SStorage
+)
+
+type Details struct {
+	//com_apis.CloudproviderDetails
+	//Host          string
+	HostId        string
+	Zone          string
+	zoneId        string
+	zoneExtId     string
+	Cloudregion   string
+	CloudregionId string
+	Tenant        string
+	TenantId      string
+	Brand         string
+	DomainId      string
+	ProjectDomain string
+}
+type Guest struct {
+	compute_models.SGuest
+	Details
+}
+
+type Host struct {
+	Id string
+	compute_models.SHost
+	Details
+}
+
+type SRds struct {
+	compute_models.SDBInstance
+	Details
+}
+
+type SRedis struct {
+	compute_models.SElasticcache
+	Details
+}
+
+type SOss struct {
+	compute_models.SBucket
+	Details
+}
+
+type SStorage struct {
+	Id string
+	compute_models.SStorage
+	Details
+}
+
+type SAccount struct {
+	Id string
+	compute_models.SCloudaccount
+	Details
+}
+
+func (s Servers) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.Servers
+}
+
+func (s Servers) NewModel() db.IModel {
+	return &Guest{}
+}
+
+func (s Servers) AddModel(i db.IModel) {
+	resource := i.(*Guest)
+	s[resource.Id] = resource
+}
+
+func (s Servers) Copy() apihelper.IModelSet {
+	return s
+}
+
+func (s Servers) GetResType() string {
+	return monitor.METRIC_RES_TYPE_GUEST
+}
+
+func (h Hosts) AddModel(i db.IModel) {
+	resource := i.(*Host)
+	h[resource.Id] = resource
+}
+
+func (h Hosts) Copy() apihelper.IModelSet {
+	return h
+}
+
+func (h Hosts) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.Hosts
+}
+
+func (h Hosts) NewModel() db.IModel {
+	return &Host{}
+}
+
+func (h Hosts) GetResType() string {
+	return monitor.METRIC_RES_TYPE_HOST
+}
+
+func (r Rds) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.DBInstance
+}
+
+func (r Rds) NewModel() db.IModel {
+	return &SRds{}
+}
+
+func (r Rds) AddModel(i db.IModel) {
+	resource := i.(*SRds)
+	r[resource.Id] = resource
+}
+
+func (r Rds) Copy() apihelper.IModelSet {
+	return r
+}
+
+func (r Rds) GetResType() string {
+	return monitor.METRIC_RES_TYPE_RDS
+}
+
+func (r Redis) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.ElasticCache
+}
+
+func (r Redis) NewModel() db.IModel {
+	return &SRedis{}
+}
+
+func (r Redis) AddModel(i db.IModel) {
+	resource := i.(*SRedis)
+	r[resource.Id] = resource
+}
+
+func (r Redis) Copy() apihelper.IModelSet {
+	return r
+}
+
+func (r Redis) GetResType() string {
+	return monitor.METRIC_RES_TYPE_REDIS
+}
+
+func (o Oss) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.Buckets
+}
+
+func (o Oss) NewModel() db.IModel {
+	return &SOss{}
+}
+
+func (o Oss) AddModel(i db.IModel) {
+	resource := i.(*SOss)
+	o[resource.Id] = resource
+}
+
+func (o Oss) Copy() apihelper.IModelSet {
+	return o
+}
+
+func (o Oss) GetResType() string {
+	return monitor.METRIC_RES_TYPE_OSS
+}
+
+func (a Accounts) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.Cloudaccounts
+}
+
+func (a Accounts) NewModel() db.IModel {
+	return &SAccount{}
+}
+
+func (a Accounts) AddModel(i db.IModel) {
+	resource := i.(*SAccount)
+	a[resource.Id] = resource
+}
+
+func (a Accounts) Copy() apihelper.IModelSet {
+	return a
+}
+
+func (a Accounts) GetResType() string {
+	return monitor.METRIC_RES_TYPE_CLOUDACCOUNT
+}
+
+func (s Storages) ModelManager() modulebase.IBaseManager {
+	return &mcclient_modules.Storages
+}
+
+func (s Storages) NewModel() db.IModel {
+	return &SStorage{}
+}
+
+func (s Storages) AddModel(i db.IModel) {
+	resource := i.(*SStorage)
+	s[resource.Id] = resource
+}
+
+func (s Storages) Copy() apihelper.IModelSet {
+	return s
+}
+
+func (s Storages) GetResType() string {
+	return monitor.METRIC_RES_TYPE_STORAGE
+}

pkg/monitor/models/modelsets.go

@@ -0,0 +1,82 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import "yunion.io/x/onecloud/pkg/apihelper"
+
+type MonitorResModelSets struct {
+	Servers  Servers
+	Hosts    Hosts
+	Rds      Rds
+	Redis    Redis
+	Oss      Oss
+	Accounts Accounts
+	Storages Storages
+}
+
+func (m *MonitorResModelSets) NewEmpty() apihelper.IModelSets {
+	return NewModelSets()
+}
+
+func (m *MonitorResModelSets) ModelSetList() []apihelper.IModelSet {
+	return []apihelper.IModelSet{
+		m.Servers,
+		m.Hosts,
+		m.Rds,
+		m.Redis,
+		m.Oss,
+		m.Accounts,
+		m.Storages,
+	}
+}
+
+func (m *MonitorResModelSets) ApplyUpdates(mssNews apihelper.IModelSets) apihelper.ModelSetsUpdateResult {
+	r := apihelper.ModelSetsUpdateResult{
+		Changed: false,
+		Correct: true,
+	}
+	mssList := m.ModelSetList()
+	mssNewsList := mssNews.ModelSetList()
+	for i, mss := range mssList {
+		mssNews := mssNewsList[i]
+		msR := apihelper.ModelSetApplyUpdates(mss, mssNews)
+		if !r.Changed && msR.Changed {
+			r.Changed = true
+		}
+	}
+	return r
+}
+
+func (m *MonitorResModelSets) Copy() apihelper.IModelSets {
+	//TODO CHECKE
+	return m
+}
+
+func (m *MonitorResModelSets) CopyJoined() apihelper.IModelSets {
+	//TODO CHECKE
+	return m
+}
+
+func NewModelSets() *MonitorResModelSets {
+	return &MonitorResModelSets{
+		Servers:  Servers{},
+		Hosts:    Hosts{},
+		Rds:      Rds{},
+		Redis:    Redis{},
+		Oss:      Oss{},
+		Accounts: Accounts{},
+		Storages: Storages{},
+	}
+}

pkg/monitor/models/monitor_resource.go

@@ -17,6 +17,8 @@ package models
 import (
 	"context"
 	"fmt"
+	"reflect"
+	"sync"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -25,10 +27,12 @@ import (
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apihelper"
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -37,6 +41,33 @@ var (
 	MonitorResourceManager *SMonitorResourceManager
 )
 
+type IMonitorResourceCache interface {
+	Get(resId string) (jsonutils.JSONObject, bool)
+}
+
+type sMonitorResourceCache struct {
+	length int
+	sync.Map
+}
+
+func (c *sMonitorResourceCache) set(resId string, obj jsonutils.JSONObject) {
+	c.Store(resId, obj)
+	c.length++
+}
+
+func (c *sMonitorResourceCache) remove(resId string) {
+	c.Delete(resId)
+	c.length--
+}
+
+func (c *sMonitorResourceCache) Get(resId string) (jsonutils.JSONObject, bool) {
+	obj, ok := c.Load(resId)
+	if !ok {
+		return nil, false
+	}
+	return obj.(jsonutils.JSONObject), true
+}
+
 func init() {
 	MonitorResourceManager = &SMonitorResourceManager{
 		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
@@ -45,6 +76,7 @@ func init() {
 			"monitorresource",
 			"monitorresources",
 		),
+		monitorResModelSets: NewModelSets(),
 	}
 	MonitorResourceManager.SetVirtualObject(MonitorResourceManager)
 	RegistryResourceSync(NewGuestResourceSync())
@@ -56,9 +88,15 @@ func init() {
 	RegistryResourceSync(NewStorageResourceSync())
 }
 
+func (manager *SMonitorResourceManager) GetModelSets() *MonitorResModelSets {
+	return manager.monitorResModelSets
+}
+
 type SMonitorResourceManager struct {
 	db.SVirtualResourceBaseManager
 	db.SEnabledResourceBaseManager
+
+	monitorResModelSets *MonitorResModelSets
 }
 
 type SMonitorResource struct {
@@ -70,22 +108,6 @@ type SMonitorResource struct {
 	ResType    string `width:"36" charset:"ascii" list:"user" update:"user" json:"res_type"`
 }
 
-func (manager *SMonitorResourceManager) SyncResources(ctx context.Context, userCred mcclient.TokenCredential,
-	isStart bool) {
-	for _, sync := range GetResourceSyncMap() {
-		err := sync.SyncResources(ctx, userCred, jsonutils.NewDict())
-		if err != nil {
-			log.Errorf("resType:%s SyncResources err:%v", sync.SyncType(), err)
-		}
-	}
-	err := CommonAlertManager.Run(ctx)
-	if err != nil {
-		log.Errorf("CommonAlertManager UpdateMonitorResourceJoint err:%v", err)
-		return
-	}
-	log.Infoln("====SMonitorResourceManager SyncResources End====")
-}
-
 func (manager *SMonitorResourceManager) GetMonitorResources(input monitor.MonitorResourceListInput) ([]SMonitorResource, error) {
 	monitorResources := make([]SMonitorResource, 0)
 	query := manager.Query()
@@ -381,3 +403,95 @@ func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match
 	return errors.NewAggregate(errs)
 
 }
+
+func (manager *SMonitorResourceManager) GetResourceObj(id string) (bool, jsonutils.JSONObject) {
+	for _, set := range manager.GetModelSets().ModelSetList() {
+		setRv := reflect.ValueOf(set)
+		for _, kRv := range setRv.MapKeys() {
+			if id == kRv.String() {
+				mRv := setRv.MapIndex(kRv)
+				if mRv.IsValid() {
+					return true, jsonutils.Marshal(mRv.Interface())
+				}
+			}
+		}
+	}
+	return false, nil
+}
+
+func (manager *SMonitorResourceManager) SyncResources(ctx context.Context, mss *MonitorResModelSets) error {
+	userCred := auth.AdminCredential()
+	errs := make([]error, 0)
+	log.Infoln("start sync")
+	for _, set := range mss.ModelSetList() {
+		setRv := reflect.ValueOf(set)
+		typ := manager.GetSetType(set)
+		log.Infof("Type: %s,length: %d", typ, len(setRv.MapKeys()))
+		for _, kRv := range setRv.MapKeys() {
+			mRv := setRv.MapIndex(kRv)
+			//log.Errorf("resID:%s", kRv.String())
+			input := monitor.MonitorResourceListInput{
+				ResId: []string{kRv.String()},
+			}
+			res, err := MonitorResourceManager.GetMonitorResources(input)
+			if err != nil {
+				return errors.Wrap(err, "GetMonitorResources err")
+			}
+			if mRv.IsValid() {
+				obj := jsonutils.Marshal(mRv.Interface())
+				if len(res) == 0 {
+					// no find to create
+					createData := newMonitorResourceCreateInput(obj, typ)
+					_, err = db.DoCreate(MonitorResourceManager, ctx, userCred, nil, createData,
+						userCred)
+					if err != nil {
+						name, _ := createData.GetString("name")
+						errs = append(errs, errors.Wrapf(err, "monitorResource:%s resType:%s DoCreate err", name, typ))
+					}
+					continue
+				}
+				_, err = db.Update(&res[0], func() error {
+					obj.(*jsonutils.JSONDict).Remove("id")
+					(&res[0]).ResType = typ
+					obj.Unmarshal(&res[0])
+					return nil
+				})
+				if err != nil {
+					errs = append(errs, errors.Wrapf(err, "monitorResource:%s Update err", res[0].Name))
+					continue
+				}
+				continue
+			}
+			if len(res) != 0 {
+				log.Infof("delete monitor resource,resId: %s,resType: %s", res[0].ResId, res[0].ResType)
+				err := (&res[0]).RealDelete(ctx, userCred)
+				if err != nil {
+					errs = append(errs, errors.Wrapf(err, "delete monitorResource:%s err", res[0].GetId()))
+				}
+			}
+		}
+	}
+	log.Infoln("SMonitorResourceManager SyncResources End")
+	err := CommonAlertManager.Run(ctx)
+	if err != nil {
+		log.Errorf("CommonAlertManager UpdateMonitorResourceJoint err:%v", err)
+	}
+	return errors.NewAggregate(errs)
+}
+
+func (manager *SMonitorResourceManager) GetSetType(set apihelper.IModelSet) string {
+	if iset, ok := set.(IMonitorResModelSet); ok {
+		return iset.GetResType()
+	}
+	return "NONE"
+}
+
+func newMonitorResourceCreateInput(input jsonutils.JSONObject, typ string) jsonutils.JSONObject {
+	monitorResource := jsonutils.DeepCopy(input).(*jsonutils.JSONDict)
+	id, _ := monitorResource.GetString("id")
+	monitorResource.Add(jsonutils.NewString(id), "res_id")
+	monitorResource.Remove("id")
+	monitorResource.Add(jsonutils.NewString(typ), "res_type")
+
+	return monitorResource
+}

pkg/monitor/models/recordshield.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/monitor/models/unifiedmonitor.go

@@ -244,11 +244,31 @@ func (self *SUnifiedMonitorManager) PerformQuery(ctx context.Context, userCred m
 		return jsonutils.NewDict(), err
 	}
 
+	if len(inputQuery.Soffset) != 0 && len(inputQuery.Slimit) != 0 {
+		seriesTotal := self.fillSearchSeriesTotalQuery(*inputQuery.MetricQuery[0])
+		rtn.SeriesTotal = seriesTotal
+	}
+
 	setSerieRowName(&rtn.Series, groupByTag)
 	fillSerieTags(&rtn.Series)
 	return jsonutils.Marshal(rtn), nil
 }
 
+func (self *SUnifiedMonitorManager) fillSearchSeriesTotalQuery(fork monitor.AlertQuery) int64 {
+	newGroupByPart := make([]monitor.MetricQueryPart, 0)
+	newGroupByPart = append(newGroupByPart, fork.Model.GroupBy[0])
+	fork.Model.GroupBy = newGroupByPart
+	forkInputQury := new(monitor.MetricInputQuery)
+	forkInputQury.MetricQuery = []*monitor.AlertQuery{&fork}
+	rtn, err := doQuery(*forkInputQury)
+	if err != nil {
+		log.Errorf("exec forkInputQury err:%v", err)
+		return 0
+	}
+	log.Errorf("series len:%d", len(rtn.Series))
+	return int64(len(rtn.Series))
+}
+
 func (self *SUnifiedMonitorManager) handleDataPreSignature(ctx context.Context, data jsonutils.JSONObject) {
 	scope, _ := data.GetString("scope")
 	isIdentityName, _ := data.Bool("identity_name")
@@ -340,6 +360,12 @@ func setDefaultValue(query *monitor.AlertQuery, inputQuery *monitor.MetricInputQ
 				Params: []string{"none"},
 			})
 	}
+	if len(inputQuery.Slimit) != 0 && len(inputQuery.Soffset) != 0 {
+		query.Model.GroupBy = append(query.Model.GroupBy,
+			monitor.MetricQueryPart{Type: "slimit", Params: []string{inputQuery.Slimit}},
+			monitor.MetricQueryPart{Type: "soffset", Params: []string{inputQuery.Soffset}},
+		)
+	}
 
 	if query.Model.Database == "" {
 		database := ""

pkg/monitor/options/options.go

@@ -30,6 +30,11 @@ type AlerterOptions struct {
 	InitScopeSuggestConfigIntervalSeconds          int   `help:"internal to init scope suggest configs" default:"900"`
 	InitAlertResourceAdminRoleUsersIntervalSeconds int   `help:"internal to init alert resource admin role users " default:"3600"`
 	MonitorResourceSyncIntervalSeconds             int   `help:"internal to sync monitor resource,unit: h " default:"1"`
+
+	APISyncInterval  int `default:"3600"`
+	APIListBatchSize int `default:"1024"`
+
+	WorkerCheckInterval int `default:"180"`
 }
 
 var (

pkg/monitor/service/service.go

@@ -40,6 +40,7 @@ import (
 	"yunion.io/x/onecloud/pkg/monitor/subscriptionmodel"
 	_ "yunion.io/x/onecloud/pkg/monitor/tasks"
 	_ "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/influxdb"
+	"yunion.io/x/onecloud/pkg/monitor/worker"
 )
 
 func StartService() {
@@ -66,15 +67,22 @@ func StartService() {
 	cron.AddJobAtIntervalsWithStartRun("InitAlertResourceAdminRoleUsers", time.Duration(opts.InitAlertResourceAdminRoleUsersIntervalSeconds)*time.Second, models.GetAlertResourceManager().GetAdminRoleUsers, true)
 	cron.AddJobEveryFewDays("DeleteRecordsOfThirtyDaysAgoRecords", 1, 0, 0, 0,
 		models.AlertRecordManager.DeleteRecordsOfThirtyDaysAgo, false)
-	cron.AddJobAtIntervalsWithStartRun("MonitorResourceSync", time.Duration(opts.MonitorResourceSyncIntervalSeconds)*time.Minute*60, models.MonitorResourceManager.SyncResources, true)
+	//cron.AddJobAtIntervalsWithStartRun("MonitorResourceSync", time.Duration(opts.MonitorResourceSyncIntervalSeconds)*time.Minute*60, models.MonitorResourceManager.SyncResources, true)
 	cron.Start()
 	defer cron.Stop()
 
 	subscriptionmodel.SubscriptionManager.AddSubscription()
 	models.CommonAlertManager.SetSubscriptionManager(subscriptionmodel.SubscriptionManager)
 
-	InitInfluxDBSubscriptionHandlers(app, baseOpts)
+	worker := worker.NewWorker(opts)
+	if worker == nil {
+		log.Fatalf("new worker failed")
+	}
+	go worker.Start(context.Background())
+
 	//common_app.ServeForever(app, baseOpts)
+	InitInfluxDBSubscriptionHandlers(app, baseOpts)
+
 }
 
 func startServices() {

pkg/monitor/tsdb/driver/influxdb/influxdb.go

@@ -94,6 +94,7 @@ func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, t
 	}
 	dsInfo.Database = db
 
+	log.Errorf("rawQuery:%s", rawQuery)
 	req, err := e.createRequest(dsInfo, rawQuery)
 	if err != nil {
 		return nil, err

pkg/monitor/tsdb/driver/influxdb/query.go

@@ -166,7 +166,7 @@ func (query *Query) renderGroupBy(queryContext *tsdb.TsdbQuery) string {
 			groupBy += " GROUP BY"
 		}
 
-		if i > 0 && group.Type != "fill" {
+		if i > 0 && utils.IsInStringArray(group.Type, []string{"field", "time", "tag"}) {
 			groupBy += ", " //fill is so very special. fill is a creep, fill is a weirdo
 		} else {
 			groupBy += " "

pkg/monitor/tsdb/driver/influxdb/query_part.go

@@ -111,6 +111,8 @@ func init() {
 
 	renders["math"] = QueryDefinition{Renderer: suffixRenderer}
 	renders["alias"] = QueryDefinition{Renderer: aliasRenderer}
+	renders["slimit"] = QueryDefinition{Renderer: typeRenderer}
+	renders["soffset"] = QueryDefinition{Renderer: typeRenderer}
 }
 
 func fieldRenderer(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
@@ -173,6 +175,10 @@ func aliasRenderer(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, inne
 	return fmt.Sprintf(`%s AS "%s"`, innerExpr, part.Params[0])
 }
 
+func typeRenderer(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
+	return fmt.Sprintf(" %s %s", part.Type, part.Params[0])
+}
+
 func (r QueryDefinition) Render(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
 	return r.Renderer(query, queryCtx, part, innerExpr)
 }

pkg/monitor/worker/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package worker // import "yunion.io/x/onecloud/pkg/monitor/worker"

pkg/monitor/worker/worker.go

@@ -0,0 +1,97 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package worker
+
+import (
+	"context"
+	"runtime"
+	"runtime/debug"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apihelper"
+	"yunion.io/x/onecloud/pkg/monitor/models"
+	"yunion.io/x/onecloud/pkg/monitor/options"
+	"yunion.io/x/onecloud/pkg/vpcagent/worker"
+)
+
+type Worker struct {
+	opts *options.AlerterOptions
+
+	apih *apihelper.APIHelper
+}
+
+func NewWorker(opts *options.AlerterOptions) worker.IWorker {
+	modelSets := models.MonitorResourceManager.GetModelSets()
+	apiOpts := &apihelper.Options{
+		CommonOptions:        opts.CommonOptions,
+		SyncInterval:         opts.APISyncInterval,
+		ListBatchSize:        opts.APIListBatchSize,
+		IncludeDetails:       true,
+		InCludeOtherCloudEnv: true,
+	}
+	apih, err := apihelper.NewAPIHelper(apiOpts, modelSets)
+	if err != nil {
+		return nil
+	}
+	w := &Worker{
+		opts: opts,
+		apih: apih,
+	}
+	return w
+}
+
+func (w *Worker) Start(ctx context.Context) {
+	defer func() {
+		log.Infoln("monitor resource: worker bye")
+	}()
+
+	log.Infoln("start to get api Resource")
+	go w.apih.Start(ctx)
+
+	var mss *models.MonitorResModelSets
+	for {
+		select {
+		case imss := <-w.apih.ModelSets():
+			log.Infof("monitorRes: got new data from api helper")
+			mss = imss.(*models.MonitorResModelSets)
+			if err := w.run(ctx, mss); err != nil {
+				log.Errorf("monitorResWork err: %v", err)
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (w *Worker) run(ctx context.Context, mss *models.MonitorResModelSets) (err error) {
+	defer func() {
+		if panicVal := recover(); panicVal != nil {
+			if panicErr, ok := panicVal.(runtime.Error); ok {
+				err = errors.Wrap(panicErr, string(debug.Stack()))
+			} else if panicErr, ok := panicVal.(error); ok {
+				err = panicErr
+			} else {
+				panic(panicVal)
+			}
+		}
+	}()
+	err = models.MonitorResourceManager.SyncResources(ctx, mss)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/multicloud/azure/azure.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Copyright 2019 Yunion
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/multicloud/cloudpods/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cloudpods // import "yunion.io/x/onecloud/pkg/multicloud/cloudpods"

pkg/multicloud/cloudpods/provider/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package provider // import "yunion.io/x/onecloud/pkg/multicloud/cloudpods/provider"

pkg/multicloud/cloudpods/shell/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package shell // import "yunion.io/x/onecloud/pkg/multicloud/cloudpods/shell"

pkg/multicloud/huaweistack/shell/endpoints.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package shell
 
 import (

pkg/multicloud/huaweistack/shell/service.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package shell
 
 import (