diff --git a/cmd/climc/shell/monitor/alertrecordshield.go b/cmd/climc/shell/monitor/alertrecordshield.go index ce74637cca..f6828508c3 100644 --- a/cmd/climc/shell/monitor/alertrecordshield.go +++ b/cmd/climc/shell/monitor/alertrecordshield.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package monitor import ( diff --git a/cmd/climc/shell/monitor/monitor_resourece_alert.go b/cmd/climc/shell/monitor/monitor_resourece_alert.go index 98f31c448b..bc73ffc904 100644 --- a/cmd/climc/shell/monitor/monitor_resourece_alert.go +++ b/cmd/climc/shell/monitor/monitor_resourece_alert.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package monitor import ( diff --git a/pkg/apihelper/apihelper.go b/pkg/apihelper/apihelper.go index b9b33a2be0..39cb9b0809 100644 --- a/pkg/apihelper/apihelper.go +++ b/pkg/apihelper/apihelper.go @@ -101,7 +101,7 @@ func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) { s := h.adminClientSession(ctx) mss := h.modelSets.Copy() - r, err := SyncModelSets(mss, s, h.opts.ListBatchSize) + r, err := SyncModelSets(mss, s, h.opts) if err != nil { return false, err } diff --git a/pkg/apihelper/interface.go b/pkg/apihelper/interface.go index 365e06cca1..eb706510bd 100644 --- a/pkg/apihelper/interface.go +++ b/pkg/apihelper/interface.go @@ -44,7 +44,7 @@ type IModelSetEmulatedIncluder interface { IncludeEmulated() bool } -func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int) (r ModelSetsUpdateResult, err error) { +func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) { mss := mssOld.ModelSetList() mssNews := mssOld.NewEmpty() for i, msNew := range mssNews.ModelSetList() { @@ -60,10 +60,11 @@ func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int) ModelManager: msNew.ModelManager(), MinUpdatedAt: minUpdatedAt, ModelSet: msNew, - BatchListSize: batchSize, + BatchListSize: opt.ListBatchSize, - IncludeDetails: false, - IncludeEmulated: includeEmulated, + IncludeDetails: opt.IncludeDetails, + IncludeEmulated: includeEmulated, + InCludeOtherCloudEnv: opt.InCludeOtherCloudEnv, }) if err != nil { return diff --git a/pkg/apihelper/options.go b/pkg/apihelper/options.go index 37484eef9e..2e7e9c62f8 100644 --- a/pkg/apihelper/options.go +++ b/pkg/apihelper/options.go @@ -21,6 +21,8 @@ import ( type Options struct { common_options.CommonOptions - SyncInterval int - ListBatchSize int + SyncInterval int + ListBatchSize int + IncludeDetails bool + InCludeOtherCloudEnv bool } diff --git a/pkg/apihelper/reflect.go b/pkg/apihelper/reflect.go index 659407f8c3..9af1a6d719 100644 --- a/pkg/apihelper/reflect.go +++ b/pkg/apihelper/reflect.go @@ -38,10 +38,11 @@ type GetModelsOptions struct { ModelManager mcclient_modulebase.IBaseManager ModelSet IModelSet - BatchListSize int - MinUpdatedAt time.Time - IncludeDetails bool - IncludeEmulated bool + BatchListSize int + MinUpdatedAt time.Time + IncludeDetails bool + IncludeEmulated bool + InCludeOtherCloudEnv bool } func GetModels(opts *GetModelsOptions) error { @@ -106,15 +107,19 @@ func GetModels(opts *GetModelsOptions) error { ShowEmulated: options.Bool(opts.IncludeEmulated), Filter: []string{ minUpdatedAtFilter(minUpdatedAt), // order matters, filter.0 - "manager_id.isnullorempty()", // len(manager_id) > 0 is for pubcloud objects - "external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects - "cloud_env=onpremise", }, OrderBy: []string{"updated_at"}, Order: "asc", Limit: options.Int(opts.BatchListSize), Offset: options.Int(0), } + if !opts.InCludeOtherCloudEnv { + listOptions.Filter = append(listOptions.Filter, + "manager_id.isnullorempty()", // len(manager_id) > 0 is for pubcloud objects + "external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects + "cloud_env=onpremise", + ) + } if !minUpdatedAt.Equal(PseudoZeroTime) { // Only fetching pending deletes when we are doing incremental fetch listOptions.PendingDeleteAll = options.Bool(true) diff --git a/pkg/apis/monitor/alert_record_shield.go b/pkg/apis/monitor/alert_record_shield.go index 2091718c5f..c322239e46 100644 --- a/pkg/apis/monitor/alert_record_shield.go +++ b/pkg/apis/monitor/alert_record_shield.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package monitor import ( diff --git a/pkg/apis/monitor/alert_resource_record.go b/pkg/apis/monitor/alert_resource_record.go index a8af13ece7..47e514a2e2 100644 --- a/pkg/apis/monitor/alert_resource_record.go +++ b/pkg/apis/monitor/alert_resource_record.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package monitor import ( diff --git a/pkg/apis/monitor/alertquery_const.go b/pkg/apis/monitor/alertquery_const.go index 5bb37052a3..26114255d7 100644 --- a/pkg/apis/monitor/alertquery_const.go +++ b/pkg/apis/monitor/alertquery_const.go @@ -15,6 +15,28 @@ package monitor var ( + UnionTags = []string{ + "host", + "host_id", + "id", + "ips", + "name", + "zone", + "zone_id", + "zone_ext_id", + "os_type", + "status", + "cloudregion", + "cloudregion_id", + "region_ext_id", + "tenant", + "tenant_id", + "brand", + "vm_scaling_group_id", + "domain_id", + "project_domain", + } + ServerTags = map[string]string{ "host": "host", "host_id": "host_id", diff --git a/pkg/apis/monitor/unifiedmonitor_const.go b/pkg/apis/monitor/unifiedmonitor_const.go index e55930f7f8..5cc3842019 100644 --- a/pkg/apis/monitor/unifiedmonitor_const.go +++ b/pkg/apis/monitor/unifiedmonitor_const.go @@ -69,9 +69,11 @@ type MetricFunc struct { } type MetricInputQuery struct { - From string `json:"from"` - To string `json:"to"` - Scope string `json:"scope"` + From string `json:"from"` + To string `json:"to"` + Scope string `json:"scope"` + Slimit string `json:"slimit"` + Soffset string `json:"soffset"` //default group by Unit bool `json:"unit"` Interval string `json:"interval"` diff --git a/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go b/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go index 9e40053e99..7e251dea85 100644 --- a/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go +++ b/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package alertrecordhistorymon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon" diff --git a/pkg/cloudmon/collectors/alimon/doc.go b/pkg/cloudmon/collectors/alimon/doc.go index b37cec1ba8..bbb4630564 100644 --- a/pkg/cloudmon/collectors/alimon/doc.go +++ b/pkg/cloudmon/collectors/alimon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package alimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon" diff --git a/pkg/cloudmon/collectors/apsaramon/doc.go b/pkg/cloudmon/collectors/apsaramon/doc.go index fbb1f87aeb..cd5cbb0d48 100644 --- a/pkg/cloudmon/collectors/apsaramon/doc.go +++ b/pkg/cloudmon/collectors/apsaramon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package apsaramon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon" diff --git a/pkg/cloudmon/collectors/awsmon/doc.go b/pkg/cloudmon/collectors/awsmon/doc.go index 69a475d946..4b901cf656 100644 --- a/pkg/cloudmon/collectors/awsmon/doc.go +++ b/pkg/cloudmon/collectors/awsmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package awsmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon" diff --git a/pkg/cloudmon/collectors/azuremon/doc.go b/pkg/cloudmon/collectors/azuremon/doc.go index 37c41c3e3f..067f44da6d 100644 --- a/pkg/cloudmon/collectors/azuremon/doc.go +++ b/pkg/cloudmon/collectors/azuremon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package azuremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon" diff --git a/pkg/cloudmon/collectors/cloudaccountmon/doc.go b/pkg/cloudmon/collectors/cloudaccountmon/doc.go index 5f90225102..212fbeef24 100644 --- a/pkg/cloudmon/collectors/cloudaccountmon/doc.go +++ b/pkg/cloudmon/collectors/cloudaccountmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cloudaccountmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon" diff --git a/pkg/cloudmon/collectors/common/doc.go b/pkg/cloudmon/collectors/common/doc.go index ee93a28f61..2eda902aae 100644 --- a/pkg/cloudmon/collectors/common/doc.go +++ b/pkg/cloudmon/collectors/common/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package common // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" diff --git a/pkg/cloudmon/collectors/doc.go b/pkg/cloudmon/collectors/doc.go index bcc8136e9f..e0221a8634 100644 --- a/pkg/cloudmon/collectors/doc.go +++ b/pkg/cloudmon/collectors/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package collectors // import "yunion.io/x/onecloud/pkg/cloudmon/collectors" diff --git a/pkg/cloudmon/collectors/ecloudmon/doc.go b/pkg/cloudmon/collectors/ecloudmon/doc.go index 52dca82637..1816bca1df 100644 --- a/pkg/cloudmon/collectors/ecloudmon/doc.go +++ b/pkg/cloudmon/collectors/ecloudmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ecloudmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/ecloudmon" diff --git a/pkg/cloudmon/collectors/gcpmon/doc.go b/pkg/cloudmon/collectors/gcpmon/doc.go index ce3e544c34..5fbf0f0f72 100644 --- a/pkg/cloudmon/collectors/gcpmon/doc.go +++ b/pkg/cloudmon/collectors/gcpmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gcpmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon" diff --git a/pkg/cloudmon/collectors/huaweimon/doc.go b/pkg/cloudmon/collectors/huaweimon/doc.go index 0d2f51138b..14851f8a3f 100644 --- a/pkg/cloudmon/collectors/huaweimon/doc.go +++ b/pkg/cloudmon/collectors/huaweimon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package huaweimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon" diff --git a/pkg/cloudmon/collectors/jdmon/doc.go b/pkg/cloudmon/collectors/jdmon/doc.go index 96712df46c..bfa9f0019e 100644 --- a/pkg/cloudmon/collectors/jdmon/doc.go +++ b/pkg/cloudmon/collectors/jdmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package jdmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/jdmon" diff --git a/pkg/cloudmon/collectors/qcmon/doc.go b/pkg/cloudmon/collectors/qcmon/doc.go index c8ce0c74d8..e7b2aa47d9 100644 --- a/pkg/cloudmon/collectors/qcmon/doc.go +++ b/pkg/cloudmon/collectors/qcmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package qcmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon" diff --git a/pkg/cloudmon/collectors/storagemon/doc.go b/pkg/cloudmon/collectors/storagemon/doc.go index 1a6ad7e3fb..d33402d4dd 100644 --- a/pkg/cloudmon/collectors/storagemon/doc.go +++ b/pkg/cloudmon/collectors/storagemon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package storagemon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon" diff --git a/pkg/cloudmon/collectors/vmwaremon/doc.go b/pkg/cloudmon/collectors/vmwaremon/doc.go index 1f87d5e8f3..4a60cd5379 100644 --- a/pkg/cloudmon/collectors/vmwaremon/doc.go +++ b/pkg/cloudmon/collectors/vmwaremon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package vmwaremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon" diff --git a/pkg/cloudmon/collectors/zstackmon/doc.go b/pkg/cloudmon/collectors/zstackmon/doc.go index 4e68378322..a3576d96af 100644 --- a/pkg/cloudmon/collectors/zstackmon/doc.go +++ b/pkg/cloudmon/collectors/zstackmon/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package zstackmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon" diff --git a/pkg/cloudmon/options/doc.go b/pkg/cloudmon/options/doc.go index d49590c17e..7632a877a6 100644 --- a/pkg/cloudmon/options/doc.go +++ b/pkg/cloudmon/options/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package options // import "yunion.io/x/onecloud/pkg/cloudmon/options" diff --git a/pkg/compute/regiondrivers/huaweistack.go b/pkg/compute/regiondrivers/huaweistack.go index de9b420ed7..3c6b88c7b4 100644 --- a/pkg/compute/regiondrivers/huaweistack.go +++ b/pkg/compute/regiondrivers/huaweistack.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package regiondrivers import ( diff --git a/pkg/mcclient/options/identity/doc.go b/pkg/mcclient/options/identity/doc.go index f48ddb8651..1d9392250f 100644 --- a/pkg/mcclient/options/identity/doc.go +++ b/pkg/mcclient/options/identity/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package identity // import "yunion.io/x/onecloud/pkg/mcclient/options/identity" diff --git a/pkg/mcclient/options/meter/cost_conversions.go b/pkg/mcclient/options/meter/cost_conversions.go new file mode 100644 index 0000000000..a9997cf366 --- /dev/null +++ b/pkg/mcclient/options/meter/cost_conversions.go @@ -0,0 +1,72 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meter + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type CostConversionListOptions struct { + options.BaseListOptions +} + +func (opt *CostConversionListOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(opt) +} + +type CostConversionCreateOptions struct { + Name string + + IsPublicCloud string `help:"public cloud filter of cost conversion" json:"is_public_cloud"` + Brand string `help:"brand filter of cost conversion" json:"brand"` + CloudaccountId string `help:"cloudaccount filter of cost conversion" json:"cloudaccount_id"` + CloudproviderId string `help:"cloudprovider filter of cost conversion" json:"cloudprovider_id"` + DomainIdFilter string `help:"domain filter of cost conversion" json:"domain_id_filter"` + + EnableDate string `help:"enable date of conversion ratio, example:202107" json:"enable_date"` + Ratio float64 `help:"cost conversion ratio" json:"ratio"` +} + +func (opt *CostConversionCreateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(opt) +} + +type CostConversionUpdateOptions struct { + ID string `help:"ID of cost conversion" json:"-"` + + Ratio float64 `help:"cost conversion ratio" json:"ratio"` +} + +func (opt *CostConversionUpdateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(opt) +} + +func (opt *CostConversionUpdateOptions) GetId() string { + return opt.ID +} + +type CostConversionDeleteOptions struct { + ID string `help:"ID of cost conversion" json:"-"` +} + +func (opt *CostConversionDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(opt) +} + +func (opt *CostConversionDeleteOptions) GetId() string { + return opt.ID +} diff --git a/pkg/mcclient/options/meter/doc.go b/pkg/mcclient/options/meter/doc.go new file mode 100644 index 0000000000..dba89c9562 --- /dev/null +++ b/pkg/mcclient/options/meter/doc.go @@ -0,0 +1 @@ +package meter // import "yunion.io/x/onecloud/pkg/mcclient/options/meter" diff --git a/pkg/mcclient/options/monitor/monitor_resource_alert.go b/pkg/mcclient/options/monitor/monitor_resource_alert.go index 462dbec315..cb0427e35d 100644 --- a/pkg/mcclient/options/monitor/monitor_resource_alert.go +++ b/pkg/mcclient/options/monitor/monitor_resource_alert.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package monitor import ( diff --git a/pkg/monitor/alerting/conditions/metricquery.go b/pkg/monitor/alerting/conditions/metricquery.go index 2e09f57e64..622d6fec14 100644 --- a/pkg/monitor/alerting/conditions/metricquery.go +++ b/pkg/monitor/alerting/conditions/metricquery.go @@ -88,12 +88,12 @@ func (query *MetricQueryCondition) ExecuteQuery() (*mq.Metrics, error) { metrics.Series = queryResult.series return &metrics, nil } - allResources, err := query.QueryCons[0].GetQueryResources() - if err != nil { - return nil, errors.Wrap(err, "MetricQueryCondition GetQueryResources err") - } + //allResources, err := query.QueryCons[0].GetQueryResources() + //if err != nil { + // return nil, errors.Wrap(err, "MetricQueryCondition GetQueryResources err") + //} for _, serie := range queryResult.series { - isLatestOfSerie, resource := query.QueryCons[0].serieIsLatestResource(allResources, serie) + isLatestOfSerie, resource := query.QueryCons[0].serieIsLatestResource(nil, serie) if !isLatestOfSerie { continue } diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index 0ce76e90ee..9b109340c7 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -156,13 +156,13 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio var matches []*monitor.EvalMatch var alertOkmatches []*monitor.EvalMatch - allResources, err := c.GetQueryResources() - if err != nil { - return nil, errors.Wrap(err, "GetQueryResources err") - } + //allResources, err := c.GetQueryResources() + //if err != nil { + // return nil, errors.Wrap(err, "GetQueryResources err") + //} for _, series := range seriesList { if len(c.ResType) != 0 { - isLatestOfSerie, resource := c.serieIsLatestResource(allResources, series) + isLatestOfSerie, resource := c.serieIsLatestResource(nil, series) if !isLatestOfSerie { continue } @@ -224,6 +224,9 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio }) } } + if evalMatchCount == 0 && len(seriesList) != 0 { + log.Errorf("sql-meta:%s", metas[0].RawQuery) + } return &alerting.ConditionResult{ Firing: evalMatchCount > 0, @@ -241,13 +244,13 @@ func (c *QueryCondition) serieIsLatestResource(resources []jsonutils.JSONObject, tagId = "host_id" } seriId := series.Tags[tagId] - for _, resource := range resources { - id, _ := resource.GetString("id") - if seriId == id { - return true, resource - } - } - return false, nil + //for _, resource := range resources { + // id, _ := resource.GetString("id") + // if seriId == id { + // return true, resource + // } + //} + return models.MonitorResourceManager.GetResourceObj(seriId) } func (c *QueryCondition) FillSerieByResourceField(resource jsonutils.JSONObject, @@ -694,7 +697,8 @@ func (c *QueryCondition) getFilterResources(start int, end int, filterObj := make([]jsonutils.JSONObject, 0) for _, res := range tmp { val, _ := res.GetString(relationKey) - if c.Query.Model.Tags[i].Operator == "=" { + op := c.Query.Model.Tags[i].Operator + if op == "=" || len(op) == 0 { if val == c.Query.Model.Tags[i].Value { filterObj = append(filterObj, res) } diff --git a/pkg/monitor/metricquery/interfaces.go b/pkg/monitor/metricquery/interfaces.go index a0a185c190..263bb9d256 100644 --- a/pkg/monitor/metricquery/interfaces.go +++ b/pkg/monitor/metricquery/interfaces.go @@ -20,8 +20,9 @@ import ( ) type Metrics struct { - Series tsdb.TimeSeriesSlice - Metas []tsdb.QueryResultMeta + SeriesTotal int64 + Series tsdb.TimeSeriesSlice + Metas []tsdb.QueryResultMeta } type MetricQuery interface { diff --git a/pkg/monitor/models/modelset.go b/pkg/monitor/models/modelset.go new file mode 100644 index 0000000000..974937de38 --- /dev/null +++ b/pkg/monitor/models/modelset.go @@ -0,0 +1,239 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/onecloud/pkg/apihelper" + "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + compute_models "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + mcclient_modules "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type IMonitorResModelSet interface { + apihelper.IModelSet + GetResType() string +} + +type ( + Servers map[string]*Guest + Hosts map[string]*Host + Rds map[string]*SRds + Redis map[string]*SRedis + Oss map[string]*SOss + Accounts map[string]*SAccount + Storages map[string]*SStorage +) + +type Details struct { + //com_apis.CloudproviderDetails + //Host string + HostId string + Zone string + zoneId string + zoneExtId string + Cloudregion string + CloudregionId string + Tenant string + TenantId string + Brand string + DomainId string + ProjectDomain string +} +type Guest struct { + compute_models.SGuest + Details +} + +type Host struct { + Id string + compute_models.SHost + Details +} + +type SRds struct { + compute_models.SDBInstance + Details +} + +type SRedis struct { + compute_models.SElasticcache + Details +} + +type SOss struct { + compute_models.SBucket + Details +} + +type SStorage struct { + Id string + compute_models.SStorage + Details +} + +type SAccount struct { + Id string + compute_models.SCloudaccount + Details +} + +func (s Servers) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.Servers +} + +func (s Servers) NewModel() db.IModel { + return &Guest{} +} + +func (s Servers) AddModel(i db.IModel) { + resource := i.(*Guest) + s[resource.Id] = resource +} + +func (s Servers) Copy() apihelper.IModelSet { + return s +} + +func (s Servers) GetResType() string { + return monitor.METRIC_RES_TYPE_GUEST +} + +func (h Hosts) AddModel(i db.IModel) { + resource := i.(*Host) + h[resource.Id] = resource +} + +func (h Hosts) Copy() apihelper.IModelSet { + return h +} + +func (h Hosts) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.Hosts +} + +func (h Hosts) NewModel() db.IModel { + return &Host{} +} + +func (h Hosts) GetResType() string { + return monitor.METRIC_RES_TYPE_HOST +} + +func (r Rds) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.DBInstance +} + +func (r Rds) NewModel() db.IModel { + return &SRds{} +} + +func (r Rds) AddModel(i db.IModel) { + resource := i.(*SRds) + r[resource.Id] = resource +} + +func (r Rds) Copy() apihelper.IModelSet { + return r +} + +func (r Rds) GetResType() string { + return monitor.METRIC_RES_TYPE_RDS +} + +func (r Redis) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.ElasticCache +} + +func (r Redis) NewModel() db.IModel { + return &SRedis{} +} + +func (r Redis) AddModel(i db.IModel) { + resource := i.(*SRedis) + r[resource.Id] = resource +} + +func (r Redis) Copy() apihelper.IModelSet { + return r +} + +func (r Redis) GetResType() string { + return monitor.METRIC_RES_TYPE_REDIS +} + +func (o Oss) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.Buckets +} + +func (o Oss) NewModel() db.IModel { + return &SOss{} +} + +func (o Oss) AddModel(i db.IModel) { + resource := i.(*SOss) + o[resource.Id] = resource +} + +func (o Oss) Copy() apihelper.IModelSet { + return o +} + +func (o Oss) GetResType() string { + return monitor.METRIC_RES_TYPE_OSS +} + +func (a Accounts) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.Cloudaccounts +} + +func (a Accounts) NewModel() db.IModel { + return &SAccount{} +} + +func (a Accounts) AddModel(i db.IModel) { + resource := i.(*SAccount) + a[resource.Id] = resource +} + +func (a Accounts) Copy() apihelper.IModelSet { + return a +} + +func (a Accounts) GetResType() string { + return monitor.METRIC_RES_TYPE_CLOUDACCOUNT +} + +func (s Storages) ModelManager() modulebase.IBaseManager { + return &mcclient_modules.Storages +} + +func (s Storages) NewModel() db.IModel { + return &SStorage{} +} + +func (s Storages) AddModel(i db.IModel) { + resource := i.(*SStorage) + s[resource.Id] = resource +} + +func (s Storages) Copy() apihelper.IModelSet { + return s +} + +func (s Storages) GetResType() string { + return monitor.METRIC_RES_TYPE_STORAGE +} diff --git a/pkg/monitor/models/modelsets.go b/pkg/monitor/models/modelsets.go new file mode 100644 index 0000000000..1e272a2ef9 --- /dev/null +++ b/pkg/monitor/models/modelsets.go @@ -0,0 +1,82 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import "yunion.io/x/onecloud/pkg/apihelper" + +type MonitorResModelSets struct { + Servers Servers + Hosts Hosts + Rds Rds + Redis Redis + Oss Oss + Accounts Accounts + Storages Storages +} + +func (m *MonitorResModelSets) NewEmpty() apihelper.IModelSets { + return NewModelSets() +} + +func (m *MonitorResModelSets) ModelSetList() []apihelper.IModelSet { + return []apihelper.IModelSet{ + m.Servers, + m.Hosts, + m.Rds, + m.Redis, + m.Oss, + m.Accounts, + m.Storages, + } +} + +func (m *MonitorResModelSets) ApplyUpdates(mssNews apihelper.IModelSets) apihelper.ModelSetsUpdateResult { + r := apihelper.ModelSetsUpdateResult{ + Changed: false, + Correct: true, + } + mssList := m.ModelSetList() + mssNewsList := mssNews.ModelSetList() + for i, mss := range mssList { + mssNews := mssNewsList[i] + msR := apihelper.ModelSetApplyUpdates(mss, mssNews) + if !r.Changed && msR.Changed { + r.Changed = true + } + } + return r +} + +func (m *MonitorResModelSets) Copy() apihelper.IModelSets { + //TODO CHECKE + return m +} + +func (m *MonitorResModelSets) CopyJoined() apihelper.IModelSets { + //TODO CHECKE + return m +} + +func NewModelSets() *MonitorResModelSets { + return &MonitorResModelSets{ + Servers: Servers{}, + Hosts: Hosts{}, + Rds: Rds{}, + Redis: Redis{}, + Oss: Oss{}, + Accounts: Accounts{}, + Storages: Storages{}, + } +} diff --git a/pkg/monitor/models/monitor_resource.go b/pkg/monitor/models/monitor_resource.go index c31db8b6e3..c8b55a86a5 100644 --- a/pkg/monitor/models/monitor_resource.go +++ b/pkg/monitor/models/monitor_resource.go @@ -17,6 +17,8 @@ package models import ( "context" "fmt" + "reflect" + "sync" "time" "yunion.io/x/jsonutils" @@ -25,10 +27,12 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apihelper" "yunion.io/x/onecloud/pkg/apis/monitor" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -37,6 +41,33 @@ var ( MonitorResourceManager *SMonitorResourceManager ) +type IMonitorResourceCache interface { + Get(resId string) (jsonutils.JSONObject, bool) +} + +type sMonitorResourceCache struct { + length int + sync.Map +} + +func (c *sMonitorResourceCache) set(resId string, obj jsonutils.JSONObject) { + c.Store(resId, obj) + c.length++ +} + +func (c *sMonitorResourceCache) remove(resId string) { + c.Delete(resId) + c.length-- +} + +func (c *sMonitorResourceCache) Get(resId string) (jsonutils.JSONObject, bool) { + obj, ok := c.Load(resId) + if !ok { + return nil, false + } + return obj.(jsonutils.JSONObject), true +} + func init() { MonitorResourceManager = &SMonitorResourceManager{ SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( @@ -45,6 +76,7 @@ func init() { "monitorresource", "monitorresources", ), + monitorResModelSets: NewModelSets(), } MonitorResourceManager.SetVirtualObject(MonitorResourceManager) RegistryResourceSync(NewGuestResourceSync()) @@ -56,9 +88,15 @@ func init() { RegistryResourceSync(NewStorageResourceSync()) } +func (manager *SMonitorResourceManager) GetModelSets() *MonitorResModelSets { + return manager.monitorResModelSets +} + type SMonitorResourceManager struct { db.SVirtualResourceBaseManager db.SEnabledResourceBaseManager + + monitorResModelSets *MonitorResModelSets } type SMonitorResource struct { @@ -70,22 +108,6 @@ type SMonitorResource struct { ResType string `width:"36" charset:"ascii" list:"user" update:"user" json:"res_type"` } -func (manager *SMonitorResourceManager) SyncResources(ctx context.Context, userCred mcclient.TokenCredential, - isStart bool) { - for _, sync := range GetResourceSyncMap() { - err := sync.SyncResources(ctx, userCred, jsonutils.NewDict()) - if err != nil { - log.Errorf("resType:%s SyncResources err:%v", sync.SyncType(), err) - } - } - err := CommonAlertManager.Run(ctx) - if err != nil { - log.Errorf("CommonAlertManager UpdateMonitorResourceJoint err:%v", err) - return - } - log.Infoln("====SMonitorResourceManager SyncResources End====") -} - func (manager *SMonitorResourceManager) GetMonitorResources(input monitor.MonitorResourceListInput) ([]SMonitorResource, error) { monitorResources := make([]SMonitorResource, 0) query := manager.Query() @@ -381,3 +403,95 @@ func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match return errors.NewAggregate(errs) } + +func (manager *SMonitorResourceManager) GetResourceObj(id string) (bool, jsonutils.JSONObject) { + for _, set := range manager.GetModelSets().ModelSetList() { + setRv := reflect.ValueOf(set) + for _, kRv := range setRv.MapKeys() { + if id == kRv.String() { + mRv := setRv.MapIndex(kRv) + if mRv.IsValid() { + return true, jsonutils.Marshal(mRv.Interface()) + } + } + } + } + return false, nil +} + +func (manager *SMonitorResourceManager) SyncResources(ctx context.Context, mss *MonitorResModelSets) error { + userCred := auth.AdminCredential() + errs := make([]error, 0) + log.Infoln("start sync") + for _, set := range mss.ModelSetList() { + setRv := reflect.ValueOf(set) + typ := manager.GetSetType(set) + log.Infof("Type: %s,length: %d", typ, len(setRv.MapKeys())) + for _, kRv := range setRv.MapKeys() { + mRv := setRv.MapIndex(kRv) + //log.Errorf("resID:%s", kRv.String()) + input := monitor.MonitorResourceListInput{ + ResId: []string{kRv.String()}, + } + res, err := MonitorResourceManager.GetMonitorResources(input) + if err != nil { + return errors.Wrap(err, "GetMonitorResources err") + } + if mRv.IsValid() { + obj := jsonutils.Marshal(mRv.Interface()) + if len(res) == 0 { + // no find to create + createData := newMonitorResourceCreateInput(obj, typ) + _, err = db.DoCreate(MonitorResourceManager, ctx, userCred, nil, createData, + userCred) + if err != nil { + name, _ := createData.GetString("name") + errs = append(errs, errors.Wrapf(err, "monitorResource:%s resType:%s DoCreate err", name, typ)) + } + continue + } + _, err = db.Update(&res[0], func() error { + obj.(*jsonutils.JSONDict).Remove("id") + (&res[0]).ResType = typ + obj.Unmarshal(&res[0]) + return nil + }) + if err != nil { + errs = append(errs, errors.Wrapf(err, "monitorResource:%s Update err", res[0].Name)) + continue + } + continue + } + if len(res) != 0 { + log.Infof("delete monitor resource,resId: %s,resType: %s", res[0].ResId, res[0].ResType) + err := (&res[0]).RealDelete(ctx, userCred) + if err != nil { + errs = append(errs, errors.Wrapf(err, "delete monitorResource:%s err", res[0].GetId())) + } + } + } + } + log.Infoln("SMonitorResourceManager SyncResources End") + err := CommonAlertManager.Run(ctx) + if err != nil { + log.Errorf("CommonAlertManager UpdateMonitorResourceJoint err:%v", err) + } + return errors.NewAggregate(errs) +} + +func (manager *SMonitorResourceManager) GetSetType(set apihelper.IModelSet) string { + if iset, ok := set.(IMonitorResModelSet); ok { + return iset.GetResType() + } + return "NONE" +} + +func newMonitorResourceCreateInput(input jsonutils.JSONObject, typ string) jsonutils.JSONObject { + monitorResource := jsonutils.DeepCopy(input).(*jsonutils.JSONDict) + id, _ := monitorResource.GetString("id") + monitorResource.Add(jsonutils.NewString(id), "res_id") + monitorResource.Remove("id") + monitorResource.Add(jsonutils.NewString(typ), "res_type") + + return monitorResource +} diff --git a/pkg/monitor/models/recordshield.go b/pkg/monitor/models/recordshield.go index 16da8774f5..174dd61fc8 100644 --- a/pkg/monitor/models/recordshield.go +++ b/pkg/monitor/models/recordshield.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models import ( diff --git a/pkg/monitor/models/unifiedmonitor.go b/pkg/monitor/models/unifiedmonitor.go index 232738c27c..aea796b127 100644 --- a/pkg/monitor/models/unifiedmonitor.go +++ b/pkg/monitor/models/unifiedmonitor.go @@ -244,11 +244,31 @@ func (self *SUnifiedMonitorManager) PerformQuery(ctx context.Context, userCred m return jsonutils.NewDict(), err } + if len(inputQuery.Soffset) != 0 && len(inputQuery.Slimit) != 0 { + seriesTotal := self.fillSearchSeriesTotalQuery(*inputQuery.MetricQuery[0]) + rtn.SeriesTotal = seriesTotal + } + setSerieRowName(&rtn.Series, groupByTag) fillSerieTags(&rtn.Series) return jsonutils.Marshal(rtn), nil } +func (self *SUnifiedMonitorManager) fillSearchSeriesTotalQuery(fork monitor.AlertQuery) int64 { + newGroupByPart := make([]monitor.MetricQueryPart, 0) + newGroupByPart = append(newGroupByPart, fork.Model.GroupBy[0]) + fork.Model.GroupBy = newGroupByPart + forkInputQury := new(monitor.MetricInputQuery) + forkInputQury.MetricQuery = []*monitor.AlertQuery{&fork} + rtn, err := doQuery(*forkInputQury) + if err != nil { + log.Errorf("exec forkInputQury err:%v", err) + return 0 + } + log.Errorf("series len:%d", len(rtn.Series)) + return int64(len(rtn.Series)) +} + func (self *SUnifiedMonitorManager) handleDataPreSignature(ctx context.Context, data jsonutils.JSONObject) { scope, _ := data.GetString("scope") isIdentityName, _ := data.Bool("identity_name") @@ -340,6 +360,12 @@ func setDefaultValue(query *monitor.AlertQuery, inputQuery *monitor.MetricInputQ Params: []string{"none"}, }) } + if len(inputQuery.Slimit) != 0 && len(inputQuery.Soffset) != 0 { + query.Model.GroupBy = append(query.Model.GroupBy, + monitor.MetricQueryPart{Type: "slimit", Params: []string{inputQuery.Slimit}}, + monitor.MetricQueryPart{Type: "soffset", Params: []string{inputQuery.Soffset}}, + ) + } if query.Model.Database == "" { database := "" diff --git a/pkg/monitor/options/options.go b/pkg/monitor/options/options.go index c6a15917d6..b79797af3b 100644 --- a/pkg/monitor/options/options.go +++ b/pkg/monitor/options/options.go @@ -30,6 +30,11 @@ type AlerterOptions struct { InitScopeSuggestConfigIntervalSeconds int `help:"internal to init scope suggest configs" default:"900"` InitAlertResourceAdminRoleUsersIntervalSeconds int `help:"internal to init alert resource admin role users " default:"3600"` MonitorResourceSyncIntervalSeconds int `help:"internal to sync monitor resource,unit: h " default:"1"` + + APISyncInterval int `default:"3600"` + APIListBatchSize int `default:"1024"` + + WorkerCheckInterval int `default:"180"` } var ( diff --git a/pkg/monitor/service/service.go b/pkg/monitor/service/service.go index 5de287e3ce..bbe94411a5 100644 --- a/pkg/monitor/service/service.go +++ b/pkg/monitor/service/service.go @@ -40,6 +40,7 @@ import ( "yunion.io/x/onecloud/pkg/monitor/subscriptionmodel" _ "yunion.io/x/onecloud/pkg/monitor/tasks" _ "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/influxdb" + "yunion.io/x/onecloud/pkg/monitor/worker" ) func StartService() { @@ -66,15 +67,22 @@ func StartService() { cron.AddJobAtIntervalsWithStartRun("InitAlertResourceAdminRoleUsers", time.Duration(opts.InitAlertResourceAdminRoleUsersIntervalSeconds)*time.Second, models.GetAlertResourceManager().GetAdminRoleUsers, true) cron.AddJobEveryFewDays("DeleteRecordsOfThirtyDaysAgoRecords", 1, 0, 0, 0, models.AlertRecordManager.DeleteRecordsOfThirtyDaysAgo, false) - cron.AddJobAtIntervalsWithStartRun("MonitorResourceSync", time.Duration(opts.MonitorResourceSyncIntervalSeconds)*time.Minute*60, models.MonitorResourceManager.SyncResources, true) + //cron.AddJobAtIntervalsWithStartRun("MonitorResourceSync", time.Duration(opts.MonitorResourceSyncIntervalSeconds)*time.Minute*60, models.MonitorResourceManager.SyncResources, true) cron.Start() defer cron.Stop() subscriptionmodel.SubscriptionManager.AddSubscription() models.CommonAlertManager.SetSubscriptionManager(subscriptionmodel.SubscriptionManager) - InitInfluxDBSubscriptionHandlers(app, baseOpts) + worker := worker.NewWorker(opts) + if worker == nil { + log.Fatalf("new worker failed") + } + go worker.Start(context.Background()) + //common_app.ServeForever(app, baseOpts) + InitInfluxDBSubscriptionHandlers(app, baseOpts) + } func startServices() { diff --git a/pkg/monitor/tsdb/driver/influxdb/influxdb.go b/pkg/monitor/tsdb/driver/influxdb/influxdb.go index 22da7f5d29..438fbf68ca 100644 --- a/pkg/monitor/tsdb/driver/influxdb/influxdb.go +++ b/pkg/monitor/tsdb/driver/influxdb/influxdb.go @@ -94,6 +94,7 @@ func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, t } dsInfo.Database = db + log.Errorf("rawQuery:%s", rawQuery) req, err := e.createRequest(dsInfo, rawQuery) if err != nil { return nil, err diff --git a/pkg/monitor/tsdb/driver/influxdb/query.go b/pkg/monitor/tsdb/driver/influxdb/query.go index 35069b2787..1d4e42e530 100644 --- a/pkg/monitor/tsdb/driver/influxdb/query.go +++ b/pkg/monitor/tsdb/driver/influxdb/query.go @@ -166,7 +166,7 @@ func (query *Query) renderGroupBy(queryContext *tsdb.TsdbQuery) string { groupBy += " GROUP BY" } - if i > 0 && group.Type != "fill" { + if i > 0 && utils.IsInStringArray(group.Type, []string{"field", "time", "tag"}) { groupBy += ", " //fill is so very special. fill is a creep, fill is a weirdo } else { groupBy += " " diff --git a/pkg/monitor/tsdb/driver/influxdb/query_part.go b/pkg/monitor/tsdb/driver/influxdb/query_part.go index f947bf44b5..e53e521921 100644 --- a/pkg/monitor/tsdb/driver/influxdb/query_part.go +++ b/pkg/monitor/tsdb/driver/influxdb/query_part.go @@ -111,6 +111,8 @@ func init() { renders["math"] = QueryDefinition{Renderer: suffixRenderer} renders["alias"] = QueryDefinition{Renderer: aliasRenderer} + renders["slimit"] = QueryDefinition{Renderer: typeRenderer} + renders["soffset"] = QueryDefinition{Renderer: typeRenderer} } func fieldRenderer(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string { @@ -173,6 +175,10 @@ func aliasRenderer(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, inne return fmt.Sprintf(`%s AS "%s"`, innerExpr, part.Params[0]) } +func typeRenderer(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string { + return fmt.Sprintf(" %s %s", part.Type, part.Params[0]) +} + func (r QueryDefinition) Render(query *Query, queryCtx *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string { return r.Renderer(query, queryCtx, part, innerExpr) } diff --git a/pkg/monitor/worker/doc.go b/pkg/monitor/worker/doc.go new file mode 100644 index 0000000000..ff10a288ad --- /dev/null +++ b/pkg/monitor/worker/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package worker // import "yunion.io/x/onecloud/pkg/monitor/worker" diff --git a/pkg/monitor/worker/worker.go b/pkg/monitor/worker/worker.go new file mode 100644 index 0000000000..196182fa5b --- /dev/null +++ b/pkg/monitor/worker/worker.go @@ -0,0 +1,97 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package worker + +import ( + "context" + "runtime" + "runtime/debug" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apihelper" + "yunion.io/x/onecloud/pkg/monitor/models" + "yunion.io/x/onecloud/pkg/monitor/options" + "yunion.io/x/onecloud/pkg/vpcagent/worker" +) + +type Worker struct { + opts *options.AlerterOptions + + apih *apihelper.APIHelper +} + +func NewWorker(opts *options.AlerterOptions) worker.IWorker { + modelSets := models.MonitorResourceManager.GetModelSets() + apiOpts := &apihelper.Options{ + CommonOptions: opts.CommonOptions, + SyncInterval: opts.APISyncInterval, + ListBatchSize: opts.APIListBatchSize, + IncludeDetails: true, + InCludeOtherCloudEnv: true, + } + apih, err := apihelper.NewAPIHelper(apiOpts, modelSets) + if err != nil { + return nil + } + w := &Worker{ + opts: opts, + apih: apih, + } + return w +} + +func (w *Worker) Start(ctx context.Context) { + defer func() { + log.Infoln("monitor resource: worker bye") + }() + + log.Infoln("start to get api Resource") + go w.apih.Start(ctx) + + var mss *models.MonitorResModelSets + for { + select { + case imss := <-w.apih.ModelSets(): + log.Infof("monitorRes: got new data from api helper") + mss = imss.(*models.MonitorResModelSets) + if err := w.run(ctx, mss); err != nil { + log.Errorf("monitorResWork err: %v", err) + } + case <-ctx.Done(): + return + } + } +} + +func (w *Worker) run(ctx context.Context, mss *models.MonitorResModelSets) (err error) { + defer func() { + if panicVal := recover(); panicVal != nil { + if panicErr, ok := panicVal.(runtime.Error); ok { + err = errors.Wrap(panicErr, string(debug.Stack())) + } else if panicErr, ok := panicVal.(error); ok { + err = panicErr + } else { + panic(panicVal) + } + } + }() + err = models.MonitorResourceManager.SyncResources(ctx, mss) + if err != nil { + return err + } + return nil +} diff --git a/pkg/multicloud/azure/azure.go b/pkg/multicloud/azure/azure.go index aec0080a01..c20c6e86bd 100644 --- a/pkg/multicloud/azure/azure.go +++ b/pkg/multicloud/azure/azure.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright 2019 Yunion // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/multicloud/cloudpods/doc.go b/pkg/multicloud/cloudpods/doc.go index dc69decba7..7b24c902e4 100644 --- a/pkg/multicloud/cloudpods/doc.go +++ b/pkg/multicloud/cloudpods/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cloudpods // import "yunion.io/x/onecloud/pkg/multicloud/cloudpods" diff --git a/pkg/multicloud/cloudpods/provider/doc.go b/pkg/multicloud/cloudpods/provider/doc.go index c822b3a82d..36e967dc9e 100644 --- a/pkg/multicloud/cloudpods/provider/doc.go +++ b/pkg/multicloud/cloudpods/provider/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package provider // import "yunion.io/x/onecloud/pkg/multicloud/cloudpods/provider" diff --git a/pkg/multicloud/cloudpods/shell/doc.go b/pkg/multicloud/cloudpods/shell/doc.go index cedd7d802b..165c93f2ee 100644 --- a/pkg/multicloud/cloudpods/shell/doc.go +++ b/pkg/multicloud/cloudpods/shell/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package shell // import "yunion.io/x/onecloud/pkg/multicloud/cloudpods/shell" diff --git a/pkg/multicloud/huaweistack/shell/endpoints.go b/pkg/multicloud/huaweistack/shell/endpoints.go index a00f1ea0f8..251dae64b3 100644 --- a/pkg/multicloud/huaweistack/shell/endpoints.go +++ b/pkg/multicloud/huaweistack/shell/endpoints.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package shell import ( diff --git a/pkg/multicloud/huaweistack/shell/service.go b/pkg/multicloud/huaweistack/shell/service.go index b7ef16b712..d3c96972c4 100644 --- a/pkg/multicloud/huaweistack/shell/service.go +++ b/pkg/multicloud/huaweistack/shell/service.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package shell import (