feat(cloudmon): add cloudmon component

build/docker/Dockerfile.cloudmon

@@ -0,0 +1,8 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+RUN mkdir -p /etc/yunion/data/
+
+ADD   ./_output/alpine-build/bin/cloudmon \
+      /opt/yunion/bin/

cmd/climc/shell/itsm/itsm_extra_process_instance.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package itsm
 
 import (

cmd/climc/shell/itsm/itsm_extra_user.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package itsm
 
 import (

cmd/climc/shell/meter/bills_dimension.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package meter
 
 import (

cmd/climc/shell/monitor/alertdashboard.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/alertpanel.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/alertrecord.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/commonalert.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/commonalertmetric.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/commonalertmetricfield.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/monitor/monitor_resource.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/climc/shell/suggestion/analysispredict.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package suggestion
 
 import (

cmd/cloudmon/main.go

@@ -0,0 +1,169 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/version"
+
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClientSession(options *options.CloudMonOptions) (*mcclient.ClientSession, error) {
+	if len(options.AuthURL) == 0 {
+		return nil, errors.Error("empty auth_url")
+	}
+	if len(options.AdminUser) == 0 {
+		return nil, errors.Error("empty admin_user")
+	}
+	if len(options.AdminPassword) == 0 {
+		return nil, errors.Error("empty admin_password")
+	}
+	if len(options.AdminProject) == 0 {
+		return nil, errors.Error("empty admin_project")
+	}
+
+	client := mcclient.NewClient(
+		options.AuthURL,
+		options.Timeout,
+		options.Debug,
+		options.Insecure,
+		options.CertFile,
+		options.KeyFile,
+	)
+
+	token, err := client.AuthenticateWithSource(
+		options.AdminUser,
+		options.AdminPassword,
+		options.AdminDomain,
+		options.AdminProject,
+		options.AdminProjectDomain,
+		mcclient.AuthSourceAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	session := client.NewSession(
+		context.Background(),
+		options.Region,
+		"",
+		options.EndpointType,
+		token,
+		options.ApiVersion)
+
+	return session, nil
+}
+
+func main() {
+	parser, err := options.GetArgumentParser()
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	err = parser.ParseArgs2(os.Args[1:], false, false)
+
+	opts := parser.Options().(*options.CloudMonOptions)
+
+	if opts.Help {
+		fmt.Println(parser.HelpString())
+		os.Exit(0)
+	}
+
+	if opts.Version {
+		fmt.Println(version.GetJsonString())
+		os.Exit(0)
+	}
+
+	if len(opts.Config) == 0 {
+		for _, p := range []string{"./etc", "/etc/yunion"} {
+			confTmp := path.Join(p, "cloudmon.conf")
+			if _, err := os.Stat(confTmp); err == nil {
+				opts.Config = confTmp
+				break
+			}
+		}
+	}
+
+	if len(opts.Config) > 0 {
+		err := parser.ParseFile(opts.Config)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+	}
+
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	parser.SetDefault()
+
+	subcmd := parser.GetSubcommand()
+	subparser := subcmd.GetSubParser()
+	if err != nil {
+		if subparser != nil {
+			fmt.Print(subparser.Usage())
+		} else {
+			fmt.Print(parser.Usage())
+		}
+		showErrorAndExit(err)
+	}
+
+	suboptions := subparser.Options()
+	if opts.SUBCOMMAND == "help" {
+		err = subcmd.Invoke(suboptions)
+	} else {
+		timout := suboptions.(*common.ReportOptions).Timeout
+		endChan := make(chan int, 1)
+		go func() {
+			ticker := time.NewTicker(time.Duration(timout) * time.Second)
+			for {
+				select {
+				case <-ticker.C:
+					log.Errorf("cmd: %s,provider: %v,end due to timeout: %d s", opts.SUBCOMMAND,
+						suboptions.(*common.ReportOptions).Provider, suboptions.(*common.ReportOptions).Timeout)
+					os.Exit(3)
+				case <-endChan:
+				}
+			}
+		}()
+
+		var session *mcclient.ClientSession
+		session, err = newClientSession(opts)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+		err = subcmd.Invoke(session, suboptions)
+		endChan <- 1
+	}
+	if err != nil {
+		showErrorAndExit(err)
+	}
+}

cmd/ecloudcli/main.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/fetcherfs/dir.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/fetcherfs/fs.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/fetcherfs/main.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/fetcherfs/options.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/telegraf-raid-plugin/main.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

go.mod

@@ -115,6 +115,7 @@ require (
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.5.1
+	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.24
 	github.com/tinylib/msgp v1.1.0 // indirect

go.sum

@@ -580,6 +580,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e h1:nt2877sKfojlHCTOBXbpWjBkuWKritFaGIfgQwbQUls=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e/go.mod h1:B4+Kq1u5FlULTjFSM707Q6e/cOHFv0z/6QRoxubDIQ8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo=

pkg/apigateway/handler/imageutils.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package handler
 
 import (

pkg/apis/ansibleserver/doc.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

pkg/apis/cloudproxy/proxy_matches.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cloudproxy
 
 import "yunion.io/x/onecloud/pkg/apis"

pkg/apis/compute/guest_metadata.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/host/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package host // import "yunion.io/x/onecloud/pkg/apis/host"

pkg/apis/monitor/alertdashboard.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/apis/monitor/alertpanel.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import "yunion.io/x/onecloud/pkg/apis"

pkg/apis/monitor/alertquery_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 var (

pkg/apis/monitor/alertrecord.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import "yunion.io/x/onecloud/pkg/apis"

pkg/apis/monitor/commalert.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 const (

pkg/apis/monitor/metric.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import "yunion.io/x/onecloud/pkg/apis"

pkg/apis/monitor/monitor_resource.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/apis/monitor/monitor_resource_alert.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

pkg/apis/monitor/unifiedmonitor_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 var (

pkg/apis/notify/robot.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package notify
 
 import "yunion.io/x/onecloud/pkg/apis"

pkg/baremetal/utils/raid/drivers/executor.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package drivers
 
 import (

pkg/baremetal/utils/uefi/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package uefi // import "yunion.io/x/onecloud/pkg/baremetal/utils/uefi"

pkg/cloudcommon/db/archs.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

pkg/cloudcommon/db/auto_delete.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

pkg/cloudcommon/db/i18n.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package db
 
 import (

pkg/cloudcommon/notifyclient/notify_internal.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package notifyclient
 
 import (

pkg/cloudcommon/notifyclient/template.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package notifyclient
 
 import (

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryreport.go

@@ -0,0 +1,133 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alertrecordhistorymon
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/timeutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAlertRecordHistoryFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAlertRecordHistoryFactory struct {
+}
+
+func (self *SAlertRecordHistoryFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions,
+	operatorType string) common.ICloudReport {
+	return &SAlertRecordHistoryReport{
+		common.CloudReportBase{
+			SProvider: nil,
+			Session:   session,
+			Args:      args,
+			Operator:  string(common.ALERT_RECORD),
+		},
+	}
+}
+
+func (S *SAlertRecordHistoryFactory) GetId() string {
+	return string(common.ALERT_RECORD)
+}
+
+type SAlertRecordHistoryReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAlertRecordHistoryReport) Report() error {
+	alerts, err := self.getMonitorCommonAlert()
+	if err != nil {
+		return err
+	}
+	errs := make([]error, 0)
+	recordGroup, _ := errgroup.WithContext(context.Background())
+	count := 0
+	for i, _ := range alerts {
+		tmp := alerts[i]
+		count++
+		recordGroup.Go(func() error {
+			alert_id, _ := tmp.GetString("id")
+			alertRecords, err := self.getAlertRecordsByAlertId(alert_id)
+			if err != nil {
+				return errors.Wrapf(err, "getAlertRecordsByAlertId:%s error", alert_id)
+			}
+			return self.collectMetric(alertRecords)
+		})
+		if count == 4 {
+			err := recordGroup.Wait()
+			if err != nil {
+				errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error"))
+			}
+			count = 0
+		}
+	}
+	err = recordGroup.Wait()
+	if err != nil {
+		errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error"))
+	}
+	return errors.NewAggregate(errs)
+}
+
+func (self *SAlertRecordHistoryReport) getMonitorCommonAlert() ([]jsonutils.JSONObject, error) {
+	alerts := make([]jsonutils.JSONObject, 0)
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), common.KEY_LIMIT)
+	//query.Add(jsonutils.NewBool(true), common.DETAILS)
+	query.Add(jsonutils.NewString("system"), "scope")
+
+	alerts, err := self.ListAllResource(modules.CommonAlertManager, query)
+	if err != nil {
+		return nil, errors.Wrap(err, "getMonitorCommonAlert error")
+	}
+	return alerts, nil
+}
+
+func (self *SAlertRecordHistoryReport) getAlertRecordsByAlertId(id string) ([]jsonutils.JSONObject, error) {
+	now := time.Now().UTC()
+	period64, err := strconv.ParseInt(self.Args.Interval, 10, 8)
+	if err != nil {
+		return nil, err
+	}
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), common.KEY_LIMIT)
+	query.Add(jsonutils.NewBool(true), common.DETAILS)
+	query.Add(jsonutils.NewString("system"), "scope")
+	query.Add(jsonutils.NewString(id), "alert_id")
+	query.Add(jsonutils.NewString("alerting"), "state")
+	query.Add(jsonutils.NewString(fmt.Sprintf(`created_at.between("%s","%s")`,
+		now.Add(-time.Hour*24*time.Duration(period64)).Format(timeutils.MysqlTimeFormat),
+		now.Format(timeutils.MysqlTimeFormat))),
+		"filter")
+
+	alertRecords, err := self.ListAllResource(modules.AlertRecordManager, query)
+	if err != nil {
+		return nil, err
+	}
+	return alertRecords, nil
+}

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryservice.go

@@ -0,0 +1,54 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alertrecordhistorymon
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAlertRecordHistoryReport) collectMetric(alertRecords []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	matchRecord := self.getMaxEvalMatchOfAlertRecord(alertRecords)
+	if matchRecord == nil {
+		return nil
+	}
+
+	metric, err := self.NewMetricFromJson(matchRecord)
+	if err != nil {
+		return err
+	}
+	metric.Timestamp = time.Now()
+	metric.Name = ALERT_RECORD_HISTORY_MEASUREMENT
+	dataList = append(dataList, metric)
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, ALERT_RECORD_HISTORY_DATABASE)
+}
+
+func (self *SAlertRecordHistoryReport) getMaxEvalMatchOfAlertRecord(alertRecords []jsonutils.JSONObject) jsonutils.JSONObject {
+	var matchRecord jsonutils.JSONObject
+	maxCount := int64(0)
+	for _, record := range alertRecords {
+		resNum, _ := record.Int("res_num")
+		if resNum > maxCount {
+			maxCount = resNum
+			matchRecord = record
+		}
+	}
+	return matchRecord
+}

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryutil.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alertrecordhistorymon
+
+const (
+	ALERT_RECORD_HISTORY_FIELD       = "res_num"
+	ALERT_RECORD_HISTORY_MEASUREMENT = "alert_record_history"
+	ALERT_RECORD_HISTORY_DATABASE    = "monitor"
+)

pkg/cloudmon/collectors/alertrecordhistorymon/doc.go

@@ -0,0 +1 @@
+package alertrecordhistorymon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon"

pkg/cloudmon/collectors/alertrecordhistoryservice.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-alertrecord", "Report alertrecord history", reportAlertrecord)
+}
+
+func reportAlertrecord(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCustomizeCloudMetric(string(common.ALERT_RECORD), session, args)
+}

pkg/cloudmon/collectors/alimon/aliUtils.go

@@ -0,0 +1,173 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alimon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var aliMetricSpecs = map[string][]string{
+	"CPUUtilization":  {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"InternetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
+	"IntranetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
+	"InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
+	"IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
+	"DiskReadBPS":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"DiskWriteBPS":    {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"DiskReadIOPS":    {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"DiskWriteIOPS":   {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}
+var aliRdsMetricSpecs = map[string][]string{
+	"CpuUsage":                {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"MemoryUsage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"MySQL_NetworkInNew":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
+	"MySQL_NetworkOutNew":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
+	"DiskUsage":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"SQLServer_NetworkInNew":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
+	"SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
+	"ConnectionUsage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE},
+}
+var aliRedisMetricSpecs = map[string][]string{
+	"CpuUsage":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"MemoryUsage":    {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"IntranetIn":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"IntranetOut":    {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"UsedQPS":        {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"Keys":           {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"ExpiredKeys":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"UsedMemory":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+var aliOSSMetricSpecs = map[string][]string{
+	"InternetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET},
+	"InternetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET},
+	"IntranetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET},
+	"IntranetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET},
+	"GetObjectE2eLatency":  {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
+	"PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST},
+	"GetObjectCount":       {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
+	"PostObjectCount":      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST},
+	"ServerErrorCount":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
+}
+var aliElbMetricSpecs = map[string][]string{
+	"InstanceTrafficRX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
+	"InstanceTrafficTX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
+	"InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
+	"InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
+	"InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
+	"InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
+}

pkg/cloudmon/collectors/alimon/alicloudreport.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alimon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAliCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAliCloudReportFactory struct {
+}
+
+func (self *SAliCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SAliCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SAliCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_ALIYUN
+}
+
+type SAliCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAliCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/alimon/aliservice.go

@@ -0,0 +1,315 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alimon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/aliyun"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAliCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	redisDeployType := make(map[string]string)
+	for _, server := range servers {
+		local_category, err := server.GetString("local_category")
+		if err != nil {
+			return err
+		}
+		switch local_category {
+		case "single":
+			redisDeployType["single"] = "Standard"
+		case "master":
+			redisDeployType["master"] = "Standard"
+		case "cluster":
+			redisDeployType["cluster"] = "Sharding"
+		case "rwsplit":
+			redisDeployType["rwsplit"] = "Splitrw"
+		}
+
+	}
+	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
+		for _, pre := range redisDeployType {
+			rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "")
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			if len(rtnArray) > 0 {
+				for _, rtnMetric := range rtnArray {
+					for _, server := range servers {
+						external_id, _ := server.GetString("external_id")
+						if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+							serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+							node_id, _ := rtnMetric.GetString("nodeId")
+							serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id})
+							if err != nil {
+								return err
+							}
+							dataList = append(dataList, serverMetric)
+						}
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliRdsMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+
+				}
+
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliOSSMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					name, _ := server.GetString("name")
+					if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name {
+						serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName,
+							influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliElbMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(UNIT_AVERAGE)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	metric.Name = measurement
+	return metric, nil
+}
+
+func (self *SAliCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(metricName)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	metric.Name = measurement
+	return metric, nil
+}

pkg/cloudmon/collectors/alimon/doc.go

@@ -0,0 +1 @@
+package alimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon"

pkg/cloudmon/collectors/apsaramon/apsaracloudreport.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apsaramon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SApsaraCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SApsaraCloudReportFactory struct {
+}
+
+func (self *SApsaraCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SApsaraCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SApsaraCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_APSARA
+}
+
+type SApsaraCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SApsaraCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/apsaramon/apsaraservice.go

@@ -0,0 +1,315 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apsaramon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/apsara"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SApsaraCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	redisDeployType := make(map[string]string)
+	for _, server := range servers {
+		local_category, err := server.GetString("local_category")
+		if err != nil {
+			return err
+		}
+		switch local_category {
+		case "single":
+			redisDeployType["single"] = "Standard"
+		case "master":
+			redisDeployType["master"] = "Standard"
+		case "cluster":
+			redisDeployType["cluster"] = "Sharding"
+		case "rwsplit":
+			redisDeployType["rwsplit"] = "Splitrw"
+		}
+
+	}
+	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
+		for _, pre := range redisDeployType {
+			rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "")
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			if len(rtnArray) > 0 {
+				for _, rtnMetric := range rtnArray {
+					for _, server := range servers {
+						external_id, _ := server.GetString("external_id")
+						if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+							serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+							node_id, _ := rtnMetric.GetString("nodeId")
+							serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id})
+							if err != nil {
+								return err
+							}
+							dataList = append(dataList, serverMetric)
+						}
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliRdsMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+
+				}
+
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliOSSMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					name, _ := server.GetString("name")
+					if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name {
+						serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName,
+							influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliElbMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(UNIT_AVERAGE)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	metric.Name = measurement
+	return metric, nil
+}
+
+func (self *SApsaraCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(metricName)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	metric.Name = measurement
+	return metric, nil
+}

pkg/cloudmon/collectors/apsaramon/apsarautils.go

@@ -0,0 +1,173 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apsaramon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var aliMetricSpecs = map[string][]string{
+	"CPUUtilization":  {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"InternetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
+	"IntranetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
+	"InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
+	"IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
+	"DiskReadBPS":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"DiskWriteBPS":    {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"DiskReadIOPS":    {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"DiskWriteIOPS":   {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}
+var aliRdsMetricSpecs = map[string][]string{
+	"CpuUsage":                {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"MemoryUsage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"MySQL_NetworkInNew":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
+	"MySQL_NetworkOutNew":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
+	"DiskUsage":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"SQLServer_NetworkInNew":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
+	"SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
+	"ConnectionUsage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE},
+}
+var aliRedisMetricSpecs = map[string][]string{
+	"CpuUsage":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"MemoryUsage":    {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"IntranetIn":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"IntranetOut":    {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"UsedQPS":        {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"Keys":           {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"ExpiredKeys":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"UsedMemory":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+var aliOSSMetricSpecs = map[string][]string{
+	"InternetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET},
+	"InternetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET},
+	"IntranetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET},
+	"IntranetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET},
+	"GetObjectE2eLatency":  {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
+	"PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST},
+	"GetObjectCount":       {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
+	"PostObjectCount":      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST},
+	"ServerErrorCount":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
+}
+var aliElbMetricSpecs = map[string][]string{
+	"InstanceTrafficRX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
+	"InstanceTrafficTX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
+	"InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
+	"InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
+	"InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
+	"InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
+}

pkg/cloudmon/collectors/apsaramon/doc.go

@@ -0,0 +1 @@
+package apsaramon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"

pkg/cloudmon/collectors/awsmon/awscloudreport.go

@@ -0,0 +1,85 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awsmon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAwsCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAwsCloudReportFactory struct {
+}
+
+func (self *SAwsCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SAwsCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SAwsCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_AWS
+}
+
+type SAwsCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAwsCloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self)
+			//err = self.collectRegionMetricOfHost(region, servers)
+			//case "redis":
+			//	err = self.collectRegionMetricOfRedis(region, servers)
+			//case "rds":
+			//	err = self.collectRegionMetricOfRds(region, servers)
+			//case "oss":
+			//	err = self.collectRegionMetricOfOss(region, servers)
+			//case "elb":
+			//	err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/awsmon/awsservice.go

@@ -0,0 +1,125 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awsmon
+
+import (
+	"strconv"
+	"strings"
+
+	aws_sdk "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/aws"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAwsCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	var err error
+	switch self.Operator {
+	case string(common.SERVER):
+		err = self.collectRegionMetricOfHost(region, servers)
+	}
+	return err
+}
+
+func (self *SAwsCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	awsRegion := region.(*aws.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range awsMetricSpecs {
+		for i, server := range servers {
+			external_id, _ := servers[i].GetString("external_id")
+			rtnArray, err := awsRegion.GetMonitorData(metricName, "AWS/EC2", external_id, since, until)
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			if len(rtnArray.Datapoints) > 0 {
+				for _, dataPoint := range rtnArray.Datapoints {
+					metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, metric)
+					serverMetric, err := self.collectMetricFromThisServerForAws(server, dataPoint, influxDbSpecs)
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, serverMetric)
+				}
+
+			}
+		}
+		err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+	return nil
+}
+
+func (self *SAwsCloudReport) collectMetricFromThisServerForAws(server jsonutils.JSONObject, datapoint *cloudwatch.Datapoint,
+	influxDbSpecs []string) (metric influxdb.SMetricData, err error) {
+	metric, err = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return metric, err
+	}
+	metric.Timestamp = aws_sdk.TimeValue(datapoint.Timestamp)
+	fieldValue := aws_sdk.Float64Value(datapoint.Average)
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	// 300:AWS metric collection period is 5m
+	if influxDbSpecs[1] == UNIT_COUNT {
+		fieldValue = (fieldValue / float64(300))
+	}
+	if influxDbSpecs[1] == UNIT_MEM {
+		fieldValue = (fieldValue / float64(300) * 8)
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{Key: common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "=")})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err != nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{Key: pairsKey,
+			Value: strconv.FormatFloat(fieldValue, 'E', -1, 64)})
+	} else {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{Key: "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10)},
+			influxdb.SKeyValue{Key: pairsKey,
+				Value: strconv.FormatFloat(fieldValue, 'E', -1, 64)})
+	}
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	metric.Name = measurement
+	return metric, nil
+}

pkg/cloudmon/collectors/awsmon/awsutils.go

@@ -0,0 +1,139 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awsmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_BPS_EBS   = INFLUXDB_FIELD_DISK_READ_BPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_DISK_WRITE_BPS_EBS  = INFLUXDB_FIELD_DISK_WRITE_BPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_DISK_READ_IOPS_EBS  = INFLUXDB_FIELD_DISK_READ_IOPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS_EBS = INFLUXDB_FIELD_DISK_WRITE_IOPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var awsMetricSpecs = map[string][]string{
+	"CPUUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"DiskReadOps":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"EBSReadOps":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_READ_IOPS_EBS},
+	"DiskWriteOps":   {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+	"EBSWriteOps":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_WRITE_IOPS_EBS},
+	"DiskReadBytes":  {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"EBSReadBytes":   {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS_EBS},
+	"DiskWriteBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"EBSWriteBytes":  {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS_EBS},
+	"NetworkIn":      {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"NetworkOut":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+}

pkg/cloudmon/collectors/awsmon/doc.go

@@ -0,0 +1 @@
+package awsmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon"

pkg/cloudmon/collectors/azuremon/azurereport.go

@@ -0,0 +1,85 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azuremon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAzureCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAzureCloudReportFactory struct {
+}
+
+func (self *SAzureCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SAzureCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SAzureCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_AZURE
+}
+
+type SAzureCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAzureCloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self)
+			//err = self.collectRegionMetricOfHost(region, servers)
+			//case "redis":
+			//	err = self.collectRegionMetricOfRedis(region, servers)
+			//case "rds":
+			//	err = self.collectRegionMetricOfRds(region, servers)
+			//case "oss":
+			//	err = self.collectRegionMetricOfOss(region, servers)
+			//case "elb":
+			//	err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/azuremon/azureservice.go

@@ -0,0 +1,142 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azuremon
+
+import (
+	"strconv"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/azure"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAzureCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	return self.collectRegionMetricOfHost(region, servers)
+}
+
+func (self *SAzureCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	azureReg := region.(*azure.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, err := server.GetString("external_id")
+		if err != nil {
+			continue
+		}
+		metricNameArr := make([]string, 0)
+		for metricName, _ := range azureMetricSpecs {
+			metricNameArr = append(metricNameArr, metricName)
+		}
+		metricNames := strings.Join(metricNameArr, ",")
+		rtnMetrics, err := azureReg.GetMonitorData(metricNames, "Microsoft.Compute/virtualMachines", external_id, since, until)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if rtnMetrics == nil || rtnMetrics.Value == nil {
+			continue
+		}
+		for metricName, influxDbSpecs := range azureMetricSpecs {
+			for _, value := range *rtnMetrics.Value {
+				if value.Name.LocalizedValue != nil {
+					if metricName == *(value.Name.LocalizedValue) {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						if value.Timeseries != nil {
+							for _, timeserie := range *value.Timeseries {
+								serverMetric, err := self.collectMetricFromThisServer(server, timeserie, influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric...)
+							}
+						}
+					}
+				}
+			}
+		}
+		err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+		if err != nil {
+			log.Errorln(err)
+		}
+		dataList = dataList[:0]
+	}
+	return nil
+}
+
+func (self *SAzureCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	rtnMetric azure.TimeSeriesElement, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	for _, data := range *rtnMetric.Data {
+		metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		if err != nil {
+			return nil, err
+		}
+		if data.TimeStamp != nil {
+			metric.Timestamp = *data.TimeStamp
+			if data.Average != nil {
+				//根据条件拼装metric的tag和metirc信息
+				fieldValue := *data.Average
+				influxDbSpec := influxDbSpecs[2]
+				measurement := common.SubstringBefore(influxDbSpec, ".")
+				metric.Name = measurement
+				var pairsKey string
+				if strings.Contains(influxDbSpec, ",") {
+					pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+				} else {
+					pairsKey = common.SubstringAfter(influxDbSpec, ".")
+				}
+				if influxDbSpecs[1] == UNIT_MEM {
+					fieldValue = fieldValue * 8 / PERIOD
+				}
+				tag := common.SubstringAfter(influxDbSpec, ",")
+				if tag != "" && strings.Contains(influxDbSpec, "=") {
+					metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+						Key:   common.SubstringBefore(tag, "="),
+						Value: common.SubstringAfter(tag, "="),
+					})
+				}
+				cpu_cout, err := server.Get("vcpu_count")
+				if err == nil {
+					metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+						Key:   "cpu_count",
+						Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+					})
+				}
+				metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+					Key:   pairsKey,
+					Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+				})
+				self.AddMetricTag(&metric, common.OtherVmTags)
+				datas = append(datas, metric)
+			}
+		}
+	}
+
+	return datas, nil
+}

pkg/cloudmon/collectors/azuremon/azureutils.go

@@ -0,0 +1,131 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azuremon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var azureMetricSpecs = map[string][]string{
+	"Percentage CPU":            {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"Network In Total":          {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"Network Out Total":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+	"Disk Read Bytes":           {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"Disk Write Bytes":          {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"Disk Read Operations/Sec":  {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"Disk Write Operations/Sec": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_WRITE_BPS},
+}

pkg/cloudmon/collectors/azuremon/doc.go

@@ -0,0 +1 @@
+package azuremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon"

pkg/cloudmon/collectors/bucket.go

@@ -0,0 +1,88 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BucketStatsOptions struct {
+	Debug bool `help:"debug"`
+}
+
+func bucketStatsCollect(s *mcclient.ClientSession, args *BucketStatsOptions) error {
+	metrics := make([]influxdb.SMetricData, 0)
+	listAll(s, modules.Buckets.List, nil,
+		func(data jsonutils.JSONObject) error {
+			m, err := collectBucket(s, data)
+			if err != nil {
+				return err
+			}
+			metrics = append(metrics, m)
+			return nil
+		},
+	)
+	return sendMetrics(s, metrics, args.Debug)
+}
+
+func collectBucket(s *mcclient.ClientSession, bucket jsonutils.JSONObject) (influxdb.SMetricData, error) {
+	metric := influxdb.SMetricData{}
+	bucketId, _ := bucket.GetString("id")
+	if len(bucketId) == 0 {
+		return metric, errors.Error("empty bucket id")
+	}
+	params := jsonutils.NewDict()
+	params.Set("stats_only", jsonutils.JSONTrue)
+	result, err := modules.Buckets.PerformAction(s, bucketId, "sync", params)
+	if err != nil {
+		return metric, errors.Wrap(err, "PerformAction")
+	}
+	return jsonToMetric(result.(*jsonutils.JSONDict), "bucket",
+		[]string{
+			"name",
+			"id",
+			"account",
+			"account_id",
+			"manager",
+			"manager_id",
+			"manager_domain",
+			"manager_domain_id",
+			"manager_project",
+			"manager_project_id",
+			"brand",
+			"provider",
+			"region_id",
+			"region_ext_id",
+			"tenant",
+			"tenant_id",
+			"domain_id",
+			"project_domain",
+		},
+		[]string{
+			"object_cnt",
+			"size_bytes",
+		},
+	)
+}
+
+func init() {
+	shellutils.R(&BucketStatsOptions{}, "buckets", "Bucket stats", bucketStatsCollect)
+}

pkg/cloudmon/collectors/cloudaccountmon/cloudaccountreport.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudaccountmon
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SCloudAccountFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SCloudAccountFactory struct {
+}
+
+func (self *SCloudAccountFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions,
+	operatorType string) common.ICloudReport {
+	return &SCloudAccountReport{
+		common.CloudReportBase{
+			SProvider: nil,
+			Session:   session,
+			Args:      args,
+			Operator:  ClOUDACCOUNT_ID,
+		},
+	}
+}
+
+func (S SCloudAccountFactory) GetId() string {
+	return ClOUDACCOUNT_ID
+}
+
+type SCloudAccountReport struct {
+	common.CloudReportBase
+}
+
+func (self *SCloudAccountReport) Report() error {
+	accounts, err := self.GetAllCloudAccount(&modules.Cloudaccounts)
+	if err != nil {
+		return err
+	}
+	return self.collectMetric(accounts)
+}

pkg/cloudmon/collectors/cloudaccountmon/cloudaccountutil.go

@@ -0,0 +1,23 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudaccountmon
+
+const (
+	ClOUDACCOUNT_ID = "cloudaccount"
+
+	CLOUDACCOUNT_FIELD       = "balance"
+	CLOUDACCOUNT_MEASUREMENT = "cloudaccount_balance"
+	CLOUDACCOUNT_DATABASE    = "meter_db"
+)

pkg/cloudmon/collectors/cloudaccountmon/cloudaccoutservice.go

@@ -0,0 +1,38 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudaccountmon
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SCloudAccountReport) collectMetric(accounts []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	for _, account := range accounts {
+		metric, err := self.NewMetricFromJson(account)
+		if err != nil {
+			return err
+		}
+		metric.Timestamp = time.Now()
+		metric.Name = CLOUDACCOUNT_MEASUREMENT
+		dataList = append(dataList, metric)
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, CLOUDACCOUNT_DATABASE)
+}

pkg/cloudmon/collectors/cloudaccountmon/doc.go

@@ -0,0 +1 @@
+package cloudaccountmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon"

pkg/cloudmon/collectors/cloudaccountservice.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-cloudaccount", "Report cloud account", reportCloudAccount)
+}
+
+func reportCloudAccount(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCustomizeCloudMetric(string(common.CLOUDACCOUNT), session, args)
+}

pkg/cloudmon/collectors/cloudmoninit.go

@@ -0,0 +1,41 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/jdmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon"
+	_ "yunion.io/x/onecloud/pkg/multicloud/aliyun/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/apsara/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/aws/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/azure/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/esxi/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/google/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/huawei/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/jdcloud/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/qcloud/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/zstack/provider"
+)

pkg/cloudmon/collectors/common/basemon.go

@@ -0,0 +1,242 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/net/http/httpproxy"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+type CloudReportBase struct {
+	SProvider *SProvider
+	Session   *mcclient.ClientSession
+	Args      *ReportOptions
+	Operator  string
+}
+
+func (self *CloudReportBase) Report() error {
+	return fmt.Errorf("No Implment the method")
+}
+
+func (self *CloudReportBase) GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+
+	query.Add(jsonutils.NewStringArray([]string{"running", "ready"}), "status")
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	query.Add(jsonutils.NewString(self.SProvider.Provider), "provider")
+	query.Add(jsonutils.NewString(self.SProvider.Id), "manager")
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) GetAllHostOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("running"), "status")
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	query.Add(jsonutils.NewString(self.SProvider.Provider), "provider")
+	query.Add(jsonutils.NewString(self.SProvider.Id), "manager")
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) GetAllCloudAccount(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewBool(true), DETAILS)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	query.Add(jsonutils.NewString(fmt.Sprintf("brand.in(%s,%s,%s,%s)", compute.CLOUD_PROVIDER_ALIYUN,
+		compute.CLOUD_PROVIDER_QCLOUD, compute.CLOUD_PROVIDER_HUAWEI, compute.CLOUD_PROVIDER_JDCLOUD)), "filter")
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) GetAllStorage(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewBool(true), DETAILS)
+	query.Add(jsonutils.NewString("system"), KEY_SCOPE)
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) ListAllResource(manager modulebase.Manager,
+	query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) {
+	resources := make([]jsonutils.JSONObject, 0)
+	offsetIndex := 0
+	for {
+		query.(*jsonutils.JSONDict).Add(jsonutils.NewInt(int64(offsetIndex)), "offset")
+		resList, err := manager.List(self.Session, query)
+		if err != nil {
+			return nil, err
+		}
+		resources = append(resources, resList.Data...)
+		offsetIndex = offsetIndex + len(resList.Data)
+		if offsetIndex >= resList.Total {
+			break
+		}
+	}
+	return resources, nil
+}
+
+func (self *CloudReportBase) InitProviderInstance() (cloudprovider.ICloudProvider, error) {
+	//secret进行Descrypt
+	secretDe, _ := utils.DescryptAESBase64(self.SProvider.Id, self.SProvider.Secret)
+	var proxyFunc httputils.TransportProxyFunc
+	{
+		cfg := &httpproxy.Config{
+			HTTPProxy:  self.SProvider.ProxySetting.HTTPProxy,
+			HTTPSProxy: self.SProvider.ProxySetting.HTTPSProxy,
+			NoProxy:    self.SProvider.ProxySetting.NoProxy,
+		}
+		cfgProxyFunc := cfg.ProxyFunc()
+		proxyFunc = func(req *http.Request) (*url.URL, error) {
+			return cfgProxyFunc(req.URL)
+		}
+	}
+	cloudAccout, err := self.getCloudAccount(&modules.Cloudaccounts)
+	if err != nil {
+		return nil, errors.Wrap(err, "getCloudAccount error")
+	}
+	endpoints := cloudprovider.SApsaraEndpoints{}
+	options, err := cloudAccout.Get("options")
+	if err == nil {
+		err := options.Unmarshal(&endpoints)
+		if err != nil {
+			log.Errorf("Unmarshal SApsaraEndpoints err:%v", err)
+		}
+	} else {
+		log.Errorf("get cloudAccout options err:%v", err)
+	}
+	cfg := cloudprovider.ProviderConfig{
+		Id:               self.SProvider.Id,
+		Name:             self.SProvider.Name,
+		URL:              self.SProvider.AccessUrl,
+		Account:          self.SProvider.Account,
+		Secret:           secretDe,
+		Vendor:           self.SProvider.Provider,
+		ProxyFunc:        proxyFunc,
+		SApsaraEndpoints: endpoints,
+	}
+	return cloudprovider.GetProvider(cfg)
+}
+
+func (self *CloudReportBase) getCloudAccount(manager modulebase.Manager) (jsonutils.JSONObject, error) {
+	return self.GetResourceById(self.SProvider.CloudaccountId, manager)
+}
+
+func (self *CloudReportBase) GetResourceById(id string, manager modulebase.Manager) (jsonutils.JSONObject, error) {
+	return manager.Get(self.Session, id, jsonutils.NewDict())
+}
+
+func (self *CloudReportBase) GetAllRegionOfServers(servers []jsonutils.JSONObject,
+	providerInstance cloudprovider.ICloudProvider) ([]cloudprovider.
+	ICloudRegion, map[string][]jsonutils.JSONObject, error) {
+	extranleIdMap := make(map[string]string)
+	regionServerList := make([]cloudprovider.ICloudRegion, 0)
+	regionServerMap := make(map[string][]jsonutils.JSONObject)
+	for i, server := range servers {
+		region_external_id, err := server.GetString("region_external_id")
+		if err != nil {
+			return nil, nil, err
+		}
+		if _, ok := extranleIdMap[region_external_id]; !ok {
+			extranleIdMap[region_external_id] = ""
+			region, err := providerInstance.GetIRegionById(region_external_id)
+			if err != nil {
+				return nil, nil, err
+			}
+			regionServerList = append(regionServerList, region)
+			regionServers := make([]jsonutils.JSONObject, 0)
+			regionServers = append(regionServers, servers[i])
+			regionServerMap[region_external_id] = regionServers
+		} else {
+			regionservers := regionServerMap[region_external_id]
+			regionservers = append(regionservers, server)
+			regionServerMap[region_external_id] = regionservers
+		}
+	}
+	return regionServerList, regionServerMap, nil
+}
+
+func (self *CloudReportBase) AddMetricTag(metric *influxdb.SMetricData, tags map[string]string) {
+	for key, value := range tags {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   key,
+			Value: value,
+		})
+	}
+}
+
+func (self *CloudReportBase) NewMetricFromJson(server jsonutils.JSONObject) (influxdb.SMetricData, error) {
+	switch MonType(self.Operator) {
+	case SERVER:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", ServerTags, make([]string, 0))
+	case HOST:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", HostTags, make([]string, 0))
+	case REDIS:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", RedisTags, make([]string, 0))
+	case RDS:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", RdsTags, make([]string, 0))
+	case OSS:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", OssTags, make([]string, 0))
+	case ELB:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", ElbTags, make([]string, 0))
+	case CLOUDACCOUNT:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", CloudAccountTags, CloudAccountFields)
+	case STORAGE:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", StorageTags, make([]string, 0))
+	case ALERT_RECORD:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", AlertRecordHistoryTags, AlertRecordHistoryFields)
+	}
+	return influxdb.SMetricData{}, fmt.Errorf("no found report operator")
+}
+
+func (self *CloudReportBase) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	panic("NO implement CollectRegionMetricOfServer")
+}
+
+func (self *CloudReportBase) ListAllResources(manager modulebase.Manager,
+	query *jsonutils.JSONDict) ([]jsonutils.JSONObject, error) {
+	offsetIndex := 0
+	resources := make([]jsonutils.JSONObject, 0)
+	for {
+		query.Add(jsonutils.NewInt(int64(offsetIndex)), "offset")
+		resList, err := manager.List(self.Session, query)
+		if err != nil {
+			return nil, err
+		}
+		resources = append(resources, resList.Data...)
+		offsetIndex = offsetIndex + len(resList.Data)
+		if offsetIndex >= resList.Total {
+			break
+		}
+	}
+	return resources, nil
+}

pkg/cloudmon/collectors/common/cloudreport.go

@@ -0,0 +1,100 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+)
+
+func init() {
+	cloudReportTable = make(map[string]ICloudReportFactory)
+}
+
+type ICloudReportFactory interface {
+	NewCloudReport(provider *SProvider, session *mcclient.ClientSession, args *ReportOptions,
+		operatorType string) ICloudReport
+	GetId() string
+}
+
+type ICloudReport interface {
+	Report() error
+	GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error)
+	InitProviderInstance() (cloudprovider.ICloudProvider, error)
+	GetAllRegionOfServers(servers []jsonutils.JSONObject, providerInstance cloudprovider.ICloudProvider) (
+		[]cloudprovider.ICloudRegion, map[string][]jsonutils.JSONObject, error)
+	CollectRegionMetric(region cloudprovider.ICloudRegion,
+		servers []jsonutils.JSONObject) error
+}
+
+type SProvider struct {
+	compute.CloudproviderDetails
+	//Id        string `json:"id"
+	//Name      string `json:"name"`
+	//Provider  string `json:"provider"`
+	//Account   string `json:"account"`
+	//Secret    string `json:"secret"`
+	//AccessUrl string `json:"access_url"`
+}
+
+var cloudReportTable map[string]ICloudReportFactory
+
+func RegisterFactory(factory ICloudReportFactory) {
+	cloudReportTable[factory.GetId()] = factory
+}
+
+func GetCloudReportFactory(provider string) (ICloudReportFactory, error) {
+	factory, ok := cloudReportTable[provider]
+	if ok {
+		return factory, nil
+	}
+	log.Errorf("Provider %s not registerd", provider)
+	return nil, fmt.Errorf("No such cloudReport %s", provider)
+}
+
+func (s *SProvider) Validate() error {
+	invalidParams := request.ErrInvalidParams{Context: "SProvider"}
+	if s.Id == "" {
+		invalidParams.Add(request.NewErrParamRequired("Id"))
+	}
+	if s.Name == "" {
+		invalidParams.Add(request.NewErrParamRequired("Name"))
+	}
+	if s.Account == "" {
+		invalidParams.Add(request.NewErrParamRequired("Account"))
+	}
+	//if s.AccessUrl == "" {
+	//	invalidParams.Add(request.NewErrParamRequired("AccountUrl"))
+	//}
+	if s.Provider == "" {
+		invalidParams.Add(request.NewErrParamRequired("Provider"))
+	}
+	if s.Secret == "" {
+		invalidParams.Add(request.NewErrParamRequired("Secret"))
+	}
+	if invalidParams.Len() > 0 {
+		return invalidParams
+	}
+	return nil
+}

pkg/cloudmon/collectors/common/doc.go

@@ -0,0 +1 @@
+package common // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common"

pkg/cloudmon/collectors/common/multiCloudMonUtils.go

@@ -0,0 +1,515 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
+
+	o "yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+type MonType string
+
+const (
+	SERVER       MonType = "server"
+	HOST         MonType = "host"
+	REDIS        MonType = "redis"
+	RDS          MonType = "rds"
+	OSS          MonType = "oss"
+	ELB          MonType = "elb"
+	CLOUDACCOUNT MonType = "cloudaccount"
+	STORAGE      MonType = "storage"
+	ALERT_RECORD MonType = "alertRecord"
+)
+
+const (
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+	DETAILS    = "details"
+	KEY_SCOPE  = "scope"
+)
+
+const (
+	TYPE_VIRTUALMACHINE = "VirtualMachine"
+	TYPE_HOSTSYSTEM     = "HostSystem"
+)
+
+var OtherVmTags = map[string]string{
+	"source":   "cloudmon",
+	"res_type": "guest",
+	"is_vm":    "true",
+}
+
+var OtherHostTag = map[string]string{
+	"source":   "cloudmon",
+	"res_type": "host",
+	"is_vm":    "false",
+}
+
+type ReportOptions struct {
+	Batch     int      `help:"batch"`
+	Count     int      `help:"count" json:"count"`
+	Interval  string   `help:"interval""`
+	Timeout   int64    `help:"command timeout unit:second" default:"10"`
+	SinceTime string   `help:"sinceTime"`
+	EndTime   string   `help:"endTime"`
+	Provider  []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara|JDcloud|Ecloud" json:"provider,omitempty"`
+	PingProbeOptions
+}
+
+type PingProbeOptions struct {
+	Debug         bool `help:"debug"`
+	ProbeCount    int  `help:"probe count, default is 3" default:"3"`
+	TimeoutSecond int  `help:"probe timeout in second, default is 1 second" default:"1"`
+}
+
+var InstanceProviders = "Aliyun,Azure,Aws,Qcloud,VMWare,Huawei,Openstack,Ucloud,ZStack"
+
+//server的key-value对应保存时的Tags和Pairs
+//var ServerTags = []string{"host", "host_id", "vm_id", "vm_ip", "vm_name", "zone", "zone_id", "zone_ext_id",
+//	"hypervisor", "os_type", "status", "region", "region_id", "region_ext_id", "tenant", "tenant_id", "brand", "name"}
+var ServerTags = map[string]string{
+	"host":             "host",
+	"host_id":          "host_id",
+	"id":               "vm_id",
+	"ips":              "vm_ip",
+	"name":             "vm_name",
+	"zone":             "zone",
+	"zone_id":          "zone_id",
+	"zone_ext_id":      "zone_ext_id",
+	"os_type":          "os_type",
+	"status":           "status",
+	"cloudregion":      "cloudregion",
+	"cloudregion_id":   "cloudregion_id",
+	"region_ext_id":    "region_ext_id",
+	"tenant":           "tenant",
+	"tenant_id":        "tenant_id",
+	"brand":            "brand",
+	"scaling_group_id": "vm_scaling_group_id",
+	"domain_id":        "domain_id",
+	"project_domain":   "project_domain",
+}
+var HostTags = map[string]string{
+	"id":             "host_id",
+	"ips":            "host_ip",
+	"name":           "host",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var RdsTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "rds_id",
+	"ips":            "rds_ip",
+	"name":           "rds_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var RedisTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "redis_id",
+	"ips":            "redis_ip",
+	"name":           "redis_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var OssTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "oss_id",
+	"ips":            "oss_ip",
+	"name":           "oss_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var ElbTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "elb_id",
+	"ips":            "elb_ip",
+	"name":           "elb_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"region":         "region",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+
+var CloudAccountTags = map[string]string{
+	"id":             "cloudaccount_id",
+	"name":           "cloudaccount_name",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+
+var StorageTags = map[string]string{
+	"id":             "storage_id",
+	"name":           "storage_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+
+var AlertRecordHistoryTags = map[string]string{
+	"alert_name":     "alert_name",
+	"alert_id":       "alert_id",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+	"tenant_id":      "tenant_id",
+	"tenant":         "tenant",
+	"res_type":       "res_type",
+}
+
+var CloudAccountFields = []string{"balance"}
+
+var AlertRecordHistoryFields = []string{"res_num"}
+
+var ServerPairs = []string{"vcpu_count", "vmem_size", "disk"}
+
+var AddTags = map[string]string{
+	"source": "cloudmon",
+}
+
+//get substring from str before separator
+func SubstringBefore(str, separator string) string {
+	if str != "" {
+		if separator == "" {
+			return ""
+		} else {
+			if pos := strings.Index(str, separator); pos == -1 {
+				return str
+			} else {
+				return str[0:pos]
+			}
+		}
+	} else {
+		return str
+	}
+}
+
+//get substring from str after separator
+func SubstringAfter(str, separator string) string {
+	if str != "" {
+		if separator == "" {
+			return ""
+		} else {
+			if pos := strings.Index(str, separator); pos == -1 {
+				return ""
+			} else {
+				return str[pos+len(separator):]
+			}
+		}
+	} else {
+		return str
+	}
+}
+
+//get a substring from str between[open,close)
+func SubstringBetween(str, open, close string) string {
+	if str != "" && open != "" && close != "" {
+		if start := strings.Index(str, open); start != -1 {
+			if end := strings.Index(str[start+len(open):], close); end != -1 {
+				return str[start+len(open) : start+len(open)+end]
+			}
+		}
+		return ""
+	} else {
+		return ""
+	}
+}
+
+func ParseTimeStr(startTime, endTime string) (since, util time.Time, err error) {
+	since, err = timeutils.ParseTimeStr(startTime)
+	if err != nil {
+		return since, util, err
+	}
+	util, err = timeutils.ParseTimeStr(endTime)
+	if err != nil {
+		return since, util, err
+	}
+	return since, util, nil
+}
+
+func TimeRangeFromArgs(args *ReportOptions) (since, until time.Time, err error) {
+	if args.SinceTime != "" && args.EndTime != "" {
+		since, until, err = ParseTimeStr(args.SinceTime, args.EndTime)
+		if err != nil {
+			return since, until, err
+		}
+	} else {
+		period64, err := strconv.ParseInt(args.Interval, 10, 8)
+		if err != nil {
+			return since, until, err
+		}
+		since = time.Now().Add(-time.Minute * time.Duration(period64))
+		until = time.Now()
+	}
+	return since, until, nil
+}
+
+//组装server相关capability
+func FillVMCapacity(server *jsonutils.JSONDict) (influxdb.SMetricData, error) {
+	metric, err := JsonToMetric(server, "vm_capacity", ServerTags, ServerPairs)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	hypevisor, _ := server.GetString("hypervisor")
+	metric.Timestamp = time.Now()
+	metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+		Key:   "res_type",
+		Value: "guest",
+	}, influxdb.SKeyValue{
+		Key:   "is_vm",
+		Value: "true",
+	}, influxdb.SKeyValue{
+		Key:   "platform",
+		Value: hypevisor,
+	})
+	return metric, nil
+}
+
+func GetMeasurement(action string, influxDbSpec string) (measurement string) {
+	// VirtualMachine -> VMware类型的虚拟机
+	if action == TYPE_VIRTUALMACHINE {
+		measurement = SubstringBefore(influxDbSpec, ".")
+	}
+	if action == TYPE_HOSTSYSTEM {
+		measurement = SubstringBetween(influxDbSpec, "vm_", ".")
+		if strings.Contains(influxDbSpec, "vm_netio") {
+			measurement = "net"
+		}
+	}
+	return measurement
+}
+
+func JsonToMetric(obj *jsonutils.JSONDict, name string, tags map[string]string, metrics []string) (influxdb.SMetricData, error) {
+	metric := influxdb.SMetricData{Name: name}
+	objMap, err := obj.GetMap()
+	if err != nil {
+		return metric, errors.Wrap(err, "obj.GetMap")
+	}
+	tagPairs := make([]influxdb.SKeyValue, 0)
+	metricPairs := make([]influxdb.SKeyValue, 0)
+	for k, v := range objMap {
+		val, _ := v.GetString()
+		if strings.Contains(k, "ip") {
+			if strings.Contains(val, ",") {
+				val = strings.ReplaceAll(val, ",", "|")
+			}
+		}
+		if tag, ok := tags[k]; ok {
+			tagPairs = append(tagPairs, influxdb.SKeyValue{
+				Key:   tag,
+				Value: val,
+			})
+		} else if utils.IsInStringArray(k, metrics) {
+			metricPairs = append(metricPairs, influxdb.SKeyValue{
+				Key: k, Value: val,
+			})
+		}
+	}
+	metric.Tags = tagPairs
+	metric.Metrics = metricPairs
+	return metric, nil
+}
+
+func SendMetrics(s *mcclient.ClientSession, metrics []influxdb.SMetricData, debug bool, database string) error {
+	urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType)
+	if err != nil {
+		return errors.Wrap(err, "GetServiceURLs")
+	}
+	if len(database) == 0 {
+		database = o.Options.InfluxDatabase
+	}
+	return influxdb.SendMetrics(urls, database, metrics, debug)
+}
+
+func ReportCloudMetricOfoperatorType(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	//query.Add(jsonutils.NewString("true"), KEY_USABLE)
+	if len(args.Provider) > 0 {
+		for _, val := range args.Provider {
+			query.Add(jsonutils.NewString(val), "provider")
+		}
+	}
+	cloudProviderList, err := (&modules.Cloudproviders).List(session, query)
+	if err != nil {
+		return errors.Wrap(err, "cloudProviders get list error")
+	}
+	providerGroup, _ := errgroup.WithContext(context.Background())
+	tmpCount := 0
+	if args.Count == 0 {
+		args.Count = 1
+	}
+	for i := 0; i < len(cloudProviderList.Data); i++ {
+		provider := cloudProviderList.Data[i]
+		status, err := provider.GetString("status")
+		if err != nil {
+			return errors.Wrap(err, "provider get status error")
+		}
+		if status == "connected" {
+			providerStruct := SProvider{}
+			err := provider.Unmarshal(&providerStruct)
+			if err != nil {
+				return errors.Wrap(err, "provider.Unmarshal")
+			}
+			err = (&providerStruct).Validate()
+			if err != nil {
+				return errors.Wrap(err, "provider Invalidate")
+			}
+			providerStr := providerStruct.Provider
+			cloudReportFactory, err := GetCloudReportFactory(providerStr)
+			if err != nil {
+				log.Errorln(errors.Wrap(err, "GetCloudReportFactory"))
+				continue
+			}
+			tmpCount++
+			providerGroup.Go(func() error {
+				err = cloudReportFactory.NewCloudReport(&providerStruct, session, args, operatorType).Report()
+				if err != nil {
+					log.Errorln(errors.Wrap(err, "cloudReport Report method"))
+				}
+				return nil
+			})
+			if tmpCount == args.Count {
+				err := providerGroup.Wait()
+				if err != nil {
+					return err
+				}
+				tmpCount = 0
+			}
+		}
+	}
+	return providerGroup.Wait()
+}
+
+func newGroupFunc(func()) {
+
+}
+
+func ReportCustomizeCloudMetric(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error {
+	cloudReportFactory, err := GetCloudReportFactory(operatorType)
+	if err != nil {
+		return errors.Wrap(err, "GetCloudReportFactory")
+	}
+	err = cloudReportFactory.NewCloudReport(nil, session, args, operatorType).Report()
+	if err != nil {
+		return errors.Wrap(err, "cloudReport Report method")
+	}
+	return nil
+}
+
+func CollectRegionMetricAsync(asynCount int, region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject, report ICloudReport) error {
+	metricGroup, _ := errgroup.WithContext(context.Background())
+	count := 0
+	if asynCount == 0 {
+		asynCount = 10
+	}
+	for i, _ := range servers {
+		server := servers[i]
+		metricGroup.Go(func() error {
+			return report.CollectRegionMetric(region, []jsonutils.JSONObject{server})
+		})
+		count++
+		if count == asynCount {
+			err := metricGroup.Wait()
+			if err != nil {
+				return err
+			}
+			count = 0
+		}
+	}
+	return metricGroup.Wait()
+}

pkg/cloudmon/collectors/doc.go

@@ -0,0 +1 @@
+package collectors // import "yunion.io/x/onecloud/pkg/cloudmon/collectors"

pkg/cloudmon/collectors/ecloudmon/doc.go

@@ -0,0 +1 @@
+package ecloudmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/ecloudmon"

pkg/cloudmon/collectors/ecloudmon/ecloudreport.go

@@ -0,0 +1,76 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ecloudmon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SECloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SECloudReportFactory struct {
+}
+
+func (self *SECloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SECloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SECloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_ECLOUD
+}
+
+type SECloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SECloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/ecloudmon/ecloudservice.go

@@ -0,0 +1,127 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ecloudmon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/ecloud"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+const (
+	SERVER_PRODUCT_TYPE = "vm"
+)
+
+func (self *SECloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	ecloudReg := region.(*ecloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, err := server.GetString("external_id")
+		if err != nil {
+			continue
+		}
+		name, _ := server.GetString("name")
+		external_id = strings.Split(external_id, "/")[1]
+		metrics := make([]ecloud.Metric, 0)
+		for metricName, _ := range ecloudMetricSpecs {
+			metrics = append(metrics, ecloud.Metric{Name: metricName})
+		}
+		data, err := ecloudReg.DescribeMetricList(SERVER_PRODUCT_TYPE, metrics, external_id, since, until)
+		if err != nil {
+			return errors.Wrap(err, "region DescribeMetricList err:")
+		}
+		for _, entity := range data.Entitys {
+			if influxDbSpecs, ok := ecloudMetricSpecs[entity.MetricName]; ok {
+				metricData, err := self.collectMetricFromThisServer(server, entity, influxDbSpecs)
+				if err != nil {
+					log.Errorf("server:%s collectMetric:%s err", name, entity.MetricName)
+					continue
+				}
+				dataList = append(dataList, metricData...)
+			}
+
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SECloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	entity ecloud.Entity, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	for _, point := range entity.Datapoints {
+		metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		if err != nil {
+			return nil, err
+		}
+		if len(point) != 2 {
+			log.Errorf("invalid point:%v", point)
+			continue
+		}
+		influxDbSpec := influxDbSpecs[2]
+
+		measurement := common.SubstringBefore(influxDbSpec, ".")
+		metric.Name = measurement
+		pointTime, err := strconv.ParseInt(point[1], 10, 64)
+		if err != nil {
+			return nil, errors.Wrap(err, "point parseInt err")
+		}
+		metric.Timestamp = time.Unix(pointTime, 0)
+		pointVal, err := strconv.ParseFloat(point[0], 64)
+		if err != nil {
+			return nil, errors.Wrap(err, "point parseInt err")
+		}
+		if influxDbSpecs[1] == UNIT_BYTEPS {
+			pointVal = pointVal * 8
+		}
+		if influxDbSpecs[1] == UNIT_KBYTEPS {
+			pointVal = pointVal * 8 * 1024
+		}
+		var pairsKey string
+		if strings.Contains(influxDbSpec, ",") {
+			pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+		} else {
+			pairsKey = common.SubstringAfter(influxDbSpec, ".")
+		}
+
+		tag := common.SubstringAfter(influxDbSpec, ",")
+		if tag != "" && strings.Contains(influxDbSpec, "=") {
+			metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+				Key:   common.SubstringBefore(tag, "="),
+				Value: common.SubstringAfter(tag, "="),
+			})
+		}
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   pairsKey,
+			Value: strconv.FormatFloat(pointVal, 'f', -1, 64),
+		})
+		self.AddMetricTag(&metric, common.AddTags)
+		datas = append(datas, metric)
+	}
+	return datas, nil
+}

pkg/cloudmon/collectors/ecloudmon/ecloudutils.go

@@ -0,0 +1,56 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ecloudmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_CPS           = "cps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_KBYTEPS       = "KBps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE       = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE       = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS   = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS  = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS  = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX      = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_TX      = "vm_netio.bps_sent"
+
+	KEY_LIMIT = "limit"
+	KEY_ADMIN = "admin"
+)
+
+var ecloudProTypeMetric = map[string]string{}
+
+//multiCloud查询指标列表组装
+var ecloudMetricSpecs = map[string][]string{
+	"cpu_util":                        {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"memory.util":                     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
+	"disk.device.read.bytes.rate":     {DEFAULT_STATISTICS, UNIT_KBYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"disk.device.write.bytes.rate":    {DEFAULT_STATISTICS, UNIT_KBYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"disk.device.read.requests.rate":  {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"disk.device.write.requests.rate": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+	"network.incoming.bytes":          {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX},
+	"network.outgoing.bytes":          {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX},
+}

pkg/cloudmon/collectors/elbservice.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-elb", "Report Elb", reportElb)
+}
+
+//入口函数[aliyun、huawei]
+func reportElb(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.ELB), session, args)
+}

pkg/cloudmon/collectors/gcpmon/doc.go

@@ -0,0 +1 @@
+package gcpmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon"

pkg/cloudmon/collectors/gcpmon/gcpreport.go

@@ -0,0 +1,84 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gcpmon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SGoogleCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SGoogleCloudReportFactory struct {
+}
+
+func (self *SGoogleCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SGoogleCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SGoogleCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_GOOGLE
+}
+
+type SGoogleCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SGoogleCloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+			//case "redis":
+			//	err = self.collectRegionMetricOfRedis(region, servers)
+			//case "rds":
+			//	err = self.collectRegionMetricOfRds(region, servers)
+			//case "oss":
+			//	err = self.collectRegionMetricOfOss(region, servers)
+			//case "elb":
+			//	err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/gcpmon/gcpservice.go

@@ -0,0 +1,157 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gcpmon
+
+import (
+	"strconv"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/timeutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/google"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SGoogleCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	googleReg := region.(*google.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, err := server.GetString("external_id")
+		if err != nil {
+			continue
+		}
+		serverName, _ := server.GetString("name")
+		external_id = strings.Split(external_id, "/")[1]
+		for metricName, influxDbSpecs := range gcpMetricSpecs {
+			rtn, err := googleReg.GetMonitorData(external_id, serverName, metricName, since, until)
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+			if err != nil {
+				return err
+			}
+			dataList = append(dataList, metric)
+			for _, resp := range rtn.Value() {
+				if !resp.(*jsonutils.JSONDict).Contains("points") {
+					continue
+				}
+				serverMetric, err := self.collectMetricFromThisServer(server, resp, influxDbSpecs)
+				if err != nil {
+					return err
+				}
+				dataList = append(dataList, serverMetric...)
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SGoogleCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	rtnMetric jsonutils.JSONObject, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	points_, _ := rtnMetric.Get("points")
+	for _, point := range points_.(*jsonutils.JSONArray).Value() {
+		metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		if err != nil {
+			return nil, err
+		}
+		if time, err := point.Get("interval", "startTime"); err == nil {
+			influxDbSpec := influxDbSpecs[2]
+
+			measurement := common.SubstringBefore(influxDbSpec, ".")
+			metric.Name = measurement
+
+			timestamp, err := timeutils.ParseTimeStr(time.(*jsonutils.JSONString).Value())
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			metric.Timestamp = timestamp
+
+			var pairsKey string
+			if strings.Contains(influxDbSpec, ",") {
+				pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+			} else {
+				pairsKey = common.SubstringAfter(influxDbSpec, ".")
+			}
+
+			tag := common.SubstringAfter(influxDbSpec, ",")
+			if tag != "" && strings.Contains(influxDbSpec, "=") {
+				metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+					Key:   common.SubstringBefore(tag, "="),
+					Value: common.SubstringAfter(tag, "="),
+				})
+			}
+
+			cpu_cout, err := server.Get("vcpu_count")
+			if err == nil {
+				metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+					Key:   "cpu_count",
+					Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+				})
+			}
+
+			valueTypeObj, _ := rtnMetric.Get("valueType")
+			valueType := strings.ToLower(valueTypeObj.(*jsonutils.JSONString).Value())
+			value := getMetricValue(point, valueType, influxDbSpecs[1])
+			metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+				Key:   pairsKey,
+				Value: value,
+			})
+			self.AddMetricTag(&metric, common.AddTags)
+			datas = append(datas, metric)
+		}
+	}
+	return datas, nil
+}
+
+func getMetricValue(point jsonutils.JSONObject, valueType string,
+	influxDbSpec string) string {
+	value, _ := point.Get("value", valueType+"Value")
+
+	switch valueType {
+	case "int64":
+		if val, ok := value.(*jsonutils.JSONString); ok {
+			fieldValue, _ := strconv.ParseInt(val.Value(), 10, 64)
+			if influxDbSpec == UNIT_MEM {
+				fieldValue = fieldValue * 8 / PERIOD
+			}
+			return strconv.FormatInt(fieldValue, 10)
+		}
+		fieldValue := value.(*jsonutils.JSONInt).Value()
+		if influxDbSpec == UNIT_MEM {
+			fieldValue = fieldValue * 8 / PERIOD
+		}
+		return strconv.FormatInt(fieldValue, 10)
+	case "double":
+		fieldValue := value.(*jsonutils.JSONFloat).Value()
+		if influxDbSpec == UNIT_MEM {
+			fieldValue = fieldValue * 8 / PERIOD
+		}
+		return strconv.FormatFloat(fieldValue, 'f', 3, 64)
+	}
+	return ""
+}

pkg/cloudmon/collectors/gcpmon/gcputils.go

@@ -0,0 +1,131 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gcpmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var gcpMetricSpecs = map[string][]string{
+	"compute.googleapis.com/instance/cpu/utilization":              {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"compute.googleapis.com/instance/network/received_bytes_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"compute.googleapis.com/instance/network/sent_bytes_count":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+	"compute.googleapis.com/instance/disk/read_bytes_count":        {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"compute.googleapis.com/instance/disk/write_bytes_count":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"compute.googleapis.com/instance/disk/read_ops_count":          {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"compute.googleapis.com/instance/disk/write_ops_count":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}

pkg/cloudmon/collectors/hostservice.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-host", "Report Host", reportHost)
+}
+
+//入口函数[aliyun、huawei]
+func reportHost(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.HOST), session, args)
+}

pkg/cloudmon/collectors/huaweimon/doc.go

@@ -0,0 +1 @@
+package huaweimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon"

pkg/cloudmon/collectors/huaweimon/huaweicloudreport.go

@@ -0,0 +1,99 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package huaweimon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SHwCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SHwCloudReportFactory struct {
+}
+
+func (self *SHwCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SHwCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SHwCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_HUAWEI
+}
+
+type SHwCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SHwCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/huaweimon/hwservice.go

@@ -0,0 +1,341 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package huaweimon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/huawei"
+	hw_moudules "yunion.io/x/onecloud/pkg/multicloud/huawei/client/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SHwCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		instanceId, _ := server.GetString("external_id")
+		metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+		if err != nil {
+			return err
+		}
+		dataList = append(dataList, metric)
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.ECS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "instance_id", Value: instanceId})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		instanceId, _ := server.GetString("external_id")
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiRedisMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.DCS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "dcs_instance_id", Value: instanceId})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			return err
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiRedisMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		instanceId, _ := server.GetString("external_id")
+		engine, _ := server.GetString("engine")
+		metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+		if err != nil {
+			return err
+		}
+		dataList = append(dataList, metric)
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiRdsMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.RDS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			switch engine {
+			case "MySQL":
+				hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "rds_cluster_id", Value: instanceId})
+			case "PostgreSQL":
+				hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "postgresql_cluster_id", Value: instanceId})
+			case "SQLServer":
+				hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "rds_cluster_sqlserver_id", Value: instanceId})
+			}
+			metas = append(metas, hwMeta)
+		}
+		index := 0
+		tmp := 0
+		for {
+			if index > len(metas) {
+				break
+			}
+			tmp = index + 10
+			if tmp > len(metas) {
+				tmp = len(metas)
+			}
+			metricDatas, err := hwReg.GetMetricsData(metas[index:tmp], since, until)
+			if err != nil {
+				return err
+			}
+			if len(metricDatas) > 0 {
+				for _, metricData := range metricDatas {
+					for metricName, influxDbSpecs := range huaweiRdsMetricSpecs {
+						if metricData.MetricName == metricName {
+							if len(metricData.Datapoints) > 0 {
+								for _, datapoint := range metricData.Datapoints {
+									serverMetric, err := self.collectMetricFromThisServer(server, datapoint, influxDbSpecs)
+									if err != nil {
+										return err
+									}
+									dataList = append(dataList, serverMetric)
+								}
+							}
+						}
+					}
+
+				}
+			}
+			index += 10
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		name, _ := server.GetString("name")
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiOSSMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.OBS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "bucket_name", Value: name})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			return err
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiOSSMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, _ := server.GetString("external_id")
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiOSSMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.ELB"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "lb_instance_id", Value: external_id})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			return err
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiOSSMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, datapoint hw_moudules.SDatapoint,
+	influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	metric.Timestamp = time.Unix(datapoint.Timestamp/1000, 0)
+	fieldValue := datapoint.Average
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	metric.Name = measurement
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	return metric, nil
+}

pkg/cloudmon/collectors/huaweimon/hwutils.go

@@ -0,0 +1,176 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package huaweimon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+)
+
+var huaweiMetricSpecs = map[string][]string{
+	"cpu_util":                              {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},          //CPU使用率，该指标用于统计测量对象的CPU使用率，以百分为单位。
+	"network_incoming_bytes_aggregate_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET}, //带外网络流入速率，该指标用于在虚拟化层统计每秒流入测量对象的网络流量，以字节/秒为单位。
+	"network_outgoing_bytes_aggregate_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET}, //带外网络流出速率，该指标用于在虚拟化层统计每秒流出测量对象的网络流量，以字节/秒为单位。
+	"disk_read_bytes_rate":                  {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},       //磁盘读速率，该指标用于统计每秒从测量对象读出数据量，以字节/秒为单位。
+	"disk_write_bytes_rate":                 {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},      //磁盘写速率，该指标用于统计每秒写到测量对象的数据量，以字节/秒为单位。
+	"disk_read_requests_rate":               {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},         //该指标用于统计每秒从测量对象读取数据的请求次数，以请求/秒为单位。
+	"disk_write_requests_rate":              {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},        //该指标用于统计每秒从测量对象写数据的请求次数，以请求/秒为单位
+}
+
+var huaweiRdsMetricSpecs = map[string][]string{
+	"rds001_cpu_util":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"rds002_mem_util":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"rds004_bytes_in":             {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_RX},
+	"rds005_bytes_out":            {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_TX},
+	"rds039_disk_util":            {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"rds049_disk_read_throughput": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_DISK_READ_BPS},
+
+	"rds050_disk_write_throughput": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_DISK_WRITE_BPS},
+
+	"rds006_conn_count": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_COUNT},
+
+	"rds008_qps": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_QPS},
+	"rds009_tps": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_TPS},
+
+	"rds013_innodb_reads":  {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_REDA_BPS},
+	"rds014_innodb_writes": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS},
+}
+
+var huaweiRedisMetricSpecs = map[string][]string{
+	"cpu_usage":                 {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"memory_usage":              {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"instantaneous_input_kbps":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"instantaneous_output_kbps": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"connected_clients":         {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"instantaneous_ops":         {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"keys":                      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"expires":                   {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"used_memory_dataset":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+
+var huaweiOSSMetricSpecs = map[string][]string{
+	"download_bytes":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX},
+	"upload_bytes":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX},
+	"first_byte_latency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
+	"get_request_count":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
+	"request_count_4xx":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_4XX},
+	"request_count_5xx":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
+}
+
+var huaweiElbMetricSpecs = map[string][]string{
+	"m7_in_Bps":      {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
+	"m8_out_Bps":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
+	"mb_l7_qps":      {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_REQ_RATE},
+	"mc_l7_http_2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
+	"md_l7_http_3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
+	"me_l7_http_4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
+	"mf_l7_http_5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
+}

pkg/cloudmon/collectors/jdmon/doc.go

@@ -0,0 +1 @@
+package jdmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/jdmon"

pkg/cloudmon/collectors/jdmon/jdcloudreport.go

@@ -0,0 +1,87 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jdmon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SJdCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SJdCloudReportFactory struct {
+}
+
+func (self *SJdCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SJdCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SJdCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_JDCLOUD
+}
+
+type SJdCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SJdCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch common.MonType(self.Operator) {
+	case common.RDS:
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch common.MonType(self.Operator) {
+		case common.SERVER:
+			err = self.collectRegionMetricOfServer(region, servers)
+		case common.RDS:
+			err = self.collectRegionMetricOfRds(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/jdmon/jdservice.go

@@ -0,0 +1,197 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jdmon
+
+import (
+	"encoding/json"
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/timeutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/jdcloud"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+const (
+	SERVICE_CODE_VM             = "vm"
+	SERVICE_CODE_RDS_SQLSERVER  = "sqlserver"
+	SERVICE_CODE_RDS_MYSQL      = "database"
+	SERVICE_CODE_RDS_PERCONA    = "percona"
+	SERVICE_CODE_RDS_MARIADB    = "mariadb"
+	SERVICE_CODE_RDS_POSTGRESQL = "pg"
+)
+
+var (
+	rdsEngineMap = map[string]string{
+		"SQL Server": SERVICE_CODE_RDS_SQLSERVER,
+		"MySQL":      SERVICE_CODE_RDS_MYSQL,
+		"Percona":    SERVICE_CODE_RDS_PERCONA,
+		"MariaDB":    SERVICE_CODE_RDS_MARIADB,
+		"PostgreSQL": SERVICE_CODE_RDS_POSTGRESQL,
+	}
+)
+
+func (self *SJdCloudReport) collectRegionMetricOfServer(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	jdReg := region.(*jdcloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	sinceStr := since.Format(timeutils.FullIsoTimeFormat)
+	untilStr := until.Format(timeutils.FullIsoTimeFormat)
+	for _, server := range servers {
+		metrics := self.GetInstanceMetric(server, jdReg, jdMetricSpecs, sinceStr, untilStr, SERVICE_CODE_VM)
+		if len(metrics) != 0 {
+			dataList = append(dataList, metrics...)
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SJdCloudReport) GetInstanceMetric(instance jsonutils.JSONObject, jdReg *jdcloud.SRegion,
+	metricMap map[string][]string,
+	startTime string, endTime string, serviceCode string) []influxdb.SMetricData {
+	dataList := make([]influxdb.SMetricData, 0)
+	name, _ := instance.GetString("name")
+	instanceId, _ := instance.GetString("external_id")
+	for metricName, influxDbSpecs := range metricMap {
+		request := jdcloud.NewDescribeMetricDataRequestWithAllParams(jdReg.GetId(), metricName, &startTime, &endTime,
+			nil, &serviceCode, instanceId)
+		response, err := jdReg.GetMetricsData(request)
+		if err != nil {
+			log.Errorf("get instance:%s metric err:%v", name, err)
+			continue
+		}
+		metricData, err := self.collectMetricFromThisServer(instance, response, influxDbSpecs)
+		if err != nil {
+			log.Errorf("collectMetricFromThisServer:%s err:%v", name, err)
+			continue
+		}
+		dataList = append(dataList, metricData...)
+	}
+	return dataList
+}
+
+func (self *SJdCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	metricRep *jdcloud.DescribeMetricDataResponse,
+	influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	for _, metricData := range metricRep.Result.MetricDatas {
+		for _, datapoint := range metricData.Data {
+			//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+			metric, err := self.NewMetricFromJson(server)
+			if err != nil {
+				return datas, err
+			}
+			metric.Timestamp = time.Unix(datapoint.Timestamp/1000, 0)
+			fieldValue := self.parseDataValue(datapoint.Value)
+			//根据条件拼装metric的tag和metirc信息
+			influxDbSpec := influxDbSpecs[2]
+			measurement := common.SubstringBefore(influxDbSpec, ".")
+			var pairsKey string
+			if strings.Contains(influxDbSpec, ",") {
+				pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+			} else {
+				pairsKey = common.SubstringAfter(influxDbSpec, ".")
+			}
+			if influxDbSpecs[1] == UNIT_BYTEPS {
+				fieldValue = fieldValue * 8
+			}
+			if influxDbSpecs[1] == UNIT_KBPS {
+				fieldValue = fieldValue * 1000
+			}
+			tag := common.SubstringAfter(influxDbSpec, ",")
+			if tag != "" && strings.Contains(influxDbSpec, "=") {
+				metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+					Key:   common.SubstringBefore(tag, "="),
+					Value: common.SubstringAfter(tag, "="),
+				})
+			}
+			metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+				Key:   pairsKey,
+				Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+			})
+			metric.Name = measurement
+			self.AddMetricTag(&metric, common.OtherVmTags)
+			datas = append(datas, metric)
+		}
+	}
+	return datas, nil
+}
+
+func (self *SJdCloudReport) parseDataValue(value interface{}) float64 {
+	str, ok := value.(string)
+	if !ok {
+		log.Errorf("parseDataValue err:%v", value)
+		return 0
+	}
+	number := json.Number(str)
+	fvalue, err := number.Float64()
+	if err == nil {
+		return fvalue
+	}
+
+	ivalue, err := number.Int64()
+	if err == nil {
+		ret := float64(ivalue)
+		return ret
+	}
+	log.Errorln("parseDataValue data type err")
+	return 0
+}
+
+func (self *SJdCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+
+	jdReg := region.(*jdcloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	sinceStr := since.Format(timeutils.FullIsoTimeFormat)
+	untilStr := until.Format(timeutils.FullIsoTimeFormat)
+
+	for _, server := range servers {
+		engine, _ := server.GetString("engine")
+		metrics := make([]influxdb.SMetricData, 0)
+		if serviceCode, ok := rdsEngineMap[engine]; ok {
+			switch serviceCode {
+			case SERVICE_CODE_RDS_SQLSERVER:
+				metrics = self.GetInstanceMetric(server, jdReg, jdRdsSqlserverMetricSpecs, sinceStr, untilStr, serviceCode)
+			case SERVICE_CODE_RDS_MYSQL:
+				metrics = self.GetInstanceMetric(server, jdReg, jdRdsMysqlMetricSpecs, sinceStr, untilStr, serviceCode)
+			case SERVICE_CODE_RDS_PERCONA:
+				metrics = self.GetInstanceMetric(server, jdReg, jdRdsPerconaMetricSpecs, sinceStr, untilStr, serviceCode)
+			case SERVICE_CODE_RDS_MARIADB:
+				metrics = self.GetInstanceMetric(server, jdReg, jdRdsMariadbMetricSpecs, sinceStr, untilStr, serviceCode)
+			case SERVICE_CODE_RDS_POSTGRESQL:
+				metrics = self.GetInstanceMetric(server, jdReg, jdRdsPostgresqlMetricSpecs, sinceStr, untilStr, serviceCode)
+			}
+			if len(metrics) != 0 {
+				dataList = append(dataList, metrics...)
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}

pkg/cloudmon/collectors/jdmon/jdutils.go

@@ -0,0 +1,130 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jdmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_KBPS          = "Kbps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE             = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE             = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX            = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_PERCONA    = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=percona"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MARIADB    = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mariadb"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_POSTGRESQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=postgresql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL      = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER  = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX            = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_PERCONA    = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=percona"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MARIADB    = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mariadb"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_POSTGRESQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=postgresql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL      = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER  = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE            = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS         = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS        = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT            = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE            = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+)
+
+var (
+	jdMetricSpecs = map[string][]string{
+		"cpu_util":                 {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},     //CPU使用率，该指标用于统计测量对象的CPU使用率，以百分为单位。
+		"memory.usage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},     //CPU使用率，该指标用于统计测量对象的CPU使用率，以百分为单位。
+		"vm.disk.dev.used":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},     //CPU使用率，该指标用于统计测量对象的CPU使用率，以百分为单位。
+		"vm.network.dev.bytes.in":  {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX},     //带外网络流入速率，该指标用于在虚拟化层统计每秒流入测量对象的网络流量，以字节/秒为单位。
+		"vm.network.dev.bytes.out": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX},     //带外网络流出速率，该指标用于在虚拟化层统计每秒流出测量对象的网络流量，以字节/秒为单位。
+		"vm.disk.dev.bytes.read":   {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},  //磁盘读速率，该指标用于统计每秒从测量对象读出数据量，以字节/秒为单位。
+		"vm.disk.dev.bytes.write":  {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, //磁盘写速率，该指标用于统计每秒写到测量对象的数据量，以字节/秒为单位。
+		"vm.disk.dev.io.read":      {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},    //该指标用于统计每秒从测量对象读取数据的请求次数，以请求/秒为单位。
+		"vm.disk.dev.io.write":     {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},   //该指标用于统计每秒从测量对象写数据的请求次数，以请求/秒为单位
+	}
+
+	jdRdsSqlserverMetricSpecs = map[string][]string{
+		"database.sqlserver.kvm.cpu.util":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+		"database.sqlserver.kvm.memory.usage":           {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+		"database.sqlserver.kvm.disk1.usedpercent":      {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+		"database.sqlserver.kvm.network.bytes.incoming": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
+		"database.sqlserver.kvm.network.bytes.outgoing": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
+	}
+	jdRdsMysqlMetricSpecs = map[string][]string{
+		"database.docker.cpu.util":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+		"database.docker.memory.pused":     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+		"database.docker.disk1.used":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+		"database.docker.network.incoming": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
+		"database.docker.network.outgoing": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
+	}
+	jdRdsPerconaMetricSpecs = map[string][]string{
+		"database.docker.cpu.util":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+		"database.docker.memory.pused":     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+		"database.docker.disk1.used":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+		"database.docker.network.incoming": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_PERCONA},
+		"database.docker.network.outgoing": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_PERCONA},
+	}
+	jdRdsMariadbMetricSpecs = map[string][]string{
+		"database.docker.cpu.util":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+		"database.docker.memory.pused":     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+		"database.docker.disk1.used":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+		"database.docker.network.incoming": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MARIADB},
+		"database.docker.network.outgoing": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MARIADB},
+	}
+	jdRdsPostgresqlMetricSpecs = map[string][]string{
+		"database.docker.cpu.util":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+		"database.docker.memory.pused":           {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+		"database.docker.disk1.used":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+		"database.docker.network.bytes.incoming": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_POSTGRESQL},
+		"database.docker.network.bytes.outgoing": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_POSTGRESQL},
+	}
+)

pkg/cloudmon/collectors/ossservice.go

@@ -0,0 +1,28 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-oss", "Report Oss", reportOss)
+}
+func reportOss(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.OSS), session, args)
+}

pkg/cloudmon/collectors/pinger.go

@@ -0,0 +1,222 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/netutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+	"yunion.io/x/onecloud/pkg/util/sysutils"
+)
+
+func pingProbeCoolector(s *mcclient.ClientSession, args *common.ReportOptions) error {
+	isRoot := sysutils.IsRootPermission()
+	if !isRoot {
+		return errors.Error("require root permissions")
+	}
+	metrics := make([]influxdb.SMetricData, 0)
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString(api.CLOUD_ENV_ON_PREMISE), "cloud_env")
+	params.Add(jsonutils.NewString(string(rbacutils.ScopeSystem)), "scope")
+	params.Add(jsonutils.JSONTrue, "is_classic")
+	listAll(s, modules.Networks.List, params, func(data jsonutils.JSONObject) error {
+		m, err := pingProbeNetwork(s, data, &args.PingProbeOptions)
+		if err != nil {
+			return err
+		}
+		if len(m) > 0 {
+			metrics = append(metrics, m...)
+		}
+		return nil
+	})
+	return sendMetrics(s, metrics, args.Debug)
+}
+
+type sNetwork struct {
+	Id           string
+	Name         string
+	GuestGateway string
+	GuestIpStart string
+	GuestIpEnd   string
+	Region       string
+	RegionId     string
+	ServerType   string
+	Vpc          string
+	VpcId        string
+	Wire         string
+	WireId       string
+	Zone         string
+	ZoneId       string
+	CloudEnv     string
+}
+
+func getNetworkAddrMap(s *mcclient.ClientSession, netId string) (map[string]api.SNetworkUsedAddress, error) {
+	addrListJson, err := modules.Networks.GetSpecific(s, netId, "addresses", nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetSpecific addresses")
+	}
+	addrList := make([]api.SNetworkUsedAddress, 0)
+	err = addrListJson.Unmarshal(&addrList, "addresses")
+	if err != nil {
+		return nil, errors.Wrap(err, "Unmarshal addreses")
+	}
+	addrMap := make(map[string]api.SNetworkUsedAddress)
+	for i := range addrList {
+		addrMap[addrList[i].IpAddr] = addrList[i]
+	}
+	return addrMap, nil
+}
+
+func pingProbeNetwork(s *mcclient.ClientSession, data jsonutils.JSONObject,
+	args *common.PingProbeOptions) ([]influxdb.SMetricData, error) {
+	metrics := make([]influxdb.SMetricData, 0)
+	net := &sNetwork{}
+	err := data.Unmarshal(&net)
+	if err != nil {
+		log.Errorf("Unmarshal network %s: %s", data, err)
+		return nil, errors.Wrap(err, "Unmarshal network")
+	}
+	if net.CloudEnv != api.CLOUD_ENV_ON_PREMISE {
+		log.Errorf("not an onpremise network: %s", data)
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "not onpremise network")
+	}
+	if net.GuestGateway == "" {
+		log.Errorf("no valid gateway %s", data)
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "unreachable network, empty gateway")
+	}
+	addrStart, err := netutils.NewIPV4Addr(net.GuestIpStart)
+	if err != nil {
+		log.Errorf("unmarshal start address %s: %s", net.GuestIpStart, err)
+		return nil, errors.Wrapf(err, "NewIPV4Addr %s", net.GuestIpStart)
+	}
+	addrEnd, err := netutils.NewIPV4Addr(net.GuestIpEnd)
+	if err != nil {
+		log.Errorf("unmarshal end address %s: %s", net.GuestIpEnd, err)
+		return nil, errors.Wrapf(err, "NewIPV4Addr %s", net.GuestIpEnd)
+	}
+	log.Infof("ping address %s - %s", addrStart, addrEnd)
+	pingAddrs := make([]string, 0)
+	for addr := addrStart; addr <= addrEnd; addr = addr.StepUp() {
+		addrStr := addr.String()
+		pingAddrs = append(pingAddrs, addrStr)
+	}
+	pingResults, err := Ping(pingAddrs, args.ProbeCount, time.Second*time.Duration(args.TimeoutSecond), args.Debug)
+	if err != nil {
+		return nil, errors.Wrap(err, "Ping")
+	}
+
+	addrMap, err := getNetworkAddrMap(s, net.Id)
+	if err != nil {
+		return nil, errors.Wrap(err, "getNetworkAddrMap")
+	}
+
+	now := time.Now().UTC()
+	for addr := addrStart; addr <= addrEnd; addr = addr.StepUp() {
+		addrStr := addr.String()
+		pingResult := pingResults[addrStr]
+		netAddr, allocated := addrMap[addrStr]
+		if allocated {
+			if netAddr.OwnerType == api.RESERVEDIP_RESOURCE_TYPES {
+				loss := pingResult.Loss()
+				status := api.RESERVEDIP_STATUS_OFFLINE
+				if loss < 100 {
+					status = api.RESERVEDIP_STATUS_ONLINE
+				}
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewString(status), "status")
+				_, err := modules.ReservedIPs.Update(s, netAddr.OwnerId, params)
+				if err != nil {
+					log.Errorf("update reserved ip %s status fail: %s", addrStr, err)
+				}
+			} else {
+				// send metrics
+				metric := influxdb.SMetricData{}
+				metric.Name = "ping"
+				metric.Timestamp = now
+				metric.Tags = []influxdb.SKeyValue{
+					{
+						Key:   "ip_addr",
+						Value: addrStr,
+					},
+					{
+						Key:   "owner_type",
+						Value: netAddr.OwnerType,
+					},
+					{
+						Key:   "owner_id",
+						Value: netAddr.OwnerId,
+					},
+					{
+						Key:   "owner",
+						Value: netAddr.Owner,
+					},
+				}
+				loss := pingResult.Loss()
+				max, avg, min := pingResult.Rtt()
+				metric.Metrics = []influxdb.SKeyValue{
+					{
+						Key:   "loss",
+						Value: strconv.FormatInt(int64(loss), 10),
+					},
+					{
+						Key:   "rtt_ms_avg",
+						Value: strconv.FormatInt(int64(avg/time.Millisecond), 10),
+					},
+					{
+						Key:   "rtt_ms_max",
+						Value: strconv.FormatInt(int64(max/time.Millisecond), 10),
+					},
+					{
+						Key:   "rtt_ms_min",
+						Value: strconv.FormatInt(int64(min/time.Millisecond), 10),
+					},
+				}
+				metrics = append(metrics, metric)
+			}
+		} else {
+			loss := pingResult.Loss()
+			if loss < 100 {
+				// reserve ip
+				log.Debugf("Free address %s is responding ping, reserve the address", addrStr)
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewStringArray([]string{addrStr}), "ips")
+				params.Add(jsonutils.NewString("ping detect online free IP"), "notes")
+				params.Add(jsonutils.NewString(api.RESERVEDIP_STATUS_ONLINE), "status")
+				_, err := modules.Networks.PerformAction(s, net.Id, "reserve-ip", params)
+				if err != nil {
+					log.Errorf("failed to reserve ip %s: %s", addrStr, err)
+				}
+			}
+		}
+		log.Debugf("%s %s allocated %v", addrStr, netAddr, allocated)
+	}
+	return metrics, nil
+}
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "ping-probe", "Ping probe IPv4 address", pingProbeCoolector)
+}

pkg/cloudmon/collectors/pingutils.go

@@ -0,0 +1,102 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/tatsushid/go-fastping"
+
+	"yunion.io/x/pkg/errors"
+)
+
+type SPingResult struct {
+	addr  string
+	rtt   []time.Duration
+	count int
+}
+
+func NewPingResult(addr string, count int) *SPingResult {
+	return &SPingResult{
+		addr:  addr,
+		rtt:   make([]time.Duration, 0),
+		count: count,
+	}
+}
+
+func (pr *SPingResult) Add(rtt time.Duration) {
+	pr.rtt = append(pr.rtt, rtt)
+}
+
+func (pr SPingResult) Rtt() (min time.Duration, avg time.Duration, max time.Duration) {
+	sum := time.Duration(0)
+	max = time.Duration(-1)
+	min = time.Duration(-1)
+	for _, d := range pr.rtt {
+		sum += d
+		if max < 0 || max < d {
+			max = d
+		}
+		if min < 0 || min > d {
+			min = d
+		}
+	}
+	if len(pr.rtt) > 0 {
+		avg = sum / time.Duration(len(pr.rtt))
+	}
+	return
+}
+
+func (pr SPingResult) Loss() int {
+	return 100 - len(pr.rtt)*100/pr.count
+}
+
+func (pr SPingResult) String() string {
+	min, avg, max := pr.Rtt()
+	return fmt.Sprintf("%d packets transmitted, %d received, %d%% packet loss, rtt min/avg/max = %d/%d/%d ms", pr.count, len(pr.rtt), pr.Loss(), min/time.Millisecond, avg/time.Millisecond, max/time.Millisecond)
+}
+
+func Ping(addrList []string, count int, timeout time.Duration, debug bool) (map[string]*SPingResult, error) {
+	p := fastping.NewPinger()
+	p.MaxRTT = timeout
+	p.Size = 64
+	p.Debug = debug
+	result := make(map[string]*SPingResult)
+	for _, addr := range addrList {
+		result[addr] = NewPingResult(addr, count)
+		p.AddIP(addr)
+	}
+	p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
+		result[addr.String()].Add(rtt)
+	}
+	p.OnIdle = func() {
+	}
+	p.RunLoop()
+	defer p.Stop()
+	ticker := time.NewTicker(time.Duration(count) * timeout)
+	defer ticker.Stop()
+	select {
+	case <-p.Done():
+		if err := p.Err(); err != nil {
+			return nil, errors.Wrap(err, "ping error")
+		}
+		break
+	case <-ticker.C:
+		break
+	}
+	return result, nil
+}

pkg/cloudmon/collectors/pingutils_test.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPing(t *testing.T) {
+	result, err := Ping([]string{
+		"114.114.114.114",
+		"118.187.65.237",
+		"10.168.26.254",
+		"10.168.26.26",
+		"192.30.253.113",
+	}, 5, time.Second, true)
+	if err != nil {
+		// ignore error
+		t.Logf("ping error %s", err)
+	}
+	for k, v := range result {
+		t.Logf("%s: %s", k, v.String())
+	}
+}

pkg/cloudmon/collectors/qcmon/doc.go

@@ -0,0 +1 @@
+package qcmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon"

pkg/cloudmon/collectors/qcmon/qcloudreport.go

@@ -0,0 +1,99 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qcmon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SQCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SQCloudReportFactory struct {
+}
+
+func (self *SQCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SQCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SQCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_QCLOUD
+}
+
+type SQCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SQCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			//err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			//err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}